diff options
| author | vvvv <[email protected]> | 2025-10-08 11:41:14 +0300 |
|---|---|---|
| committer | vvvv <[email protected]> | 2025-10-08 12:20:42 +0300 |
| commit | d73f13cfdb331365ddad0da51ec36e0a3e4cf187 (patch) | |
| tree | 88cab10170ce9aa3389be7f1a09247386dcf5ebd | |
| parent | f377d8ad9e0741cd904c1d4934afdf24af517d93 (diff) | |
YQL-20086 public
commit_hash:68b0c2e9c2960587af7d57ecedcb38f4d05890b7
227 files changed, 12484 insertions, 12171 deletions
diff --git a/yql/essentials/public/decimal/ut/ya.make b/yql/essentials/public/decimal/ut/ya.make index aad0eb010ba..1197400f3f4 100644 --- a/yql/essentials/public/decimal/ut/ya.make +++ b/yql/essentials/public/decimal/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(yql/essentials/public/decimal) +ENABLE(YQL_STYLE_CPP) + SRCS(yql_decimal_ut.cpp yql_wide_int_ut.cpp) END() diff --git a/yql/essentials/public/decimal/ut/yql_decimal_ut.cpp b/yql/essentials/public/decimal/ut/yql_decimal_ut.cpp index c93993190fe..8d9fea6551e 100644 --- a/yql/essentials/public/decimal/ut/yql_decimal_ut.cpp +++ b/yql/essentials/public/decimal/ut/yql_decimal_ut.cpp @@ -6,346 +6,346 @@ namespace NYql { namespace NDecimal { Y_UNIT_TEST_SUITE(TYqlDecimalTest) { - void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const TString& expected) { - TString result = ToString(v, precision, scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - TInt128 parsed = FromString(result, precision, scale); - UNIT_ASSERT(parsed == v); - } +void SimplePositiveTest(TInt128 v, ui8 precision, ui8 scale, const TString& expected) { + TString result = ToString(v, precision, scale); + UNIT_ASSERT_VALUES_EQUAL(result, expected); + TInt128 parsed = FromString(result, precision, scale); + UNIT_ASSERT(parsed == v); +} - void SimpleNegativeFormatTest(TInt128 v, ui8 precision, ui8 scale) { - TString result = ToString(v, precision, scale); - UNIT_ASSERT_VALUES_EQUAL(result, ""); - } +void SimpleNegativeFormatTest(TInt128 v, ui8 precision, ui8 scale) { + TString result = ToString(v, precision, scale); + UNIT_ASSERT_VALUES_EQUAL(result, ""); +} - void SimpleSerializeAndDeserialize(TInt128 v, size_t expectedSize) { - char buff[sizeof(TInt128)]; - const auto s = Serialize(v, buff); - UNIT_ASSERT_VALUES_EQUAL(s, expectedSize); - const auto& des = Deserialize(buff, expectedSize); - UNIT_ASSERT_VALUES_EQUAL(des.second, expectedSize); - UNIT_ASSERT(des.first == v); - const auto& e = Deserialize(buff, expectedSize - 1); - UNIT_ASSERT(e.first == Err()); - } +void SimpleSerializeAndDeserialize(TInt128 v, size_t expectedSize) { + char buff[sizeof(TInt128)]; + const auto s = Serialize(v, buff); + UNIT_ASSERT_VALUES_EQUAL(s, expectedSize); + const auto& des = Deserialize(buff, expectedSize); + UNIT_ASSERT_VALUES_EQUAL(des.second, expectedSize); + UNIT_ASSERT(des.first == v); + const auto& e = Deserialize(buff, expectedSize - 1); + UNIT_ASSERT(e.first == Err()); +} - template<ui8 Precision, ui8 Scale> - void CheckMulAndRescale(const TStringBuf& lhs, const TStringBuf& rhs, const TStringBuf& expected) { - const auto l = FromString(lhs, Precision, Scale); - const auto r = FromString(rhs, Precision, Scale); - const auto m = MulAndDivNormalDivider(l, r, GetDivider<Scale>()); - const auto result = ToString(m, Precision, Scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - } +template <ui8 Precision, ui8 Scale> +void CheckMulAndRescale(const TStringBuf& lhs, const TStringBuf& rhs, const TStringBuf& expected) { + const auto l = FromString(lhs, Precision, Scale); + const auto r = FromString(rhs, Precision, Scale); + const auto m = MulAndDivNormalDivider(l, r, GetDivider<Scale>()); + const auto result = ToString(m, Precision, Scale); + UNIT_ASSERT_VALUES_EQUAL(result, expected); +} - template<ui8 Precision, ui8 Scale> - void CheckDivAndRescale(const TStringBuf& lhs, const TStringBuf& rhs, const TStringBuf& expected) { - const auto l = FromString(lhs, Precision, Scale); - const auto r = FromString(rhs, Precision, Scale); - const auto m = MulAndDivNormalMultiplier(l, GetDivider<Scale>(), r); - const auto result = ToString(m, Precision, Scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - } +template <ui8 Precision, ui8 Scale> +void CheckDivAndRescale(const TStringBuf& lhs, const TStringBuf& rhs, const TStringBuf& expected) { + const auto l = FromString(lhs, Precision, Scale); + const auto r = FromString(rhs, Precision, Scale); + const auto m = MulAndDivNormalMultiplier(l, GetDivider<Scale>(), r); + const auto result = ToString(m, Precision, Scale); + UNIT_ASSERT_VALUES_EQUAL(result, expected); +} - template<ui8 Precision, ui8 Scale = 0> - void CheckMul(const TStringBuf& lhs, const TStringBuf& rhs, const TStringBuf& expected) { - const auto l = FromString(lhs, Precision, Scale); - const auto r = FromString(rhs, Precision, Scale); - const auto m = Mul(l, r); - const auto result = ToString(m, Precision, Scale); - UNIT_ASSERT_VALUES_EQUAL(result, expected); - } +template <ui8 Precision, ui8 Scale = 0> +void CheckMul(const TStringBuf& lhs, const TStringBuf& rhs, const TStringBuf& expected) { + const auto l = FromString(lhs, Precision, Scale); + const auto r = FromString(rhs, Precision, Scale); + const auto m = Mul(l, r); + const auto result = ToString(m, Precision, Scale); + UNIT_ASSERT_VALUES_EQUAL(result, expected); +} - Y_UNIT_TEST(TestZeroFormat) { - UNIT_ASSERT_VALUES_EQUAL(ToString(0, 1, 0), "0"); - UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 6), "0"); - UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 0), "0"); - } +Y_UNIT_TEST(TestZeroFormat) { + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 1, 0), "0"); + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 6), "0"); + UNIT_ASSERT_VALUES_EQUAL(ToString(0, 15, 0), "0"); +} - Y_UNIT_TEST(TestZeroScale) { - SimplePositiveTest(1, 5, 0, "1"); - SimplePositiveTest(10, 5, 0, "10"); - SimplePositiveTest(100, 5, 0, "100"); - SimplePositiveTest(1000, 5, 0, "1000"); - SimplePositiveTest(10000, 5, 0, "10000"); - SimpleNegativeFormatTest(100000, 5, 0); - SimpleNegativeFormatTest(1000000, 5, 0); - - // negative numbers - SimplePositiveTest(-1, 5, 0, "-1"); - SimplePositiveTest(-10, 5, 0, "-10"); - SimplePositiveTest(-100, 5, 0, "-100"); - SimplePositiveTest(-1000, 5, 0, "-1000"); - SimplePositiveTest(-10000, 5, 0, "-10000"); - SimpleNegativeFormatTest(-100000, 5, 0); - } +Y_UNIT_TEST(TestZeroScale) { + SimplePositiveTest(1, 5, 0, "1"); + SimplePositiveTest(10, 5, 0, "10"); + SimplePositiveTest(100, 5, 0, "100"); + SimplePositiveTest(1000, 5, 0, "1000"); + SimplePositiveTest(10000, 5, 0, "10000"); + SimpleNegativeFormatTest(100000, 5, 0); + SimpleNegativeFormatTest(1000000, 5, 0); + + // negative numbers + SimplePositiveTest(-1, 5, 0, "-1"); + SimplePositiveTest(-10, 5, 0, "-10"); + SimplePositiveTest(-100, 5, 0, "-100"); + SimplePositiveTest(-1000, 5, 0, "-1000"); + SimplePositiveTest(-10000, 5, 0, "-10000"); + SimpleNegativeFormatTest(-100000, 5, 0); +} - Y_UNIT_TEST(TestFormats) { - // we have no trailing zeros - SimplePositiveTest(1, 15, 6, "0.000001"); - SimplePositiveTest(10, 15, 6, "0.00001"); - SimplePositiveTest(100, 15, 6, "0.0001"); - SimplePositiveTest(1000, 15, 6, "0.001"); - SimplePositiveTest(10000, 15, 6, "0.01"); - SimplePositiveTest(100000, 15, 6, "0.1"); - SimplePositiveTest(1000000, 15, 6, "1"); - SimplePositiveTest(10000000, 15, 6, "10"); - SimplePositiveTest(100000000, 15, 6, "100"); - - SimplePositiveTest(2020000, 15, 6, "2.02"); - SimplePositiveTest(3003000, 15, 6, "3.003"); - - // negative numbers - SimplePositiveTest(-1, 15, 6, "-0.000001"); - SimplePositiveTest(-10, 15, 6, "-0.00001"); - SimplePositiveTest(-100, 15, 6, "-0.0001"); - SimplePositiveTest(-1000, 15, 6, "-0.001"); - SimplePositiveTest(-10000, 15, 6, "-0.01"); - SimplePositiveTest(-100000, 15, 6, "-0.1"); - SimplePositiveTest(-1000000, 15, 6, "-1"); - SimplePositiveTest(-10000000, 15, 6, "-10"); - SimplePositiveTest(-100000000, 15, 6, "-100"); - - SimplePositiveTest(-2020000, 15, 6, "-2.02"); - SimplePositiveTest(-3003000, 15, 6, "-3.003"); - - SimplePositiveTest(1, 15, 6, "0.000001"); - SimplePositiveTest(12, 15, 6, "0.000012"); - SimplePositiveTest(123, 15, 6, "0.000123"); - SimplePositiveTest(1234, 15, 6, "0.001234"); - SimplePositiveTest(12345, 15, 6, "0.012345"); - SimplePositiveTest(123456, 15, 6, "0.123456"); - SimplePositiveTest(1234567, 15, 6, "1.234567"); - SimplePositiveTest(12345678, 15, 6, "12.345678"); - SimplePositiveTest(123456789, 15, 6, "123.456789"); - SimplePositiveTest(1234567898, 15, 6, "1234.567898"); - SimplePositiveTest(12345678987ll, 15, 6, "12345.678987"); - SimplePositiveTest(123456789876ll, 15, 6, "123456.789876"); - } +Y_UNIT_TEST(TestFormats) { + // we have no trailing zeros + SimplePositiveTest(1, 15, 6, "0.000001"); + SimplePositiveTest(10, 15, 6, "0.00001"); + SimplePositiveTest(100, 15, 6, "0.0001"); + SimplePositiveTest(1000, 15, 6, "0.001"); + SimplePositiveTest(10000, 15, 6, "0.01"); + SimplePositiveTest(100000, 15, 6, "0.1"); + SimplePositiveTest(1000000, 15, 6, "1"); + SimplePositiveTest(10000000, 15, 6, "10"); + SimplePositiveTest(100000000, 15, 6, "100"); + + SimplePositiveTest(2020000, 15, 6, "2.02"); + SimplePositiveTest(3003000, 15, 6, "3.003"); + + // negative numbers + SimplePositiveTest(-1, 15, 6, "-0.000001"); + SimplePositiveTest(-10, 15, 6, "-0.00001"); + SimplePositiveTest(-100, 15, 6, "-0.0001"); + SimplePositiveTest(-1000, 15, 6, "-0.001"); + SimplePositiveTest(-10000, 15, 6, "-0.01"); + SimplePositiveTest(-100000, 15, 6, "-0.1"); + SimplePositiveTest(-1000000, 15, 6, "-1"); + SimplePositiveTest(-10000000, 15, 6, "-10"); + SimplePositiveTest(-100000000, 15, 6, "-100"); + + SimplePositiveTest(-2020000, 15, 6, "-2.02"); + SimplePositiveTest(-3003000, 15, 6, "-3.003"); + + SimplePositiveTest(1, 15, 6, "0.000001"); + SimplePositiveTest(12, 15, 6, "0.000012"); + SimplePositiveTest(123, 15, 6, "0.000123"); + SimplePositiveTest(1234, 15, 6, "0.001234"); + SimplePositiveTest(12345, 15, 6, "0.012345"); + SimplePositiveTest(123456, 15, 6, "0.123456"); + SimplePositiveTest(1234567, 15, 6, "1.234567"); + SimplePositiveTest(12345678, 15, 6, "12.345678"); + SimplePositiveTest(123456789, 15, 6, "123.456789"); + SimplePositiveTest(1234567898, 15, 6, "1234.567898"); + SimplePositiveTest(12345678987ll, 15, 6, "12345.678987"); + SimplePositiveTest(123456789876ll, 15, 6, "123456.789876"); +} - Y_UNIT_TEST(TestHugeNumberFormat) { - TInt128 max120 = Inf() - 1; - const char max120String[] = "99999999999999999999999999999999999"; // 35 digits - static_assert(sizeof(max120String) == 36, "sizeof(max120String) == 36"); - SimplePositiveTest(max120, MaxPrecision, 0, max120String); - SimplePositiveTest(max120 + 1, MaxPrecision, 0, "inf"); - - TInt128 min120 = -Inf() + 1; - const char min120String[] = "-99999999999999999999999999999999999"; - static_assert(sizeof(min120String) == 37, "sizeof(min120String) == 37"); - SimplePositiveTest(min120, MaxPrecision, 0, min120String); - SimplePositiveTest(min120 - 1, MaxPrecision, 0, "-inf"); - - // take spot for sign and zero before dot - const char min120StringAfterDot[] = "-0.99999999999999999999999999999999999"; // 35 by nine + leading zero - static_assert(sizeof(min120StringAfterDot) == 39, "sizeof(min120StringAfterDot) == 39"); - SimplePositiveTest(min120, MaxPrecision, MaxPrecision, min120StringAfterDot); - - SimpleNegativeFormatTest(1, MaxPrecision + 1, MaxPrecision + 1); - SimpleNegativeFormatTest(1, MaxPrecision + 1, 0); - SimpleNegativeFormatTest(1, 2, 3); - } +Y_UNIT_TEST(TestHugeNumberFormat) { + TInt128 max120 = Inf() - 1; + const char max120String[] = "99999999999999999999999999999999999"; // 35 digits + static_assert(sizeof(max120String) == 36, "sizeof(max120String) == 36"); + SimplePositiveTest(max120, MaxPrecision, 0, max120String); + SimplePositiveTest(max120 + 1, MaxPrecision, 0, "inf"); + + TInt128 min120 = -Inf() + 1; + const char min120String[] = "-99999999999999999999999999999999999"; + static_assert(sizeof(min120String) == 37, "sizeof(min120String) == 37"); + SimplePositiveTest(min120, MaxPrecision, 0, min120String); + SimplePositiveTest(min120 - 1, MaxPrecision, 0, "-inf"); + + // take spot for sign and zero before dot + const char min120StringAfterDot[] = "-0.99999999999999999999999999999999999"; // 35 by nine + leading zero + static_assert(sizeof(min120StringAfterDot) == 39, "sizeof(min120StringAfterDot) == 39"); + SimplePositiveTest(min120, MaxPrecision, MaxPrecision, min120StringAfterDot); + + SimpleNegativeFormatTest(1, MaxPrecision + 1, MaxPrecision + 1); + SimpleNegativeFormatTest(1, MaxPrecision + 1, 0); + SimpleNegativeFormatTest(1, 2, 3); +} - Y_UNIT_TEST(TestFormStringRoundToEven) { - UNIT_ASSERT(FromString(".51", 1, 0) == 1); - UNIT_ASSERT(FromString("-0.51", 1, 0) == -1); +Y_UNIT_TEST(TestFormStringRoundToEven) { + UNIT_ASSERT(FromString(".51", 1, 0) == 1); + UNIT_ASSERT(FromString("-0.51", 1, 0) == -1); - UNIT_ASSERT(FromString("+00000008.5", 1, 0) == 8); - UNIT_ASSERT(FromString("-8.5000000000000000000000000000000", 1, 0) == -8); + UNIT_ASSERT(FromString("+00000008.5", 1, 0) == 8); + UNIT_ASSERT(FromString("-8.5000000000000000000000000000000", 1, 0) == -8); - UNIT_ASSERT(FromString("00008.51", 1, 0) == 9); - UNIT_ASSERT(FromString("-8.5000000000000000000000000000001", 1, 0) == -9); + UNIT_ASSERT(FromString("00008.51", 1, 0) == 9); + UNIT_ASSERT(FromString("-8.5000000000000000000000000000001", 1, 0) == -9); - UNIT_ASSERT(FromString("09.499999999999999999999999999999999999999999999999999999999", 1, 0) == 9); - UNIT_ASSERT(FromString("-9.499999999999999999999999999999999999999999999999999999999", 1, 0) == -9); + UNIT_ASSERT(FromString("09.499999999999999999999999999999999999999999999999999999999", 1, 0) == 9); + UNIT_ASSERT(FromString("-9.499999999999999999999999999999999999999999999999999999999", 1, 0) == -9); - UNIT_ASSERT(FromString("9.50", 2, 0) == 10); - UNIT_ASSERT(FromString("-9.5", 2, 0) == -10); + UNIT_ASSERT(FromString("9.50", 2, 0) == 10); + UNIT_ASSERT(FromString("-9.5", 2, 0) == -10); - UNIT_ASSERT(FromString("+0.9949", 2, 2) == 99); - UNIT_ASSERT(FromString("-0.9949", 2, 2) == -99); - } + UNIT_ASSERT(FromString("+0.9949", 2, 2) == 99); + UNIT_ASSERT(FromString("-0.9949", 2, 2) == -99); +} - Y_UNIT_TEST(TestInfinityValues) { - UNIT_ASSERT(FromString("+1", 1, 1) == Inf()); - UNIT_ASSERT(FromString("-1", 1, 1) == -Inf()); +Y_UNIT_TEST(TestInfinityValues) { + UNIT_ASSERT(FromString("+1", 1, 1) == Inf()); + UNIT_ASSERT(FromString("-1", 1, 1) == -Inf()); - UNIT_ASSERT(FromString("10.000", 1, 0) == Inf()); - UNIT_ASSERT(FromString("-10.000", 1, 0) == -Inf()); + UNIT_ASSERT(FromString("10.000", 1, 0) == Inf()); + UNIT_ASSERT(FromString("-10.000", 1, 0) == -Inf()); - UNIT_ASSERT(FromString("9.500", 1, 0) == Inf()); - UNIT_ASSERT(FromString("-9.500", 1, 0) == -Inf()); + UNIT_ASSERT(FromString("9.500", 1, 0) == Inf()); + UNIT_ASSERT(FromString("-9.500", 1, 0) == -Inf()); - UNIT_ASSERT(FromString("+0.950", 1, 1) == Inf()); - UNIT_ASSERT(FromString("-0.950", 1, 1) == -Inf()); + UNIT_ASSERT(FromString("+0.950", 1, 1) == Inf()); + UNIT_ASSERT(FromString("-0.950", 1, 1) == -Inf()); - UNIT_ASSERT(FromString("+0.9950", 2, 2) == Inf()); - UNIT_ASSERT(FromString("-0.9950", 2, 2) == -Inf()); + UNIT_ASSERT(FromString("+0.9950", 2, 2) == Inf()); + UNIT_ASSERT(FromString("-0.9950", 2, 2) == -Inf()); - UNIT_ASSERT(FromString("9999999999999999999999999999999999999.5", 35, 0) == Inf()); - UNIT_ASSERT(FromString("-9999999999999999999999999999999999999.5", 35, 0) == -Inf()); - } + UNIT_ASSERT(FromString("9999999999999999999999999999999999999.5", 35, 0) == Inf()); + UNIT_ASSERT(FromString("-9999999999999999999999999999999999999.5", 35, 0) == -Inf()); +} - Y_UNIT_TEST(TestInvalidValues) { - UNIT_ASSERT(IsValid("+999999999999999991234567890.039493804903849038490312345678909999999999999999990")); +Y_UNIT_TEST(TestInvalidValues) { + UNIT_ASSERT(IsValid("+999999999999999991234567890.039493804903849038490312345678909999999999999999990")); - UNIT_ASSERT(!IsValid("")); // empty - UNIT_ASSERT(!IsValid("12.2.3")); // double dot - UNIT_ASSERT(!IsValid("+-12")); // extra sign - UNIT_ASSERT(!IsValid("463786378O74674")); // letter inside + UNIT_ASSERT(!IsValid("")); // empty + UNIT_ASSERT(!IsValid("12.2.3")); // double dot + UNIT_ASSERT(!IsValid("+-12")); // extra sign + UNIT_ASSERT(!IsValid("463786378O74674")); // letter inside - UNIT_ASSERT(IsError(FromString("", 35, 15))); // empty - UNIT_ASSERT(IsError(FromString("12.2.3", 35, 15))); // double dot - UNIT_ASSERT(IsError(FromString("+-12", 35, 15))); // extra sign - UNIT_ASSERT(IsError(FromString("463786378O74674", 35, 15))); // letter inside - UNIT_ASSERT(IsError(FromString("+7.039493804E1", 35, 5))); // letter in tail after scale - } + UNIT_ASSERT(IsError(FromString("", 35, 15))); // empty + UNIT_ASSERT(IsError(FromString("12.2.3", 35, 15))); // double dot + UNIT_ASSERT(IsError(FromString("+-12", 35, 15))); // extra sign + UNIT_ASSERT(IsError(FromString("463786378O74674", 35, 15))); // letter inside + UNIT_ASSERT(IsError(FromString("+7.039493804E1", 35, 5))); // letter in tail after scale +} - Y_UNIT_TEST(TestFormStringEx) { - UNIT_ASSERT(FromStringEx("NAN", 13, 1) == Nan()); - UNIT_ASSERT(FromStringEx("+inf", 11, 7) == Inf()); - UNIT_ASSERT(FromStringEx("-inf", 7, 7) == -Inf()); - - UNIT_ASSERT(FromStringEx("0.1E3", 10, 1) == 1000); - UNIT_ASSERT(FromStringEx("0.51e-3", 10, 3) == 1); - - UNIT_ASSERT(FromStringEx("1E30", 10, 0) == Inf()); - UNIT_ASSERT(FromStringEx("1e-30", 10, 0) == 0); - UNIT_ASSERT(FromStringEx("-1E+99", 10, 2) == -Inf()); - UNIT_ASSERT(FromStringEx("-1e-99", 10, 2) == 0); - UNIT_ASSERT(FromStringEx("-510e-3", 1, 0) == -1); - UNIT_ASSERT(FromStringEx("+99E3", 5, 0) == 99000); - UNIT_ASSERT(FromStringEx("2.1E-130", 35, 2) == 0); - UNIT_ASSERT(FromStringEx("2.1E0", 35, 2) == 210); - } +Y_UNIT_TEST(TestFormStringEx) { + UNIT_ASSERT(FromStringEx("NAN", 13, 1) == Nan()); + UNIT_ASSERT(FromStringEx("+inf", 11, 7) == Inf()); + UNIT_ASSERT(FromStringEx("-inf", 7, 7) == -Inf()); + + UNIT_ASSERT(FromStringEx("0.1E3", 10, 1) == 1000); + UNIT_ASSERT(FromStringEx("0.51e-3", 10, 3) == 1); + + UNIT_ASSERT(FromStringEx("1E30", 10, 0) == Inf()); + UNIT_ASSERT(FromStringEx("1e-30", 10, 0) == 0); + UNIT_ASSERT(FromStringEx("-1E+99", 10, 2) == -Inf()); + UNIT_ASSERT(FromStringEx("-1e-99", 10, 2) == 0); + UNIT_ASSERT(FromStringEx("-510e-3", 1, 0) == -1); + UNIT_ASSERT(FromStringEx("+99E3", 5, 0) == 99000); + UNIT_ASSERT(FromStringEx("2.1E-130", 35, 2) == 0); + UNIT_ASSERT(FromStringEx("2.1E0", 35, 2) == 210); +} - Y_UNIT_TEST(TestFormStringExInvalidValues) { - UNIT_ASSERT(IsError(FromStringEx("", 35, 15))); // empty - UNIT_ASSERT(IsError(FromStringEx("12.2.3", 35, 15))); // double dot - UNIT_ASSERT(IsError(FromStringEx("+-12", 35, 15))); // extra sign - UNIT_ASSERT(IsError(FromStringEx("463786378O74674", 35, 15))); // letter inside - - UNIT_ASSERT(IsError(FromStringEx("E2", 35, 15))); // empty - UNIT_ASSERT(IsError(FromStringEx("E2E4", 35, 15))); // empty - UNIT_ASSERT(IsError(FromStringEx("NANE5", 35, 15))); // nan with exp - UNIT_ASSERT(IsError(FromStringEx("infE5", 35, 15))); // inf with exp - UNIT_ASSERT(IsError(FromStringEx("-infe-5", 35, 15))); // inf with exp - UNIT_ASSERT(IsError(FromStringEx("2.1E0X", 35, 2))); // not fully parsed exp - UNIT_ASSERT(IsError(FromStringEx("2.1E+-1", 35, 2))); // two signs - UNIT_ASSERT(IsError(FromStringEx("ae30", 10, 0))); // bad mantissa - } +Y_UNIT_TEST(TestFormStringExInvalidValues) { + UNIT_ASSERT(IsError(FromStringEx("", 35, 15))); // empty + UNIT_ASSERT(IsError(FromStringEx("12.2.3", 35, 15))); // double dot + UNIT_ASSERT(IsError(FromStringEx("+-12", 35, 15))); // extra sign + UNIT_ASSERT(IsError(FromStringEx("463786378O74674", 35, 15))); // letter inside + + UNIT_ASSERT(IsError(FromStringEx("E2", 35, 15))); // empty + UNIT_ASSERT(IsError(FromStringEx("E2E4", 35, 15))); // empty + UNIT_ASSERT(IsError(FromStringEx("NANE5", 35, 15))); // nan with exp + UNIT_ASSERT(IsError(FromStringEx("infE5", 35, 15))); // inf with exp + UNIT_ASSERT(IsError(FromStringEx("-infe-5", 35, 15))); // inf with exp + UNIT_ASSERT(IsError(FromStringEx("2.1E0X", 35, 2))); // not fully parsed exp + UNIT_ASSERT(IsError(FromStringEx("2.1E+-1", 35, 2))); // two signs + UNIT_ASSERT(IsError(FromStringEx("ae30", 10, 0))); // bad mantissa +} - Y_UNIT_TEST(TestSpecialAsString) { - UNIT_ASSERT(IsValid("Nan")); - UNIT_ASSERT(IsValid("INF")); - UNIT_ASSERT(IsValid("-inf")); +Y_UNIT_TEST(TestSpecialAsString) { + UNIT_ASSERT(IsValid("Nan")); + UNIT_ASSERT(IsValid("INF")); + UNIT_ASSERT(IsValid("-inf")); - UNIT_ASSERT_VALUES_EQUAL(ToString(Nan(), 10, 2), "nan"); + UNIT_ASSERT_VALUES_EQUAL(ToString(Nan(), 10, 2), "nan"); - UNIT_ASSERT_VALUES_EQUAL(ToString(+Inf(), 10, 2), "inf"); - UNIT_ASSERT_VALUES_EQUAL(ToString(-Inf(), 10, 2), "-inf"); + UNIT_ASSERT_VALUES_EQUAL(ToString(+Inf(), 10, 2), "inf"); + UNIT_ASSERT_VALUES_EQUAL(ToString(-Inf(), 10, 2), "-inf"); + + UNIT_ASSERT(IsNan(FromString("nan", 10, 2))); + UNIT_ASSERT(IsInf(FromString("+INf", MaxPrecision, 6))); + UNIT_ASSERT(IsInf(FromString("-inF", 4, 2))); +} - UNIT_ASSERT(IsNan(FromString("nan", 10, 2))); - UNIT_ASSERT(IsInf(FromString("+INf", MaxPrecision, 6))); - UNIT_ASSERT(IsInf(FromString("-inF", 4, 2))); +Y_UNIT_TEST(TestToStringOfNonNormal) { + // above Inf + for (TInt128 i = Inf() + 2, end = Inf() + 100; i < end; i++) { + UNIT_ASSERT(!IsNormal(i)); + UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); } - Y_UNIT_TEST(TestToStringOfNonNormal) { - // above Inf - for (TInt128 i = Inf() + 2, end = Inf() + 100; i < end; i++) { - UNIT_ASSERT(!IsNormal(i)); - UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); - } - - // below -Inf - for (TInt128 i = -Inf() - 2, end = -Inf() - 100; i < end; i--) { - UNIT_ASSERT(!IsNormal(i)); - UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); - } + // below -Inf + for (TInt128 i = -Inf() - 2, end = -Inf() - 100; i < end; i--) { + UNIT_ASSERT(!IsNormal(i)); + UNIT_ASSERT(ToString(i, MaxPrecision, 0) == nullptr); } +} - Y_UNIT_TEST(TestSerializeAndDeserialize) { - SimpleSerializeAndDeserialize(-Nan(), 1U); - SimpleSerializeAndDeserialize(-Inf(), 1U); +Y_UNIT_TEST(TestSerializeAndDeserialize) { + SimpleSerializeAndDeserialize(-Nan(), 1U); + SimpleSerializeAndDeserialize(-Inf(), 1U); - SimpleSerializeAndDeserialize(-Inf() + 1, 16U); - SimpleSerializeAndDeserialize(-Inf() + 2, 16U); + SimpleSerializeAndDeserialize(-Inf() + 1, 16U); + SimpleSerializeAndDeserialize(-Inf() + 2, 16U); - SimpleSerializeAndDeserialize(-65537, 4U); - SimpleSerializeAndDeserialize(-65536, 3U); + SimpleSerializeAndDeserialize(-65537, 4U); + SimpleSerializeAndDeserialize(-65536, 3U); - SimpleSerializeAndDeserialize(-257, 3U); - SimpleSerializeAndDeserialize(-256, 2U); + SimpleSerializeAndDeserialize(-257, 3U); + SimpleSerializeAndDeserialize(-256, 2U); - SimpleSerializeAndDeserialize(-3, 2U); - SimpleSerializeAndDeserialize(-2, 2U); + SimpleSerializeAndDeserialize(-3, 2U); + SimpleSerializeAndDeserialize(-2, 2U); - SimpleSerializeAndDeserialize(-1, 1U); - SimpleSerializeAndDeserialize(0, 1U); + SimpleSerializeAndDeserialize(-1, 1U); + SimpleSerializeAndDeserialize(0, 1U); - SimpleSerializeAndDeserialize(+1, 2U); - SimpleSerializeAndDeserialize(+2, 2U); + SimpleSerializeAndDeserialize(+1, 2U); + SimpleSerializeAndDeserialize(+2, 2U); - SimpleSerializeAndDeserialize(+255, 2U); - SimpleSerializeAndDeserialize(+256, 3U); + SimpleSerializeAndDeserialize(+255, 2U); + SimpleSerializeAndDeserialize(+256, 3U); - SimpleSerializeAndDeserialize(+65535, 3U); - SimpleSerializeAndDeserialize(+65536, 4U); + SimpleSerializeAndDeserialize(+65535, 3U); + SimpleSerializeAndDeserialize(+65536, 4U); - SimpleSerializeAndDeserialize(+Inf() - 2, 16U); - SimpleSerializeAndDeserialize(+Inf() - 1, 16U); + SimpleSerializeAndDeserialize(+Inf() - 2, 16U); + SimpleSerializeAndDeserialize(+Inf() - 1, 16U); - SimpleSerializeAndDeserialize(+Inf(), 1U); - SimpleSerializeAndDeserialize(+Nan(), 1U); - } + SimpleSerializeAndDeserialize(+Inf(), 1U); + SimpleSerializeAndDeserialize(+Nan(), 1U); +} - Y_UNIT_TEST(TestMulAndRescale) { - CheckMulAndRescale<35,35>("0.99999999999999999999999999999999999", "-0.99999999999999999999999999999999999", "-0.99999999999999999999999999999999998"); - CheckMulAndRescale<35,35>("-0.99999999999999999999999999999999999", "0.33333333333333333333333333333333333", "-0.33333333333333333333333333333333333"); - CheckMulAndRescale<35,35>("0.33333333333333333333333333333333333", "0.33333333333333333333333333333333333", "0.11111111111111111111111111111111111"); - CheckMulAndRescale<35,35>("0.99999999999999999999999999999999999", "0.000000000000001", "0.000000000000001"); - CheckMulAndRescale<35,35>("0.99999999999999999999999999999999999", "0.00000000000000101010101", "0.00000000000000101010101"); - CheckMulAndRescale<35,35>("0.12345678901234567890123456789012345", "0.12345678901234567890123456789012345", "0.01524157875323883675049535156256668"); - - CheckMulAndRescale<35,34>("9.9999999999999999999999999999999999", "-1.9999999999999999999999999999999999", "-inf"); - CheckMulAndRescale<35,34>("3.3333333333333333333333333333333333", "3.3333333333333333333333333333333333", "inf"); - CheckMulAndRescale<35,34>("3.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "4.4444444444444444444444444444444443"); - CheckMulAndRescale<35,34>("-1.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "-1.7777777777777777777777777777777777"); - - CheckMulAndRescale<35,34>("-7", "0", "0"); - CheckMulAndRescale<35,34>("inf", "nan", "nan"); - CheckMulAndRescale<35,34>("inf", "0", "nan"); - CheckMulAndRescale<35,34>("-inf", "-inf", "inf"); - } +Y_UNIT_TEST(TestMulAndRescale) { + CheckMulAndRescale<35, 35>("0.99999999999999999999999999999999999", "-0.99999999999999999999999999999999999", "-0.99999999999999999999999999999999998"); + CheckMulAndRescale<35, 35>("-0.99999999999999999999999999999999999", "0.33333333333333333333333333333333333", "-0.33333333333333333333333333333333333"); + CheckMulAndRescale<35, 35>("0.33333333333333333333333333333333333", "0.33333333333333333333333333333333333", "0.11111111111111111111111111111111111"); + CheckMulAndRescale<35, 35>("0.99999999999999999999999999999999999", "0.000000000000001", "0.000000000000001"); + CheckMulAndRescale<35, 35>("0.99999999999999999999999999999999999", "0.00000000000000101010101", "0.00000000000000101010101"); + CheckMulAndRescale<35, 35>("0.12345678901234567890123456789012345", "0.12345678901234567890123456789012345", "0.01524157875323883675049535156256668"); + + CheckMulAndRescale<35, 34>("9.9999999999999999999999999999999999", "-1.9999999999999999999999999999999999", "-inf"); + CheckMulAndRescale<35, 34>("3.3333333333333333333333333333333333", "3.3333333333333333333333333333333333", "inf"); + CheckMulAndRescale<35, 34>("3.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "4.4444444444444444444444444444444443"); + CheckMulAndRescale<35, 34>("-1.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "-1.7777777777777777777777777777777777"); + + CheckMulAndRescale<35, 34>("-7", "0", "0"); + CheckMulAndRescale<35, 34>("inf", "nan", "nan"); + CheckMulAndRescale<35, 34>("inf", "0", "nan"); + CheckMulAndRescale<35, 34>("-inf", "-inf", "inf"); +} - Y_UNIT_TEST(TestDivAndRescale) { - CheckDivAndRescale<35,35>("-0.99999999999999999999999999999999999", "0.33333333333333333333333333333333333", "-inf"); - CheckDivAndRescale<35,35>("0.33333333333333333333333333333333333", "-0.33333333333333333333333333333333333", "-inf"); - CheckDivAndRescale<35,35>("0.12345678901234567890123456789012345", "0.12345678901234567890123456789012345", "inf"); +Y_UNIT_TEST(TestDivAndRescale) { + CheckDivAndRescale<35, 35>("-0.99999999999999999999999999999999999", "0.33333333333333333333333333333333333", "-inf"); + CheckDivAndRescale<35, 35>("0.33333333333333333333333333333333333", "-0.33333333333333333333333333333333333", "-inf"); + CheckDivAndRescale<35, 35>("0.12345678901234567890123456789012345", "0.12345678901234567890123456789012345", "inf"); - CheckDivAndRescale<35,34>("9.9999999999999999999999999999999999", "-1.9999999999999999999999999999999999", "-5.0000000000000000000000000000000002"); - CheckDivAndRescale<35,34>("3.3333333333333333333333333333333333", "3.3333333333333333333333333333333333", "1"); - CheckDivAndRescale<35,34>("3.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "2.5"); - CheckDivAndRescale<35,34>("-1.7777777777777777777777777777777777", "1.3333333333333333333333333333333333", "-1.3333333333333333333333333333333333"); + CheckDivAndRescale<35, 34>("9.9999999999999999999999999999999999", "-1.9999999999999999999999999999999999", "-5.0000000000000000000000000000000002"); + CheckDivAndRescale<35, 34>("3.3333333333333333333333333333333333", "3.3333333333333333333333333333333333", "1"); + CheckDivAndRescale<35, 34>("3.3333333333333333333333333333333333", "1.3333333333333333333333333333333333", "2.5"); + CheckDivAndRescale<35, 34>("-1.7777777777777777777777777777777777", "1.3333333333333333333333333333333333", "-1.3333333333333333333333333333333333"); - CheckDivAndRescale<35,34>("-7", "0", "-inf"); - CheckDivAndRescale<35,34>("inf", "0", "inf"); - CheckDivAndRescale<35,34>("inf", "0", "inf"); - CheckDivAndRescale<35,34>("-inf", "inf", "nan"); - } + CheckDivAndRescale<35, 34>("-7", "0", "-inf"); + CheckDivAndRescale<35, 34>("inf", "0", "inf"); + CheckDivAndRescale<35, 34>("inf", "0", "inf"); + CheckDivAndRescale<35, 34>("-inf", "inf", "nan"); +} - Y_UNIT_TEST(TestWideMul) { - CheckMul<35>("999999999999999", "99999999999999999999", "99999999999999899999000000000000001"); - CheckMul<35>("9999999999999999", "99999999999999999999", "inf"); - CheckMul<35>("-99999999999999999999999999999999999", "10000000000000000000000000000000000", "-inf"); - CheckMul<35>("-99999999999999999999999999999999999", "-1", "99999999999999999999999999999999999"); - CheckMul<35>("-99999999999999999999999999999999999", "-2", "inf"); +Y_UNIT_TEST(TestWideMul) { + CheckMul<35>("999999999999999", "99999999999999999999", "99999999999999899999000000000000001"); + CheckMul<35>("9999999999999999", "99999999999999999999", "inf"); + CheckMul<35>("-99999999999999999999999999999999999", "10000000000000000000000000000000000", "-inf"); + CheckMul<35>("-99999999999999999999999999999999999", "-1", "99999999999999999999999999999999999"); + CheckMul<35>("-99999999999999999999999999999999999", "-2", "inf"); - CheckMul<35>("nan", "0", "nan"); - CheckMul<35>("inf", "-inf", "-inf"); - CheckMul<35>("inf", "nan", "nan"); - } + CheckMul<35>("nan", "0", "nan"); + CheckMul<35>("inf", "-inf", "-inf"); + CheckMul<35>("inf", "nan", "nan"); } +} // Y_UNIT_TEST_SUITE(TYqlDecimalTest) -} -} +} // namespace NDecimal +} // namespace NYql diff --git a/yql/essentials/public/decimal/ut/yql_wide_int_ut.cpp b/yql/essentials/public/decimal/ut/yql_wide_int_ut.cpp index 5238249da17..16b5ad68c77 100644 --- a/yql/essentials/public/decimal/ut/yql_wide_int_ut.cpp +++ b/yql/essentials/public/decimal/ut/yql_wide_int_ut.cpp @@ -3,463 +3,455 @@ namespace NYql { Y_UNIT_TEST_SUITE(TYqlWideIntTest) { - template<typename T> - void TestUnary(const T aa) { - using Test = TWide<typename THalfOf<T>::Type>; - const Test at(aa); - static_assert(sizeof(at) == sizeof(aa), "Bad wide int size!"); - - UNIT_ASSERT_VALUES_EQUAL(static_cast<i8>(aa), static_cast<i8>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<ui8>(aa), static_cast<ui8>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<i16>(aa), static_cast<i16>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<ui16>(aa), static_cast<ui16>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<i32>(aa), static_cast<i32>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(aa), static_cast<ui32>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<i64>(aa), static_cast<i64>(at)); - UNIT_ASSERT_VALUES_EQUAL(static_cast<ui64>(aa), static_cast<ui64>(at)); +template <typename T> +void TestUnary(const T aa) { + using Test = TWide<typename THalfOf<T>::Type>; + const Test at(aa); + static_assert(sizeof(at) == sizeof(aa), "Bad wide int size!"); + + UNIT_ASSERT_VALUES_EQUAL(static_cast<i8>(aa), static_cast<i8>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui8>(aa), static_cast<ui8>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<i16>(aa), static_cast<i16>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui16>(aa), static_cast<ui16>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<i32>(aa), static_cast<i32>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(aa), static_cast<ui32>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<i64>(aa), static_cast<i64>(at)); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui64>(aa), static_cast<ui64>(at)); #ifndef _win_ - UNIT_ASSERT(static_cast<i128_t>(aa) == static_cast<i128_t>(at)); - UNIT_ASSERT(static_cast<ui128_t>(aa) == static_cast<ui128_t>(at)); + UNIT_ASSERT(static_cast<i128_t>(aa) == static_cast<i128_t>(at)); + UNIT_ASSERT(static_cast<ui128_t>(aa) == static_cast<ui128_t>(at)); #endif - { - const auto exp = ~aa; - const auto tst = ~at; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = +aa; - const auto tst = +at; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = -aa; - const auto tst = -at; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - auto exp = aa; - auto tst = at; - ++exp; - ++tst; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - auto exp = aa; - auto tst = at; - --exp; - --tst; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - auto exp = aa; - auto tst = at; - exp++; - tst++; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - auto exp = aa; - auto tst = at; - exp--; - tst--; - UNIT_ASSERT(T(tst) == T(exp)); - } + { + const auto exp = ~aa; + const auto tst = ~at; + UNIT_ASSERT(T(tst) == T(exp)); } - template<typename T> - void TestBinary(const T ll, const T rr) { - using Test = TWide<typename THalfOf<T>::Type>; - const Test lt(ll), rt(rr); - - { - const auto exp = ll & rr; - const auto tst = lt & rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = ll | rr; - const auto tst = lt | rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = ll ^ rr; - const auto tst = lt ^ rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = ll + rr; - const auto tst = lt + rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = ll - rr; - const auto tst = lt - rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - if (rr > 0 && rr < T(sizeof(T) << 3U)) - { - const auto exp = ll >> rr; - const auto tst = lt >> rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - if (rr > 0 && rr < T(sizeof(T) << 3U)) - { - const auto exp = ll << rr; - const auto tst = lt << rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = ll * rr; - const auto tst = lt * rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - if (rr) - { - const auto exp = ll / rr; - const auto tst = lt / rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - if (rr) - { - const auto exp = ll % rr; - const auto tst = lt % rt; - UNIT_ASSERT(T(tst) == T(exp)); - } - - { - const auto exp = ll == rr; - const auto tst = lt == rt; - UNIT_ASSERT_VALUES_EQUAL(tst, exp); - } - - { - const auto exp = ll != rr; - const auto tst = lt != rt; - UNIT_ASSERT_VALUES_EQUAL(tst, exp); - } - - { - const auto exp = ll > rr; - const auto tst = lt > rt; - UNIT_ASSERT_VALUES_EQUAL(tst, exp); - } - - { - const auto exp = ll < rr; - const auto tst = lt < rt; - UNIT_ASSERT_VALUES_EQUAL(tst, exp); - } - - { - const auto exp = ll >= rr; - const auto tst = lt >= rt; - UNIT_ASSERT_VALUES_EQUAL(tst, exp); - } - - { - const auto exp = ll <= rr; - const auto tst = lt <= rt; - UNIT_ASSERT_VALUES_EQUAL(tst, exp); - } + { + const auto exp = +aa; + const auto tst = +at; + UNIT_ASSERT(T(tst) == T(exp)); } - template<typename T> - void TestsForUnsignedType() { - static_assert(std::is_unsigned<T>::value, "Tests for unsigned type."); - TestUnary<T>(2U); - TestUnary<T>(4U); - TestUnary<T>(17U); - TestUnary<T>(42U); - TestUnary<T>(127U); - TestUnary<T>(128U); - TestUnary<T>(129U); - TestUnary<T>(200U); - TestUnary<T>(255U); - TestUnary<T>(256U); - TestUnary<T>(257U); - - TestUnary<T>(std::numeric_limits<T>::min()); - TestUnary<T>(std::numeric_limits<T>::max()); - TestUnary<T>(std::numeric_limits<T>::max() - 1U); - TestUnary<T>(std::numeric_limits<T>::max() >> 1U); - TestUnary<T>(std::numeric_limits<T>::max() >> 3U); - TestUnary<T>(std::numeric_limits<T>::max() >> 7U); - - - TestUnary<T>(std::numeric_limits<T>::min() + 1U); - TestUnary<T>(std::numeric_limits<T>::max() - 1U); - - TestUnary<T>(std::numeric_limits<T>::min() + 3U); - TestUnary<T>(std::numeric_limits<T>::max() - 3U); - - TestUnary<T>(std::numeric_limits<T>::min() + 7U); - TestUnary<T>(std::numeric_limits<T>::max() - 7U); - - - TestBinary<T>(1U, 1U); - TestBinary<T>(7U, 31U); - TestBinary<T>(30000U, 13U); - TestBinary<T>(127U, 13U); - TestBinary<T>(128U, 19U); - TestBinary<T>(129U, 17U); - - - TestBinary<T>(std::numeric_limits<T>::min(), 7U); - TestBinary<T>(std::numeric_limits<T>::max(), 7U); - - TestBinary<T>(std::numeric_limits<T>::min(), 8U); - TestBinary<T>(std::numeric_limits<T>::max(), 8U); - - TestBinary<T>(std::numeric_limits<T>::min(), 9U); - TestBinary<T>(std::numeric_limits<T>::max(), 9U); - - TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min() + 1U); - - - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::max()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); - - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() + 1U); - TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min() + 1U); - - TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::min() + 1U, std::numeric_limits<T>::min()); - - TestBinary<T>(std::numeric_limits<T>::max(), 1U); - TestBinary<T>(std::numeric_limits<T>::min(), 1U); - TestBinary<T>(std::numeric_limits<T>::max() - 1U, 1U); - TestBinary<T>(std::numeric_limits<T>::min() + 1U, 1U); - - TestBinary<T>(std::numeric_limits<T>::max(), 7U); - TestBinary<T>(std::numeric_limits<T>::min(), 7U); - TestBinary<T>(std::numeric_limits<T>::max() - 1U, 7U); - TestBinary<T>(std::numeric_limits<T>::min() + 1U, 7U); - - TestBinary<T>(std::numeric_limits<T>::max() >> 1U, std::numeric_limits<T>::min() >> 1U); - TestBinary<T>(std::numeric_limits<T>::max() >> 1U, std::numeric_limits<T>::min() + 1U); - TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min() >> 1U); - TestBinary<T>(std::numeric_limits<T>::max() >> 1U, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 1U); - - TestBinary<T>(std::numeric_limits<T>::max() >> 3U, std::numeric_limits<T>::min() >> 3U); - TestBinary<T>(std::numeric_limits<T>::max() >> 3U, std::numeric_limits<T>::min() + 3U); - TestBinary<T>(std::numeric_limits<T>::max() - 3U, std::numeric_limits<T>::min() >> 3U); - TestBinary<T>(std::numeric_limits<T>::max() >> 3U, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 3U); + { + const auto exp = -aa; + const auto tst = -at; + UNIT_ASSERT(T(tst) == T(exp)); } - template<typename T> - void TestsForSignedType() { - static_assert(std::is_signed<T>::value, "Tests for signed type."); - TestUnary<T>(0); + { + auto exp = aa; + auto tst = at; + ++exp; + ++tst; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(1); - TestUnary<T>(-1); + { + auto exp = aa; + auto tst = at; + --exp; + --tst; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(2); - TestUnary<T>(-2); + { + auto exp = aa; + auto tst = at; + exp++; + tst++; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(3); - TestUnary<T>(-3); + { + auto exp = aa; + auto tst = at; + exp--; + tst--; + UNIT_ASSERT(T(tst) == T(exp)); + } +} - TestUnary<T>(4); - TestUnary<T>(-4); +template <typename T> +void TestBinary(const T ll, const T rr) { + using Test = TWide<typename THalfOf<T>::Type>; + const Test lt(ll), rt(rr); - TestUnary<T>(17); - TestUnary<T>(-17); + { + const auto exp = ll & rr; + const auto tst = lt & rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(42); - TestUnary<T>(-42); + { + const auto exp = ll | rr; + const auto tst = lt | rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(127); - TestUnary<T>(-127); + { + const auto exp = ll ^ rr; + const auto tst = lt ^ rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(128); - TestUnary<T>(-128); + { + const auto exp = ll + rr; + const auto tst = lt + rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(129); - TestUnary<T>(-129); + { + const auto exp = ll - rr; + const auto tst = lt - rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(200); - TestUnary<T>(-200); + if (rr > 0 && rr < T(sizeof(T) << 3U)) + { + const auto exp = ll >> rr; + const auto tst = lt >> rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(255); - TestUnary<T>(-255); + if (rr > 0 && rr < T(sizeof(T) << 3U)) + { + const auto exp = ll << rr; + const auto tst = lt << rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(256); - TestUnary<T>(-256); + { + const auto exp = ll * rr; + const auto tst = lt * rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(257); - TestUnary<T>(-257); + if (rr) + { + const auto exp = ll / rr; + const auto tst = lt / rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(258); - TestUnary<T>(-258); + if (rr) + { + const auto exp = ll % rr; + const auto tst = lt % rt; + UNIT_ASSERT(T(tst) == T(exp)); + } - TestUnary<T>(std::numeric_limits<T>::min()); - TestUnary<T>(std::numeric_limits<T>::max()); + { + const auto exp = ll == rr; + const auto tst = lt == rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } - TestUnary<T>(std::numeric_limits<T>::min() + 1); - TestUnary<T>(std::numeric_limits<T>::max() - 1); + { + const auto exp = ll != rr; + const auto tst = lt != rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } - TestUnary<T>(std::numeric_limits<T>::min() + 2); - TestUnary<T>(std::numeric_limits<T>::max() - 2); + { + const auto exp = ll > rr; + const auto tst = lt > rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } - TestUnary<T>(std::numeric_limits<T>::min() + 3); - TestUnary<T>(std::numeric_limits<T>::max() - 3); + { + const auto exp = ll < rr; + const auto tst = lt < rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } - TestUnary<T>(std::numeric_limits<T>::min() + 7); - TestUnary<T>(std::numeric_limits<T>::max() - 7); + { + const auto exp = ll >= rr; + const auto tst = lt >= rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } - TestUnary<T>(std::numeric_limits<T>::min() >> 1); - TestUnary<T>(std::numeric_limits<T>::max() >> 1); + { + const auto exp = ll <= rr; + const auto tst = lt <= rt; + UNIT_ASSERT_VALUES_EQUAL(tst, exp); + } +} - TestUnary<T>(std::numeric_limits<T>::min() >> 3); - TestUnary<T>(std::numeric_limits<T>::max() >> 3); +template <typename T> +void TestsForUnsignedType() { + static_assert(std::is_unsigned<T>::value, "Tests for unsigned type."); + TestUnary<T>(2U); + TestUnary<T>(4U); + TestUnary<T>(17U); + TestUnary<T>(42U); + TestUnary<T>(127U); + TestUnary<T>(128U); + TestUnary<T>(129U); + TestUnary<T>(200U); + TestUnary<T>(255U); + TestUnary<T>(256U); + TestUnary<T>(257U); + + TestUnary<T>(std::numeric_limits<T>::min()); + TestUnary<T>(std::numeric_limits<T>::max()); + TestUnary<T>(std::numeric_limits<T>::max() - 1U); + TestUnary<T>(std::numeric_limits<T>::max() >> 1U); + TestUnary<T>(std::numeric_limits<T>::max() >> 3U); + TestUnary<T>(std::numeric_limits<T>::max() >> 7U); + + TestUnary<T>(std::numeric_limits<T>::min() + 1U); + TestUnary<T>(std::numeric_limits<T>::max() - 1U); + + TestUnary<T>(std::numeric_limits<T>::min() + 3U); + TestUnary<T>(std::numeric_limits<T>::max() - 3U); + + TestUnary<T>(std::numeric_limits<T>::min() + 7U); + TestUnary<T>(std::numeric_limits<T>::max() - 7U); + + TestBinary<T>(1U, 1U); + TestBinary<T>(7U, 31U); + TestBinary<T>(30000U, 13U); + TestBinary<T>(127U, 13U); + TestBinary<T>(128U, 19U); + TestBinary<T>(129U, 17U); + + TestBinary<T>(std::numeric_limits<T>::min(), 7U); + TestBinary<T>(std::numeric_limits<T>::max(), 7U); + + TestBinary<T>(std::numeric_limits<T>::min(), 8U); + TestBinary<T>(std::numeric_limits<T>::max(), 8U); + + TestBinary<T>(std::numeric_limits<T>::min(), 9U); + TestBinary<T>(std::numeric_limits<T>::max(), 9U); + + TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min() + 1U); + + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::max()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); + + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() + 1U); + TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min() + 1U); + + TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::min() + 1U, std::numeric_limits<T>::min()); + + TestBinary<T>(std::numeric_limits<T>::max(), 1U); + TestBinary<T>(std::numeric_limits<T>::min(), 1U); + TestBinary<T>(std::numeric_limits<T>::max() - 1U, 1U); + TestBinary<T>(std::numeric_limits<T>::min() + 1U, 1U); + + TestBinary<T>(std::numeric_limits<T>::max(), 7U); + TestBinary<T>(std::numeric_limits<T>::min(), 7U); + TestBinary<T>(std::numeric_limits<T>::max() - 1U, 7U); + TestBinary<T>(std::numeric_limits<T>::min() + 1U, 7U); + + TestBinary<T>(std::numeric_limits<T>::max() >> 1U, std::numeric_limits<T>::min() >> 1U); + TestBinary<T>(std::numeric_limits<T>::max() >> 1U, std::numeric_limits<T>::min() + 1U); + TestBinary<T>(std::numeric_limits<T>::max() - 1U, std::numeric_limits<T>::min() >> 1U); + TestBinary<T>(std::numeric_limits<T>::max() >> 1U, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 1U); + + TestBinary<T>(std::numeric_limits<T>::max() >> 3U, std::numeric_limits<T>::min() >> 3U); + TestBinary<T>(std::numeric_limits<T>::max() >> 3U, std::numeric_limits<T>::min() + 3U); + TestBinary<T>(std::numeric_limits<T>::max() - 3U, std::numeric_limits<T>::min() >> 3U); + TestBinary<T>(std::numeric_limits<T>::max() >> 3U, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 3U); +} - TestUnary<T>(std::numeric_limits<T>::min() >> 7); - TestUnary<T>(std::numeric_limits<T>::max() >> 7); +template <typename T> +void TestsForSignedType() { + static_assert(std::is_signed<T>::value, "Tests for signed type."); + TestUnary<T>(0); + TestUnary<T>(1); + TestUnary<T>(-1); - TestUnary<T>(std::numeric_limits<T>::min() + 1); - TestUnary<T>(std::numeric_limits<T>::max() - 1); + TestUnary<T>(2); + TestUnary<T>(-2); - TestUnary<T>(std::numeric_limits<T>::min() + 3); - TestUnary<T>(std::numeric_limits<T>::max() - 3); + TestUnary<T>(3); + TestUnary<T>(-3); - TestUnary<T>(std::numeric_limits<T>::min() + 7); - TestUnary<T>(std::numeric_limits<T>::max() - 7); + TestUnary<T>(4); + TestUnary<T>(-4); + TestUnary<T>(17); + TestUnary<T>(-17); - TestBinary<T>(0, 0); - TestBinary<T>(1, 0); - TestBinary<T>(0, -1); + TestUnary<T>(42); + TestUnary<T>(-42); - TestBinary<T>(1, 1); - TestBinary<T>(1, -1); - TestBinary<T>(-1, 1); - TestBinary<T>(-1, -1); + TestUnary<T>(127); + TestUnary<T>(-127); - TestBinary<T>(7, 42); - TestBinary<T>(-7, 42); + TestUnary<T>(128); + TestUnary<T>(-128); - TestBinary<T>(0, -43); + TestUnary<T>(129); + TestUnary<T>(-129); - TestBinary<T>(-30000, 64); - TestBinary<T>(30000, -64); - TestBinary<T>(30000, 64); + TestUnary<T>(200); + TestUnary<T>(-200); - TestBinary<T>(21, 0); - TestBinary<T>(13, -127); - TestBinary<T>(-19, 128); - TestBinary<T>(-77, -129); - TestBinary<T>(13, 127); - TestBinary<T>(19, 128); - TestBinary<T>(77, 129); + TestUnary<T>(255); + TestUnary<T>(-255); - TestBinary<T>(std::numeric_limits<T>::max(), -1); + TestUnary<T>(256); + TestUnary<T>(-256); - TestBinary<T>(std::numeric_limits<T>::min(), -7); - TestBinary<T>(std::numeric_limits<T>::max(), -7); + TestUnary<T>(257); + TestUnary<T>(-257); - TestBinary<T>(std::numeric_limits<T>::min(), -8); - TestBinary<T>(std::numeric_limits<T>::max(), -8); + TestUnary<T>(258); + TestUnary<T>(-258); - TestBinary<T>(std::numeric_limits<T>::min(), -9); - TestBinary<T>(std::numeric_limits<T>::max(), -9); + TestUnary<T>(std::numeric_limits<T>::min()); + TestUnary<T>(std::numeric_limits<T>::max()); - TestBinary<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::min() >> 5); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 5); + TestUnary<T>(std::numeric_limits<T>::min() + 1); + TestUnary<T>(std::numeric_limits<T>::max() - 1); + TestUnary<T>(std::numeric_limits<T>::min() + 2); + TestUnary<T>(std::numeric_limits<T>::max() - 2); - TestBinary<T>(std::numeric_limits<T>::min(), 7); - TestBinary<T>(std::numeric_limits<T>::max(), 7); + TestUnary<T>(std::numeric_limits<T>::min() + 3); + TestUnary<T>(std::numeric_limits<T>::max() - 3); - TestBinary<T>(std::numeric_limits<T>::min(), 8); - TestBinary<T>(std::numeric_limits<T>::max(), 8); + TestUnary<T>(std::numeric_limits<T>::min() + 7); + TestUnary<T>(std::numeric_limits<T>::max() - 7); - TestBinary<T>(std::numeric_limits<T>::min(), 9); - TestBinary<T>(std::numeric_limits<T>::max(), 9); + TestUnary<T>(std::numeric_limits<T>::min() >> 1); + TestUnary<T>(std::numeric_limits<T>::max() >> 1); - TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min() + 1); + TestUnary<T>(std::numeric_limits<T>::min() >> 3); + TestUnary<T>(std::numeric_limits<T>::max() >> 3); + TestUnary<T>(std::numeric_limits<T>::min() >> 7); + TestUnary<T>(std::numeric_limits<T>::max() >> 7); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::max()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); + TestUnary<T>(std::numeric_limits<T>::min() + 1); + TestUnary<T>(std::numeric_limits<T>::max() - 1); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() + 1); - TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min() + 1); + TestUnary<T>(std::numeric_limits<T>::min() + 3); + TestUnary<T>(std::numeric_limits<T>::max() - 3); - TestBinary<T>(std::numeric_limits<T>::max(), 0); - TestBinary<T>(std::numeric_limits<T>::min(), 0); - TestBinary<T>(std::numeric_limits<T>::max() - 1, 0); - TestBinary<T>(std::numeric_limits<T>::min() + 1, 0); + TestUnary<T>(std::numeric_limits<T>::min() + 7); + TestUnary<T>(std::numeric_limits<T>::max() - 7); - TestBinary<T>(std::numeric_limits<T>::max(), 1); - TestBinary<T>(std::numeric_limits<T>::min(), 1); - TestBinary<T>(std::numeric_limits<T>::max() - 1, 1); - TestBinary<T>(std::numeric_limits<T>::min() + 1, 1); + TestBinary<T>(0, 0); + TestBinary<T>(1, 0); + TestBinary<T>(0, -1); - TestBinary<T>(std::numeric_limits<T>::max(), 7); - TestBinary<T>(std::numeric_limits<T>::min(), 7); - TestBinary<T>(std::numeric_limits<T>::max() - 1, 7); - TestBinary<T>(std::numeric_limits<T>::min() + 1, 7); + TestBinary<T>(1, 1); + TestBinary<T>(1, -1); + TestBinary<T>(-1, 1); + TestBinary<T>(-1, -1); - TestBinary<T>(std::numeric_limits<T>::max() >> 1, std::numeric_limits<T>::min() >> 1); - TestBinary<T>(std::numeric_limits<T>::max() >> 1, std::numeric_limits<T>::min() + 1); - TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min() >> 1); - TestBinary<T>(std::numeric_limits<T>::max() >> 1, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 1); + TestBinary<T>(7, 42); + TestBinary<T>(-7, 42); - TestBinary<T>(std::numeric_limits<T>::max() >> 3, std::numeric_limits<T>::min() >> 3); - TestBinary<T>(std::numeric_limits<T>::max() >> 3, std::numeric_limits<T>::min() + 3); - TestBinary<T>(std::numeric_limits<T>::max() - 3, std::numeric_limits<T>::min() >> 3); - TestBinary<T>(std::numeric_limits<T>::max() >> 3, std::numeric_limits<T>::min()); - TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 3); - } + TestBinary<T>(0, -43); - Y_UNIT_TEST(CheckUnsignedByCompilerIntegrals) { - TestsForUnsignedType<ui32>(); - TestsForUnsignedType<ui64>(); -#ifndef _win_ - TestsForUnsignedType<ui128_t>(); -#endif - } + TestBinary<T>(-30000, 64); + TestBinary<T>(30000, -64); + TestBinary<T>(30000, 64); -#ifndef _ubsan_enabled_ -#ifndef _msan_enabled_ - Y_UNIT_TEST(CheckSignedByCompilerIntegrals) { - TestsForSignedType<i32>(); - TestsForSignedType<i64>(); + TestBinary<T>(21, 0); + TestBinary<T>(13, -127); + TestBinary<T>(-19, 128); + TestBinary<T>(-77, -129); + TestBinary<T>(13, 127); + TestBinary<T>(19, 128); + TestBinary<T>(77, 129); + + TestBinary<T>(std::numeric_limits<T>::max(), -1); + + TestBinary<T>(std::numeric_limits<T>::min(), -7); + TestBinary<T>(std::numeric_limits<T>::max(), -7); + + TestBinary<T>(std::numeric_limits<T>::min(), -8); + TestBinary<T>(std::numeric_limits<T>::max(), -8); + + TestBinary<T>(std::numeric_limits<T>::min(), -9); + TestBinary<T>(std::numeric_limits<T>::max(), -9); + + TestBinary<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::min() >> 5); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 5); + + TestBinary<T>(std::numeric_limits<T>::min(), 7); + TestBinary<T>(std::numeric_limits<T>::max(), 7); + + TestBinary<T>(std::numeric_limits<T>::min(), 8); + TestBinary<T>(std::numeric_limits<T>::max(), 8); + + TestBinary<T>(std::numeric_limits<T>::min(), 9); + TestBinary<T>(std::numeric_limits<T>::max(), 9); + + TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min() + 1); + + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::max()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::min(), std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min()); + + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() + 1); + TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min() + 1); + + TestBinary<T>(std::numeric_limits<T>::max(), 0); + TestBinary<T>(std::numeric_limits<T>::min(), 0); + TestBinary<T>(std::numeric_limits<T>::max() - 1, 0); + TestBinary<T>(std::numeric_limits<T>::min() + 1, 0); + + TestBinary<T>(std::numeric_limits<T>::max(), 1); + TestBinary<T>(std::numeric_limits<T>::min(), 1); + TestBinary<T>(std::numeric_limits<T>::max() - 1, 1); + TestBinary<T>(std::numeric_limits<T>::min() + 1, 1); + + TestBinary<T>(std::numeric_limits<T>::max(), 7); + TestBinary<T>(std::numeric_limits<T>::min(), 7); + TestBinary<T>(std::numeric_limits<T>::max() - 1, 7); + TestBinary<T>(std::numeric_limits<T>::min() + 1, 7); + + TestBinary<T>(std::numeric_limits<T>::max() >> 1, std::numeric_limits<T>::min() >> 1); + TestBinary<T>(std::numeric_limits<T>::max() >> 1, std::numeric_limits<T>::min() + 1); + TestBinary<T>(std::numeric_limits<T>::max() - 1, std::numeric_limits<T>::min() >> 1); + TestBinary<T>(std::numeric_limits<T>::max() >> 1, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 1); + + TestBinary<T>(std::numeric_limits<T>::max() >> 3, std::numeric_limits<T>::min() >> 3); + TestBinary<T>(std::numeric_limits<T>::max() >> 3, std::numeric_limits<T>::min() + 3); + TestBinary<T>(std::numeric_limits<T>::max() - 3, std::numeric_limits<T>::min() >> 3); + TestBinary<T>(std::numeric_limits<T>::max() >> 3, std::numeric_limits<T>::min()); + TestBinary<T>(std::numeric_limits<T>::max(), std::numeric_limits<T>::min() >> 3); +} + +Y_UNIT_TEST(CheckUnsignedByCompilerIntegrals) { + TestsForUnsignedType<ui32>(); + TestsForUnsignedType<ui64>(); #ifndef _win_ - TestsForSignedType<i128_t>(); -#endif - } -#endif + TestsForUnsignedType<ui128_t>(); #endif } +#ifndef _ubsan_enabled_ + #ifndef _msan_enabled_ +Y_UNIT_TEST(CheckSignedByCompilerIntegrals) { + TestsForSignedType<i32>(); + TestsForSignedType<i64>(); + #ifndef _win_ + TestsForSignedType<i128_t>(); + #endif } + #endif +#endif +} // Y_UNIT_TEST_SUITE(TYqlWideIntTest) + +} // namespace NYql diff --git a/yql/essentials/public/decimal/ya.make b/yql/essentials/public/decimal/ya.make index a9c23995b33..a014a579fb4 100644 --- a/yql/essentials/public/decimal/ya.make +++ b/yql/essentials/public/decimal/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( yql_decimal.h yql_decimal.cpp diff --git a/yql/essentials/public/decimal/yql_decimal.cpp b/yql/essentials/public/decimal/yql_decimal.cpp index 91564a1a5f1..9c62939686f 100644 --- a/yql/essentials/public/decimal/yql_decimal.cpp +++ b/yql/essentials/public/decimal/yql_decimal.cpp @@ -16,8 +16,9 @@ TUint128 GetDivider(ui8 scale) { } TUint128 d(1U); - while (scale--) + while (scale--) { d *= Ten; + } return d; } @@ -46,12 +47,15 @@ const char* ToString(TInt128 val, ui8 precision, ui8 scale) { return nullptr; } - if (val == Inf()) + if (val == Inf()) { return "inf"; - if (val == -Inf()) + } + if (val == -Inf()) { return "-inf"; - if (val == Nan()) + } + if (val == Nan()) { return "nan"; + } if (!IsNormal(val)) { return nullptr; @@ -77,7 +81,6 @@ const char* ToString(TInt128 val, ui8 precision, ui8 scale) { return nullptr; } - const auto digit = ui8(v % Ten); if (digit || !scale || s != end) { *--s = "0123456789"[digit]; @@ -112,25 +115,26 @@ const char* ToString(TInt128 val, ui8 precision, ui8 scale) { } namespace { - bool IsNan(const char* s) { - return (s[0] == 'N' || s[0] == 'n') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'N' || s[2] == 'n'); - } - - bool IsInf(const char* s) { - return (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'F' || s[2] == 'f'); - } +bool IsNan(const char* s) { + return (s[0] == 'N' || s[0] == 'n') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'N' || s[2] == 'n'); } +bool IsInf(const char* s) { + return (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'F' || s[2] == 'f'); +} +} // namespace TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { - if (scale > precision) + if (scale > precision) { return Err(); + } auto s = str.data(); auto l = str.size(); - if (!s || !l) + if (!s || !l) { return Err(); + } const bool neg = '-' == *s; if (neg || '+' == *s) { @@ -139,10 +143,12 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { } if (3U == l) { - if (IsInf(s)) + if (IsInf(s)) { return neg ? -Inf() : Inf(); - if (IsNan(s)) + } + if (IsNan(s)) { return Nan(); + } } TUint128 v = 0U; @@ -150,8 +156,9 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { for (bool dot = false; l; --l) { if (*s == '.') { - if (dot) + if (dot) { return Err(); + } ++s; dot = true; @@ -159,15 +166,17 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { } if (dot) { - if (scale) + if (scale) { --scale; - else + } else { break; + } } const char c = *s++; - if (!std::isdigit(c)) + if (!std::isdigit(c)) { return Err(); + } v *= Ten; v += c - '0'; @@ -179,64 +188,73 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) { if (l--) { const char c = *s++; - if (!std::isdigit(c)) + if (!std::isdigit(c)) { return Err(); + } bool plus = c > '5'; if (!plus && c == '5') { for (plus = v & 1; !plus && l; --l) { const char c = *s++; - if (!std::isdigit(c)) + if (!std::isdigit(c)) { return Err(); + } plus = c != '0'; } } - while (l--) - if (!std::isdigit(*s++)) + while (l--) { + if (!std::isdigit(*s++)) { return Err(); + } + } - if (plus) - if (++v >= GetDivider(precision)) + if (plus) { + if (++v >= GetDivider(precision)) { v = Inf(); + } + } } - while (scale--) + while (scale--) { v *= Ten; + } return neg ? -v : v; } TInt128 FromStringEx(const TStringBuf& str, ui8 precision, ui8 scale) { - if (scale > precision) + if (scale > precision) { return Err(); + } const auto s = str.data(); for (auto ptr = s + str.size() - 1U; ptr > s; --ptr) { if (*ptr == 'E' || *ptr == 'e') { const auto len = ptr - s; - if (!len) + if (!len) { return Err(); + } ++ptr; if (ptr != s + str.size() && *ptr == '+') { ++ptr; - if (ptr != s + str.size() && *ptr == '-') + if (ptr != s + str.size() && *ptr == '-') { return Err(); + } } int exp; auto [finish, ec] = std::from_chars(ptr, s + str.size(), exp); - if (ec != std::errc() || finish != s + str.size()) + if (ec != std::errc() || finish != s + str.size()) { return Err(); + } const int p = precision, s = int(scale) + exp; - const auto r = exp > 0 ? - FromString(str.Head(len), precision, std::min(s, p)): - FromString(str.Head(len), std::min(p - exp, int(MaxPrecision)), std::max(s, 0)); + const auto r = exp > 0 ? FromString(str.Head(len), precision, std::min(s, p)) : FromString(str.Head(len), std::min(p - exp, int(MaxPrecision)), std::max(s, 0)); if (IsError(r) || IsNan(r)) { return Err(); @@ -244,21 +262,25 @@ TInt128 FromStringEx(const TStringBuf& str, ui8 precision, ui8 scale) { if (IsInf(r)) { auto p = str.data(); - if (*p == '+' || *p == '-') + if (*p == '+' || *p == '-') { ++p; + } - if (!std::isdigit(*p)) + if (!std::isdigit(*p)) { return Err(); + } return r; } if (const auto e = exp > 0 ? std::max(0, s - p) : std::min(0, s)) { if (r) { - if (exp > 0) + if (exp > 0) { return Mul(r, GetDivider(+e)); - if (exp < 0) + } + if (exp < 0) { return Div(r, GetDivider(-e)); + } } } @@ -273,8 +295,9 @@ bool IsValid(const TStringBuf& str) { auto s = str.data(); auto l = str.size(); - if (!s || !l) + if (!s || !l) { return false; + } if ('-' == *s || '+' == *s) { ++s; @@ -288,57 +311,67 @@ bool IsValid(const TStringBuf& str) { for (bool dot = false; l--;) { const char c = *s++; if (c == '.') { - if (dot) + if (dot) { return false; + } dot = true; continue; } - if (!std::isdigit(c)) + if (!std::isdigit(c)) { return false; + } } return true; } TInt128 Mod(TInt128 a, TInt128 b) { - if (!b || !(IsNormal(a) && IsNormal(b))) + if (!b || !(IsNormal(a) && IsNormal(b))) { return Nan(); + } return a % b; } TInt128 Div(TInt128 a, TInt128 b) { - if (IsNan(a) || IsNan(b)) + if (IsNan(a) || IsNan(b)) { return Nan(); + } if (!b) { - if (a > 0) + if (a > 0) { return Inf(); - else if (a < 0) + } else if (a < 0) { return -Inf(); - else + } else { return Nan(); + } } else if (IsInf(b)) { return IsInf(a) ? Nan() : TInt128(0); } else if (IsInf(a)) { return b > 0 ? a : -a; } - if (b & 1) + if (b & 1) { a = TUint128(a) << 1U; - else + } else { b >>= 1; + } auto d = a / b; if (d & 1) { if (const auto m = a % b) { - if (m > 0) ++d; + if (m > 0) { + ++d; + } // else --d; } else { - if (d & 2) ++d; + if (d & 2) { + ++d; + } } } @@ -352,10 +385,12 @@ using TInt256 = TWide<TInt128, TInt128, TUint128>; TInt128 Normalize(const TInt256& v) { static const TInt256 PInf256(+Inf()), NInf256(-Inf()); - if (v > PInf256) + if (v > PInf256) { return +Inf(); - if (v < NInf256) + } + if (v < NInf256) { return -Inf(); + } return *reinterpret_cast<const TInt128*>(&v); } @@ -390,53 +425,63 @@ TInt256 WidenMul(const TInt128& lhs, const TInt128& rhs) { return nl == nr ? +combine : -combine; } -template<bool MayOddDivider> +template <bool MayOddDivider> TInt256 Div(TInt256&& a, TInt256&& b) { - if (MayOddDivider && b & 1) + if (MayOddDivider && b & 1) { a <<= 1; - else + } else { b >>= 1; + } auto d = a / b; if (d & 1) { if (const auto m = a % b) { - if (m > 0) ++d; + if (m > 0) { + ++d; + } // else --d; } else { - if (d & 2) ++d; + if (d & 2) { + ++d; + } } } return d >>= 1; } -} +} // namespace TInt128 Mul(TInt128 a, TInt128 b) { - if (IsNan(a) || IsNan(b)) + if (IsNan(a) || IsNan(b)) { return Nan(); + } - if (IsInf(a)) + if (IsInf(a)) { return !b ? Nan() : (b > 0 ? a : -a); + } - if (IsInf(b)) + if (IsInf(b)) { return !a ? Nan() : (a > 0 ? b : -b); + } return Normalize(WidenMul(a, b)); } TInt128 MulAndDivNormalMultiplier(TInt128 a, TInt128 b, TInt128 c) { - if (IsNan(a) || IsNan(c)) + if (IsNan(a) || IsNan(c)) { return Nan(); + } if (!c) { - if (a > 0) + if (a > 0) { return Inf(); - else if (a < 0) + } else if (a < 0) { return -Inf(); - else + } else { return Nan(); + } } else if (IsInf(c)) { return IsInf(a) ? Nan() : TInt128(0); } else if (IsInf(a)) { @@ -447,17 +492,20 @@ TInt128 MulAndDivNormalMultiplier(TInt128 a, TInt128 b, TInt128 c) { } TInt128 MulAndDivNormalDivider(TInt128 a, TInt128 b, TInt128 c) { - if (IsNan(a) || IsNan(b)) + if (IsNan(a) || IsNan(b)) { return Nan(); + } - if (IsInf(a)) + if (IsInf(a)) { return !b ? Nan() : (b > 0 ? a : -a); + } - if (IsInf(b)) + if (IsInf(b)) { return !a ? Nan() : (a > 0 ? b : -b); + } return Normalize(Div<false>(WidenMul(a, b), TInt256(c))); } -} -} +} // namespace NDecimal +} // namespace NYql diff --git a/yql/essentials/public/decimal/yql_decimal.h b/yql/essentials/public/decimal/yql_decimal.h index 01740ecfe2f..1967b4c120f 100644 --- a/yql/essentials/public/decimal/yql_decimal.h +++ b/yql/essentials/public/decimal/yql_decimal.h @@ -10,9 +10,9 @@ namespace NYql { namespace NDecimal { #ifdef _win_ -#ifndef DONT_USE_NATIVE_INT128 -#define DONT_USE_NATIVE_INT128 -#endif + #ifndef DONT_USE_NATIVE_INT128 + #define DONT_USE_NATIVE_INT128 + #endif #endif #ifdef DONT_USE_NATIVE_INT128 @@ -23,13 +23,26 @@ using TInt128 = signed __int128; using TUint128 = unsigned __int128; #endif -template<ui8 Scale> struct TDivider; +template <ui8 Scale> +struct TDivider; #if defined(__clang__) && defined(DONT_USE_NATIVE_INT128) -template<> struct TDivider<0> { static inline constexpr TUint128 Value = 1U; }; -template<ui8 Scale> struct TDivider { static inline constexpr TInt128 Value = TDivider<Scale - 1U>::Value * 10U; }; +template <> +struct TDivider<0> { + static inline constexpr TUint128 Value = 1U; +}; +template <ui8 Scale> +struct TDivider { + static inline constexpr TInt128 Value = TDivider<Scale - 1U>::Value * 10U; +}; #else -template<> struct TDivider<0> { static constexpr TUint128 Value = 1U; }; -template<ui8 Scale> struct TDivider { static constexpr TUint128 Value = TDivider<Scale - 1U>::Value * 10U; }; +template <> +struct TDivider<0> { + static constexpr TUint128 Value = 1U; +}; +template <ui8 Scale> +struct TDivider { + static constexpr TUint128 Value = TDivider<Scale - 1U>::Value * 10U; +}; #endif constexpr ui8 MaxPrecision = 35; @@ -50,12 +63,12 @@ inline constexpr TInt128 Err() { TUint128 GetDivider(ui8 scale); -template<ui8 Precision> +template <ui8 Precision> inline constexpr TUint128 GetDivider() { return TDivider<Precision>::Value; } -template<ui8 Precision, bool IncLow = false, bool DecHigh = false> +template <ui8 Precision, bool IncLow = false, bool DecHigh = false> inline constexpr std::pair<TInt128, TInt128> GetBounds() { return std::make_pair(-GetDivider<Precision>() + (IncLow ? 1 : 0), +GetDivider<Precision>() - (DecHigh ? 1 : 0)); } @@ -67,7 +80,7 @@ bool IsInf(TInt128 v); bool IsNormal(TInt128 v); bool IsComparable(TInt128 v); -template<ui8 Precision> +template <ui8 Precision> inline bool IsNormal(TInt128 v) { const auto& b = GetBounds<Precision>(); return v > b.first && v < b.second; @@ -79,7 +92,7 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale = 0); // Accept string representation with exponent. TInt128 FromStringEx(const TStringBuf& str, ui8 precision, ui8 scale); -template<typename TMkqlProto> +template <typename TMkqlProto> inline TInt128 FromProto(const TMkqlProto& val) { ui64 half[2] = {val.GetLow128(), val.GetHi128()}; TInt128 val128; @@ -87,22 +100,22 @@ inline TInt128 FromProto(const TMkqlProto& val) { return val128; } -template<typename TValue> +template <typename TValue> inline constexpr TValue YtDecimalNan() { return std::numeric_limits<TValue>::max(); } -template<> +template <> inline constexpr TInt128 YtDecimalNan<TInt128>() { return ~(TInt128(1) << 127); } -template<typename TValue> +template <typename TValue> inline constexpr TValue YtDecimalInf() { return YtDecimalNan<TValue>() - 1; } -template<typename TValue> +template <typename TValue> inline TInt128 FromYtDecimal(TValue val) { static_assert(std::is_same<TInt128, TValue>::value || std::is_signed<TValue>::value, "Expected signed value"); if (YtDecimalNan<TValue>() == val) { @@ -116,7 +129,7 @@ inline TInt128 FromYtDecimal(TValue val) { } } -template<typename TValue> +template <typename TValue> inline TValue ToYtDecimal(TInt128 val) { static_assert(std::is_same<TInt128, TValue>::value || std::is_signed<TValue>::value, "Expected signed value"); if (IsNormal(val)) { @@ -160,8 +173,11 @@ struct TDecimal { TDecimal() = default; - template<typename T> - TDecimal(T t): Value(t) { } + template <typename T> + TDecimal(T t) + : Value(t) + { + } explicit operator TInt128() const { return Value; @@ -177,8 +193,8 @@ struct TDecimal { Value = Nan(); } else { Value = a > 0 - ? +Inf() - : -Inf(); + ? +Inf() + : -Inf(); } return *this; } @@ -209,7 +225,7 @@ struct TDecimal { } }; -template<typename TRight> +template <typename TRight> class TDecimalMultiplicator { protected: const TInt128 Bound_; @@ -226,14 +242,15 @@ public: TInt128 Do(TInt128 left, TRight right) const { TInt128 mul = Mul(left, right); - if (mul > -Bound_ && mul < +Bound_) + if (mul > -Bound_ && mul < +Bound_) { return mul; + } return IsNan(mul) ? Nan() : (mul > 0 ? +Inf() : -Inf()); } }; -template<> +template <> class TDecimalMultiplicator<TInt128> { protected: const TInt128 Bound_; @@ -245,21 +262,21 @@ public: ui8 scale) : Bound_(GetDivider(precision)) , Divider_(GetDivider(scale)) - { } + { + } TInt128 Do(TInt128 left, TInt128 right) const { - TInt128 mul = Divider_ > 1 ? - MulAndDivNormalDivider(left, right, Divider_): - Mul(left, right); + TInt128 mul = Divider_ > 1 ? MulAndDivNormalDivider(left, right, Divider_) : Mul(left, right); - if (mul > -Bound_ && mul < +Bound_) + if (mul > -Bound_ && mul < +Bound_) { return mul; + } return IsNan(mul) ? Nan() : (mul > 0 ? +Inf() : -Inf()); } }; -template<typename TRight> +template <typename TRight> class TDecimalDivisor { public: TDecimalDivisor( @@ -275,7 +292,7 @@ public: } }; -template<> +template <> class TDecimalDivisor<TInt128> { protected: const TInt128 Bound_; @@ -287,7 +304,8 @@ public: ui8 scale) : Bound_(GetDivider(precision)) , Divider_(GetDivider(scale)) - { } + { + } TInt128 Do(TInt128 left, TInt128 right) const { TInt128 div = MulAndDivNormalMultiplier(left, Divider_, right); @@ -299,7 +317,7 @@ public: } }; -template<typename TRight> +template <typename TRight> class TDecimalRemainder { protected: const TInt128 Bound_; @@ -311,22 +329,25 @@ public: ui8 scale) : Bound_(NYql::NDecimal::GetDivider(precision - scale)) , Divider_(NYql::NDecimal::GetDivider(scale)) - { } + { + } TInt128 Do(TInt128 left, TRight right) const { if constexpr (std::is_signed<TRight>::value) { - if (TInt128(right) >= +Bound_ || TInt128(right) <= -Bound_) + if (TInt128(right) >= +Bound_ || TInt128(right) <= -Bound_) { return left; + } } else { - if (TInt128(right) >= Bound_) + if (TInt128(right) >= Bound_) { return left; + } } return Mod(left, Mul(Divider_, right)); } }; -template<> +template <> class TDecimalRemainder<TInt128> { public: TDecimalRemainder( @@ -342,5 +363,5 @@ public: } }; -} -} +} // namespace NDecimal +} // namespace NYql diff --git a/yql/essentials/public/decimal/yql_decimal_serialize.cpp b/yql/essentials/public/decimal/yql_decimal_serialize.cpp index 5fb844f5f89..46b353424f8 100644 --- a/yql/essentials/public/decimal/yql_decimal_serialize.cpp +++ b/yql/essentials/public/decimal/yql_decimal_serialize.cpp @@ -30,12 +30,14 @@ size_t Serialize(TInt128 value, char* buf) { auto p = reinterpret_cast<const char*>(&value) + size - 1U; if (*(p - 1U) & 0x80) { - while (size > 1U && ~0 == *--p) + while (size > 1U && ~0 == *--p) { --size; + } *buf = 0x80 - size; } else { - while (size > 1U && 0 == *--p) + while (size > 1U && 0 == *--p) { --size; + } *buf = 0x7F + size; } @@ -47,36 +49,42 @@ size_t Serialize(TInt128 value, char* buf) { } std::pair<TInt128, size_t> Deserialize(const char* b, size_t len) { - if (!b || len == 0U) + if (!b || len == 0U) { return std::make_pair(Err(), 0U); + } const auto mark = ui8(*b); const bool neg = mark < 0x80u; - if (mark == 0x00u || mark == 0xFFu) + if (mark == 0x00u || mark == 0xFFu) { return std::make_pair(neg ? -Nan() : +Nan(), 1U); - if (mark == 0x01u || mark == 0xFEu) + } + if (mark == 0x01u || mark == 0xFEu) { return std::make_pair(neg ? -Inf() : +Inf(), 1U); + } if (mark < 0x70u || mark > 0x8Fu) { return std::make_pair(Err(), 0U); } const auto used = neg ? 0x80u - mark : mark - 0x7Fu; - if (len < used) + if (len < used) { return std::make_pair(Err(), 0U); + } TInt128 v; const auto size = sizeof(v); auto p = reinterpret_cast<char*>(&v) + size; - for (auto fill = size - used + 2U; --fill;) + for (auto fill = size - used + 2U; --fill;) { *--p = neg ? ~0 : 0; + } - for (auto copy = used; --copy;) + for (auto copy = used; --copy;) { *--p = *++b; + } return std::make_pair(v, used); } -} -} +} // namespace NDecimal +} // namespace NYql diff --git a/yql/essentials/public/decimal/yql_decimal_serialize.h b/yql/essentials/public/decimal/yql_decimal_serialize.h index 6fdb32e6f2a..0965125d841 100644 --- a/yql/essentials/public/decimal/yql_decimal_serialize.h +++ b/yql/essentials/public/decimal/yql_decimal_serialize.h @@ -9,5 +9,5 @@ namespace NDecimal { size_t Serialize(TInt128 v, char* buff); std::pair<TInt128, size_t> Deserialize(const char* buff, size_t len); -} -} +} // namespace NDecimal +} // namespace NYql diff --git a/yql/essentials/public/decimal/yql_wide_int.h b/yql/essentials/public/decimal/yql_wide_int.h index ce0506844ed..87922724284 100644 --- a/yql/essentials/public/decimal/yql_wide_int.h +++ b/yql/essentials/public/decimal/yql_wide_int.h @@ -13,7 +13,7 @@ typedef __int128 i128_t; typedef unsigned __int128 ui128_t; #endif -template<typename TOneHalf, typename TSignedPart = std::make_signed_t<TOneHalf>, typename TUnsignedPart = std::make_unsigned_t<TOneHalf>> +template <typename TOneHalf, typename TSignedPart = std::make_signed_t<TOneHalf>, typename TUnsignedPart = std::make_unsigned_t<TOneHalf>> class TWide { private: using THalf = TOneHalf; @@ -41,17 +41,17 @@ private: return h & mask; } - template<typename T> + template <typename T> constexpr std::enable_if_t<sizeof(T) <= sizeof(THalf), T> CastImpl() const { return static_cast<T>(Lo_); } - template<typename T> + template <typename T> constexpr std::enable_if_t<sizeof(T) == sizeof(THalf) << 1U, T> CastImpl() const { return *reinterpret_cast<const T*>(this); } - template<typename T> + template <typename T> constexpr std::enable_if_t<sizeof(THalf) << 1U < sizeof(T), T> CastImpl() const { return (T(std::make_unsigned_t<T>(Hi_) << PartBitSize)) | Lo_; } @@ -67,8 +67,9 @@ private: using TUnsigned = TWide<TUnsignedPart, TSignedPart, TUnsignedPart>; using TSibling = std::conditional_t<std::is_same<THalf, TPart>::value, - TWide<TSignedPart, TSignedPart, TUnsignedPart>, TWide<TUnsignedPart, TSignedPart, TUnsignedPart>>; + TWide<TSignedPart, TSignedPart, TUnsignedPart>, TWide<TUnsignedPart, TSignedPart, TUnsignedPart>>; friend TSibling; + public: constexpr TWide() = default; constexpr TWide(const TWide& rhs) = default; @@ -78,28 +79,38 @@ public: constexpr TWide& operator=(TWide&& rhs) = default; constexpr TWide(const TSibling& rhs) - : Lo_(rhs.Lo_), Hi_(rhs.Hi_) - {} + : Lo_(rhs.Lo_) + , Hi_(rhs.Hi_) + { + } - template<typename U, typename L> + template <typename U, typename L> constexpr TWide(const U upper_rhs, const L lower_rhs) - : Lo_(lower_rhs), Hi_(upper_rhs) - {} + : Lo_(lower_rhs) + , Hi_(upper_rhs) + { + } template <typename T, std::enable_if_t<std::is_integral<T>::value && sizeof(THalf) < sizeof(T), size_t> Shift = PartBitSize> constexpr TWide(const T rhs) - : Lo_(rhs), Hi_(rhs >> Shift) - {} + : Lo_(rhs) + , Hi_(rhs >> Shift) + { + } template <typename T, std::enable_if_t<std::is_integral<T>::value && sizeof(T) <= sizeof(THalf), bool> Signed = std::is_signed<T>::value> constexpr TWide(const T rhs) - : Lo_(rhs), Hi_(Signed && rhs < 0 ? ~0 : 0) - {} + : Lo_(rhs) + , Hi_(Signed && rhs < 0 ? ~0 : 0) + { + } template <typename T, typename TArg = std::enable_if_t<std::is_class<T>::value && std::is_same<T, THalf>::value, THalf>> constexpr explicit TWide(const T& rhs) - : Lo_(rhs), Hi_(TIsSigned::value && rhs < 0 ? ~0 : 0) - {} + : Lo_(rhs) + , Hi_(TIsSigned::value && rhs < 0 ? ~0 : 0) + { + } template <typename T, std::enable_if_t<std::is_integral<T>::value && sizeof(THalf) < sizeof(T), size_t> Shift = PartBitSize> constexpr TWide& operator=(const T rhs) { @@ -115,19 +126,41 @@ public: return *this; } - constexpr explicit operator bool() const { return bool(Hi_) || bool(Lo_); } + constexpr explicit operator bool() const { + return bool(Hi_) || bool(Lo_); + } - constexpr explicit operator i8() const { return CastImpl<i8>(); } - constexpr explicit operator ui8() const { return CastImpl<ui8>(); } - constexpr explicit operator i16() const { return CastImpl<i16>(); } - constexpr explicit operator ui16() const { return CastImpl<ui16>(); } - constexpr explicit operator i32() const { return CastImpl<i32>(); } - constexpr explicit operator ui32() const { return CastImpl<ui32>(); } - constexpr explicit operator i64() const { return CastImpl<i64>(); } - constexpr explicit operator ui64() const { return CastImpl<ui64>(); } + constexpr explicit operator i8() const { + return CastImpl<i8>(); + } + constexpr explicit operator ui8() const { + return CastImpl<ui8>(); + } + constexpr explicit operator i16() const { + return CastImpl<i16>(); + } + constexpr explicit operator ui16() const { + return CastImpl<ui16>(); + } + constexpr explicit operator i32() const { + return CastImpl<i32>(); + } + constexpr explicit operator ui32() const { + return CastImpl<ui32>(); + } + constexpr explicit operator i64() const { + return CastImpl<i64>(); + } + constexpr explicit operator ui64() const { + return CastImpl<ui64>(); + } #ifndef _win_ - constexpr explicit operator i128_t() const { return CastImpl<i128_t>(); } - constexpr explicit operator ui128_t() const { return CastImpl<ui128_t>(); } + constexpr explicit operator i128_t() const { + return CastImpl<i128_t>(); + } + constexpr explicit operator ui128_t() const { + return CastImpl<ui128_t>(); + } #endif constexpr explicit operator float() const { return TIsSigned::value && Hi_ < 0 ? -float(-*this) : std::fma(float(Hi_), std::exp2(float(PartBitSize)), float(Lo_)); @@ -147,18 +180,23 @@ public: constexpr TWide operator-() const { const auto sign = THalf(1) << PartBitSize - 1U; - if (TIsSigned::value && !Lo_ && Hi_ == sign) + if (TIsSigned::value && !Lo_ && Hi_ == sign) { return *this; + } return ++~*this; } constexpr TWide& operator++() { - if (!++Lo_) ++Hi_; + if (!++Lo_) { + ++Hi_; + } return *this; } constexpr TWide& operator--() { - if (!Lo_--) --Hi_; + if (!Lo_--) { + --Hi_; + } return *this; } @@ -196,7 +234,9 @@ public: const auto l = Lo_; Lo_ += rhs.Lo_; Hi_ += rhs.Hi_; - if (l > Lo_) ++Hi_; + if (l > Lo_) { + ++Hi_; + } return *this; } @@ -204,7 +244,9 @@ public: const auto l = Lo_; Lo_ -= rhs.Lo_; Hi_ -= rhs.Hi_; - if (l < Lo_) --Hi_; + if (l < Lo_) { + --Hi_; + } return *this; } @@ -238,30 +280,70 @@ public: return *this; } - constexpr TWide& operator*=(const TWide& rhs) { return *this = Mul(*this, rhs); } - constexpr TWide& operator/=(const TWide& rhs) { return *this = DivMod(*this, rhs).first; } - constexpr TWide& operator%=(const TWide& rhs) { return *this = DivMod(*this, rhs).second; } + constexpr TWide& operator*=(const TWide& rhs) { + return *this = Mul(*this, rhs); + } + constexpr TWide& operator/=(const TWide& rhs) { + return *this = DivMod(*this, rhs).first; + } + constexpr TWide& operator%=(const TWide& rhs) { + return *this = DivMod(*this, rhs).second; + } - constexpr TWide operator&(const TWide& rhs) const { return TWide(*this) &= rhs; } - constexpr TWide operator|(const TWide& rhs) const { return TWide(*this) |= rhs; } - constexpr TWide operator^(const TWide& rhs) const { return TWide(*this) ^= rhs; } - constexpr TWide operator+(const TWide& rhs) const { return TWide(*this) += rhs; } - constexpr TWide operator-(const TWide& rhs) const { return TWide(*this) -= rhs; } - constexpr TWide operator*(const TWide& rhs) const { return Mul(*this, rhs); } - constexpr TWide operator/(const TWide& rhs) const { return DivMod(*this, rhs).first; } - constexpr TWide operator%(const TWide& rhs) const { return DivMod(*this, rhs).second; } - constexpr TWide operator<<(const TWide& rhs) const { return TWide(*this) <<= rhs; } - constexpr TWide operator>>(const TWide& rhs) const { return TWide(*this) >>= rhs; } + constexpr TWide operator&(const TWide& rhs) const { + return TWide(*this) &= rhs; + } + constexpr TWide operator|(const TWide& rhs) const { + return TWide(*this) |= rhs; + } + constexpr TWide operator^(const TWide& rhs) const { + return TWide(*this) ^= rhs; + } + constexpr TWide operator+(const TWide& rhs) const { + return TWide(*this) += rhs; + } + constexpr TWide operator-(const TWide& rhs) const { + return TWide(*this) -= rhs; + } + constexpr TWide operator*(const TWide& rhs) const { + return Mul(*this, rhs); + } + constexpr TWide operator/(const TWide& rhs) const { + return DivMod(*this, rhs).first; + } + constexpr TWide operator%(const TWide& rhs) const { + return DivMod(*this, rhs).second; + } + constexpr TWide operator<<(const TWide& rhs) const { + return TWide(*this) <<= rhs; + } + constexpr TWide operator>>(const TWide& rhs) const { + return TWide(*this) >>= rhs; + } template <typename T> - constexpr std::enable_if_t<std::is_integral<T>::value, T> operator&(const T rhs) const { return T(*this) & rhs; } + constexpr std::enable_if_t<std::is_integral<T>::value, T> operator&(const T rhs) const { + return T(*this) & rhs; + } - constexpr bool operator==(const TWide& rhs) const { return std::tie(Hi_, Lo_) == std::tie(rhs.Hi_, rhs.Lo_); } - constexpr bool operator!=(const TWide& rhs) const { return std::tie(Hi_, Lo_) != std::tie(rhs.Hi_, rhs.Lo_); } - constexpr bool operator>=(const TWide& rhs) const { return std::tie(Hi_, Lo_) >= std::tie(rhs.Hi_, rhs.Lo_); } - constexpr bool operator<=(const TWide& rhs) const { return std::tie(Hi_, Lo_) <= std::tie(rhs.Hi_, rhs.Lo_); } - constexpr bool operator>(const TWide& rhs) const { return std::tie(Hi_, Lo_) > std::tie(rhs.Hi_, rhs.Lo_); } - constexpr bool operator<(const TWide& rhs) const { return std::tie(Hi_, Lo_) < std::tie(rhs.Hi_, rhs.Lo_); } + constexpr bool operator==(const TWide& rhs) const { + return std::tie(Hi_, Lo_) == std::tie(rhs.Hi_, rhs.Lo_); + } + constexpr bool operator!=(const TWide& rhs) const { + return std::tie(Hi_, Lo_) != std::tie(rhs.Hi_, rhs.Lo_); + } + constexpr bool operator>=(const TWide& rhs) const { + return std::tie(Hi_, Lo_) >= std::tie(rhs.Hi_, rhs.Lo_); + } + constexpr bool operator<=(const TWide& rhs) const { + return std::tie(Hi_, Lo_) <= std::tie(rhs.Hi_, rhs.Lo_); + } + constexpr bool operator>(const TWide& rhs) const { + return std::tie(Hi_, Lo_) > std::tie(rhs.Hi_, rhs.Lo_); + } + constexpr bool operator<(const TWide& rhs) const { + return std::tie(Hi_, Lo_) < std::tie(rhs.Hi_, rhs.Lo_); + } private: static constexpr TWide Mul(const TWide& lhs, const TWide& rhs) { @@ -274,14 +356,9 @@ private: const TPart prod3[] = {TPart(lq[0] * rq[3]), TPart(lq[1] * rq[2]), TPart(lq[2] * rq[1]), TPart(lq[3] * rq[0])}; const TPart fourthQ = GetLowerQuarter(prod0[0]); - const TPart thirdQ = GetUpperQuarter(prod0[0]) - + GetLowerQuarter(prod1[0]) + GetLowerQuarter(prod1[1]); - const TPart secondQ = GetUpperQuarter(thirdQ) - + GetUpperQuarter(prod1[0]) + GetUpperQuarter(prod1[1]) - + GetLowerQuarter(prod2[0]) + GetLowerQuarter(prod2[1]) + GetLowerQuarter(prod2[2]); - const TPart firstQ = GetUpperQuarter(secondQ) - + GetUpperQuarter(prod2[0]) + GetUpperQuarter(prod2[1]) + GetUpperQuarter(prod2[2]) - + GetLowerQuarter(prod3[0]) + GetLowerQuarter(prod3[1]) + GetLowerQuarter(prod3[2]) + GetLowerQuarter(prod3[3]); + const TPart thirdQ = GetUpperQuarter(prod0[0]) + GetLowerQuarter(prod1[0]) + GetLowerQuarter(prod1[1]); + const TPart secondQ = GetUpperQuarter(thirdQ) + GetUpperQuarter(prod1[0]) + GetUpperQuarter(prod1[1]) + GetLowerQuarter(prod2[0]) + GetLowerQuarter(prod2[1]) + GetLowerQuarter(prod2[2]); + const TPart firstQ = GetUpperQuarter(secondQ) + GetUpperQuarter(prod2[0]) + GetUpperQuarter(prod2[1]) + GetUpperQuarter(prod2[2]) + GetLowerQuarter(prod3[0]) + GetLowerQuarter(prod3[1]) + GetLowerQuarter(prod3[2]) + GetLowerQuarter(prod3[3]); return TWide((firstQ << QuarterBitSize) | GetLowerQuarter(secondQ), (thirdQ << QuarterBitSize) | fourthQ); } @@ -308,37 +385,93 @@ private: } } - if (nl) mod = -mod; - if (nr != nl) div = -div; + if (nl) { + mod = -mod; + } + if (nr != nl) { + div = -div; + } return {div, mod}; } }; -template<typename T> struct THalfOf; -template<> struct THalfOf<i16> { typedef i8 Type; }; -template<> struct THalfOf<ui16> { typedef ui8 Type; }; -template<> struct THalfOf<i32> { typedef i16 Type; }; -template<> struct THalfOf<ui32> { typedef ui16 Type; }; -template<> struct THalfOf<i64> { typedef i32 Type; }; -template<> struct THalfOf<ui64> { typedef ui32 Type; }; +template <typename T> +struct THalfOf; +template <> +struct THalfOf<i16> { + typedef i8 Type; +}; +template <> +struct THalfOf<ui16> { + typedef ui8 Type; +}; +template <> +struct THalfOf<i32> { + typedef i16 Type; +}; +template <> +struct THalfOf<ui32> { + typedef ui16 Type; +}; +template <> +struct THalfOf<i64> { + typedef i32 Type; +}; +template <> +struct THalfOf<ui64> { + typedef ui32 Type; +}; #ifndef _win_ -template<> struct THalfOf<i128_t> { typedef i64 Type; }; -template<> struct THalfOf<ui128_t> { typedef ui64 Type; }; +template <> +struct THalfOf<i128_t> { + typedef i64 Type; +}; +template <> +struct THalfOf<ui128_t> { + typedef ui64 Type; +}; #endif -template<typename T> struct THalfOf {}; - -template<typename T> struct TPairOf; -template<> struct TPairOf<i8> { typedef i16 Type; }; -template<> struct TPairOf<ui8> { typedef ui16 Type; }; -template<> struct TPairOf<i16> { typedef i32 Type; }; -template<> struct TPairOf<ui16> { typedef ui32 Type; }; -template<> struct TPairOf<i32> { typedef i64 Type; }; -template<> struct TPairOf<ui32> { typedef ui64 Type; }; +template <typename T> +struct THalfOf {}; + +template <typename T> +struct TPairOf; +template <> +struct TPairOf<i8> { + typedef i16 Type; +}; +template <> +struct TPairOf<ui8> { + typedef ui16 Type; +}; +template <> +struct TPairOf<i16> { + typedef i32 Type; +}; +template <> +struct TPairOf<ui16> { + typedef ui32 Type; +}; +template <> +struct TPairOf<i32> { + typedef i64 Type; +}; +template <> +struct TPairOf<ui32> { + typedef ui64 Type; +}; #ifndef _win_ -template<> struct TPairOf<i64> { typedef i128_t Type; }; -template<> struct TPairOf<ui64> { typedef ui128_t Type; }; +template <> +struct TPairOf<i64> { + typedef i128_t Type; +}; +template <> +struct TPairOf<ui64> { + typedef ui128_t Type; +}; #endif -template<typename T> struct TPairOf {}; +template <typename T> +struct TPairOf {}; -} +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/check_runner.cpp b/yql/essentials/public/fastcheck/check_runner.cpp index 97889d18858..1c7172675c0 100644 --- a/yql/essentials/public/fastcheck/check_runner.cpp +++ b/yql/essentials/public/fastcheck/check_runner.cpp @@ -27,5 +27,5 @@ TCheckResponse TCheckRunnerBase::Run(const TChecksRequest& request) { return response; } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/check_runner.h b/yql/essentials/public/fastcheck/check_runner.h index 74edd882ef0..1ba685c3176 100644 --- a/yql/essentials/public/fastcheck/check_runner.h +++ b/yql/essentials/public/fastcheck/check_runner.h @@ -12,7 +12,7 @@ public: virtual TCheckResponse Run(const TChecksRequest& request) = 0; }; -class TCheckRunnerBase : public ICheckRunner { +class TCheckRunnerBase: public ICheckRunner { public: virtual TCheckResponse Run(const TChecksRequest& request) final; @@ -32,5 +32,5 @@ std::unique_ptr<ICheckRunner> MakeParserRunner(); std::unique_ptr<ICheckRunner> MakeTranslatorRunner(); std::unique_ptr<ICheckRunner> MakeFormatRunner(); -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/fastcheck.cpp b/yql/essentials/public/fastcheck/fastcheck.cpp index 65b84a654b7..e534c2fa326 100644 --- a/yql/essentials/public/fastcheck/fastcheck.cpp +++ b/yql/essentials/public/fastcheck/fastcheck.cpp @@ -30,7 +30,7 @@ void FillSettings(NSQLTranslation::TTranslationSettings& settings, const TOption settings.Flags = TranslationFlags(); } -} +} // namespace bool CheckProgram(const TString& program, const TOptions& options, TIssues& errors) { TMaybe<TIssue> verIssue; @@ -53,8 +53,7 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro NSQLTranslation::TTranslators translators( nullptr, NSQLTranslationV1::MakeTranslator(lexers, parsers), - NSQLTranslationPG::MakeTranslator() - ); + NSQLTranslationPG::MakeTranslator()); TAstParseResult astRes; if (options.IsSql) { @@ -133,5 +132,5 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro return true; } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/fastcheck.h b/yql/essentials/public/fastcheck/fastcheck.h index 8edb554e922..45c1087284a 100644 --- a/yql/essentials/public/fastcheck/fastcheck.h +++ b/yql/essentials/public/fastcheck/fastcheck.h @@ -8,16 +8,16 @@ namespace NYql { namespace NFastCheck { struct TOptions { - bool IsSql = true; - bool ParseOnly = false; - THashMap<TString, TString> ClusterMapping; - ui16 SyntaxVersion = 1; - TLangVersion LangVer = MinLangVersion; - bool IsLibrary = false; - THashMap<TString, TString> SqlLibs = {}; // mapping file name => SQL + bool IsSql = true; + bool ParseOnly = false; + THashMap<TString, TString> ClusterMapping; + ui16 SyntaxVersion = 1; + TLangVersion LangVer = MinLangVersion; + bool IsLibrary = false; + THashMap<TString, TString> SqlLibs = {}; // mapping file name => SQL }; bool CheckProgram(const TString& program, const TOptions& options, TIssues& errors); -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/fastcheck_ut.cpp b/yql/essentials/public/fastcheck/fastcheck_ut.cpp index 8aa403e6f57..4e7c894ae77 100644 --- a/yql/essentials/public/fastcheck/fastcheck_ut.cpp +++ b/yql/essentials/public/fastcheck/fastcheck_ut.cpp @@ -5,143 +5,143 @@ using namespace NYql; using namespace NYql::NFastCheck; Y_UNIT_TEST_SUITE(TFastCheckTests) { - Y_UNIT_TEST(ParsePureYqlGood) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = false; - options.ParseOnly = true; - TIssues errors; - UNIT_ASSERT(CheckProgram("(return world)", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(ParsePureYqlBad) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = false; - options.ParseOnly = true; - TIssues errors; - UNIT_ASSERT(CheckProgram("(return world1)", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(ParsePureSqlGood) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = true; - TIssues errors; - UNIT_ASSERT(CheckProgram("select 1", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(ParsePureSqlBad) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = true; - TIssues errors; - UNIT_ASSERT(!CheckProgram("select1", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); - } - - Y_UNIT_TEST(CompilePureYqlBad) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = false; - options.ParseOnly = false; - TIssues errors; - UNIT_ASSERT(!CheckProgram("(return world1)", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); - } - - Y_UNIT_TEST(CompileTableSqlGood) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = false; - options.ClusterMapping["plato"] = YtProviderName; - TIssues errors; - UNIT_ASSERT(CheckProgram("select key,count(*) from plato.Input group by key", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(CompileTableSqlBad) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = false; - TIssues errors; - UNIT_ASSERT(!CheckProgram("select key,count(*) from plato.Input", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); - } - - Y_UNIT_TEST(CompileLibrary) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.IsLibrary = true; - TIssues errors; - UNIT_ASSERT(CheckProgram("$x = 1; export $x", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(CompileSqlWithLibsGood) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = false; - options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; - TIssues errors; - UNIT_ASSERT(CheckProgram("pragma library('foo.sql');import foo symbols $x; select $x", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(ParseSqlWithBadLib) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = true; - options.SqlLibs["foo.sql"] = "$x = 1; zexport $x;"; - TIssues errors; - UNIT_ASSERT(!CheckProgram("pragma library('foo.sql');import foo symbols $x; select $x", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); - } - - Y_UNIT_TEST(CompileSqlWithUnresolvedLib) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = false; - options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; - TIssues errors; - UNIT_ASSERT(!CheckProgram("pragma library('foo.sql');import foo symbols $y; select $y", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); - } - - Y_UNIT_TEST(ParseSqlWithUnresolvedLib) { - TOptions options; - options.LangVer = GetMaxReleasedLangVersion(); - options.IsSql = true; - options.ParseOnly = true; - options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; - TIssues errors; - UNIT_ASSERT(CheckProgram("pragma library('foo.sql');import foo symbols $y; select $y", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); - } - - Y_UNIT_TEST(TooHighLangVer) { - TOptions options; - options.LangVer = GetMaxLangVersion(); - options.IsSql = false; - options.ParseOnly = true; - TIssues errors; - UNIT_ASSERT(!CheckProgram("(return world)", options, errors)); - UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); - } - - Y_UNIT_TEST(UsedFlags) { - TString program = R"sql( +Y_UNIT_TEST(ParsePureYqlGood) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = false; + options.ParseOnly = true; + TIssues errors; + UNIT_ASSERT(CheckProgram("(return world)", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(ParsePureYqlBad) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = false; + options.ParseOnly = true; + TIssues errors; + UNIT_ASSERT(CheckProgram("(return world1)", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(ParsePureSqlGood) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = true; + TIssues errors; + UNIT_ASSERT(CheckProgram("select 1", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(ParsePureSqlBad) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = true; + TIssues errors; + UNIT_ASSERT(!CheckProgram("select1", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); +} + +Y_UNIT_TEST(CompilePureYqlBad) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = false; + options.ParseOnly = false; + TIssues errors; + UNIT_ASSERT(!CheckProgram("(return world1)", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); +} + +Y_UNIT_TEST(CompileTableSqlGood) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = false; + options.ClusterMapping["plato"] = YtProviderName; + TIssues errors; + UNIT_ASSERT(CheckProgram("select key,count(*) from plato.Input group by key", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(CompileTableSqlBad) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = false; + TIssues errors; + UNIT_ASSERT(!CheckProgram("select key,count(*) from plato.Input", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); +} + +Y_UNIT_TEST(CompileLibrary) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.IsLibrary = true; + TIssues errors; + UNIT_ASSERT(CheckProgram("$x = 1; export $x", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(CompileSqlWithLibsGood) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = false; + options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; + TIssues errors; + UNIT_ASSERT(CheckProgram("pragma library('foo.sql');import foo symbols $x; select $x", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(ParseSqlWithBadLib) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = true; + options.SqlLibs["foo.sql"] = "$x = 1; zexport $x;"; + TIssues errors; + UNIT_ASSERT(!CheckProgram("pragma library('foo.sql');import foo symbols $x; select $x", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); +} + +Y_UNIT_TEST(CompileSqlWithUnresolvedLib) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = false; + options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; + TIssues errors; + UNIT_ASSERT(!CheckProgram("pragma library('foo.sql');import foo symbols $y; select $y", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); +} + +Y_UNIT_TEST(ParseSqlWithUnresolvedLib) { + TOptions options; + options.LangVer = GetMaxReleasedLangVersion(); + options.IsSql = true; + options.ParseOnly = true; + options.SqlLibs["foo.sql"] = "$x = 1; export $x;"; + TIssues errors; + UNIT_ASSERT(CheckProgram("pragma library('foo.sql');import foo symbols $y; select $y", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(0, errors.Size()); +} + +Y_UNIT_TEST(TooHighLangVer) { + TOptions options; + options.LangVer = GetMaxLangVersion(); + options.IsSql = false; + options.ParseOnly = true; + TIssues errors; + UNIT_ASSERT(!CheckProgram("(return world)", options, errors)); + UNIT_ASSERT_VALUES_EQUAL(1, errors.Size()); +} + +Y_UNIT_TEST(UsedFlags) { + TString program = R"sql( $input = AsList( AsStruct(1 AS key, 1001 AS subkey, 'AAA' AS value), ); @@ -155,11 +155,11 @@ Y_UNIT_TEST_SUITE(TFastCheckTests) { AS_TABLE($input) AS i1 )sql"; - TOptions options; - options.IsSql = true; - - TIssues errors; - UNIT_ASSERT_C(CheckProgram(program, options, errors), errors.ToOneLineString()); - } + TOptions options; + options.IsSql = true; + TIssues errors; + UNIT_ASSERT_C(CheckProgram(program, options, errors), errors.ToOneLineString()); } + +} // Y_UNIT_TEST_SUITE(TFastCheckTests) diff --git a/yql/essentials/public/fastcheck/format.cpp b/yql/essentials/public/fastcheck/format.cpp index c17b2a027c3..f9f0b12520a 100644 --- a/yql/essentials/public/fastcheck/format.cpp +++ b/yql/essentials/public/fastcheck/format.cpp @@ -40,7 +40,7 @@ TString ReplaceHidden(TStringBuf input) { return res; } -class TFormatRunner : public TCheckRunnerBase { +class TFormatRunner: public TCheckRunnerBase { public: TString GetCheckName() const final { return "format"; @@ -48,12 +48,12 @@ public: TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { - case ESyntax::SExpr: - return RunSExpr(request); - case ESyntax::PG: - return RunPg(request); - case ESyntax::YQL: - return RunYql(request); + case ESyntax::SExpr: + return RunSExpr(request); + case ESyntax::PG: + return RunPg(request); + case ESyntax::YQL: + return RunYql(request); } } @@ -71,7 +71,7 @@ private: } TCheckResponse RunYql(const TChecksRequest& request) { - TCheckResponse res {.CheckName = GetCheckName()}; + TCheckResponse res{.CheckName = GetCheckName()}; if (request.SyntaxVersion != 1) { res.Issues.AddIssue(TIssue({}, "Only SyntaxVersion 1 is supported")); return res; @@ -122,8 +122,9 @@ private: origSample.erase(origSample.size() - 1); } - auto issue = TIssue(origPos, TStringBuilder() << - "Format mismatch, expected:\n" << ReplaceHidden(formattedSample) << "\nbut got:\n" << ReplaceHidden(origSample) << "\n"); + auto issue = TIssue(origPos, TStringBuilder() << "Format mismatch, expected:\n" + << ReplaceHidden(formattedSample) << "\nbut got:\n" + << ReplaceHidden(origSample) << "\n"); issue.SetCode(EYqlIssueCode::TIssuesIds_EIssueCode_WARNING, ESeverity::TSeverityIds_ESeverityId_S_WARNING); res.Issues.AddIssue(issue); } @@ -132,11 +133,11 @@ private: } }; -} +} // namespace std::unique_ptr<ICheckRunner> MakeFormatRunner() { return std::make_unique<TFormatRunner>(); } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/lexer.cpp b/yql/essentials/public/fastcheck/lexer.cpp index b1f26361704..e2c7b7d1fb4 100644 --- a/yql/essentials/public/fastcheck/lexer.cpp +++ b/yql/essentials/public/fastcheck/lexer.cpp @@ -9,7 +9,7 @@ namespace NFastCheck { namespace { -class TLexerRunner : public TCheckRunnerBase { +class TLexerRunner: public TCheckRunnerBase { public: TString GetCheckName() const final { return "lexer"; @@ -17,12 +17,12 @@ public: TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { - case ESyntax::SExpr: - return RunSExpr(request); - case ESyntax::PG: - return RunPg(request); - case ESyntax::YQL: - return RunYql(request); + case ESyntax::SExpr: + return RunSExpr(request); + case ESyntax::PG: + return RunPg(request); + case ESyntax::YQL: + return RunYql(request); } } @@ -40,7 +40,7 @@ private: } TCheckResponse RunYql(const TChecksRequest& request) { - TCheckResponse res {.CheckName = GetCheckName()}; + TCheckResponse res{.CheckName = GetCheckName()}; NSQLTranslation::TTranslationSettings settings; settings.SyntaxVersion = request.SyntaxVersion; settings.AnsiLexer = request.IsAnsiLexer; @@ -65,11 +65,11 @@ private: } }; -} +} // namespace std::unique_ptr<ICheckRunner> MakeLexerRunner() { return std::make_unique<TLexerRunner>(); } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/linter.cpp b/yql/essentials/public/fastcheck/linter.cpp index a32814e3ec3..1a2e904fd91 100644 --- a/yql/essentials/public/fastcheck/linter.cpp +++ b/yql/essentials/public/fastcheck/linter.cpp @@ -13,7 +13,7 @@ namespace NFastCheck { namespace { -class TCheckRunnerFactory : public ICheckRunnerFactory { +class TCheckRunnerFactory: public ICheckRunnerFactory { public: using TRunnerFactoryFunction = std::function<std::unique_ptr<ICheckRunner>()>; @@ -76,11 +76,11 @@ TCheckRunnerFactory& GetCheckRunnerFactory() { return *Singleton<TCheckRunnerFactory>(); }; -} +} // namespace TVector<TCheckFilter> ParseChecks(const TString& checks) { TVector<TCheckFilter> res; - for (TStringBuf one: StringSplitter(checks).SplitByString(",")) { + for (TStringBuf one : StringSplitter(checks).SplitByString(",")) { TCheckFilter f; TStringBuf afterPrefix = one; if (one.AfterPrefix("-", afterPrefix)) { @@ -111,5 +111,5 @@ TChecksResponse RunChecks(const TChecksRequest& request) { return res; } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/linter.h b/yql/essentials/public/fastcheck/linter.h index 1314f7c065f..31a802f3f97 100644 --- a/yql/essentials/public/fastcheck/linter.h +++ b/yql/essentials/public/fastcheck/linter.h @@ -61,5 +61,5 @@ TVector<TCheckFilter> ParseChecks(const TString& checks); TSet<TString> ListChecks(const TMaybe<TVector<TCheckFilter>>& filters = Nothing()); TChecksResponse RunChecks(const TChecksRequest& request); -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/linter_ut.cpp b/yql/essentials/public/fastcheck/linter_ut.cpp index 740202cebd3..134bf48b113 100644 --- a/yql/essentials/public/fastcheck/linter_ut.cpp +++ b/yql/essentials/public/fastcheck/linter_ut.cpp @@ -8,631 +8,630 @@ namespace { TChecksRequest MakeCheckRequest() { return TChecksRequest{ - .LangVer = GetMaxReleasedLangVersion() - }; + .LangVer = GetMaxReleasedLangVersion()}; } -} +} // namespace Y_UNIT_TEST_SUITE(TLinterTests) { - Y_UNIT_TEST(ListChecksResult) { - auto res = ListChecks(); - UNIT_ASSERT(!res.empty()); - } +Y_UNIT_TEST(ListChecksResult) { + auto res = ListChecks(); + UNIT_ASSERT(!res.empty()); +} - Y_UNIT_TEST(DummyLexerSExpr) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "((return world))"; - request.Syntax = ESyntax::SExpr; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(DummyLexerSExpr) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "((return world))"; + request.Syntax = ESyntax::SExpr; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(DummyLexerPg) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1::text"; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(DummyLexerPg) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1::text"; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodLexerYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "1"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(GoodLexerYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "1"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadLexerYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "Я"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadLexerYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "Я"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "lexer"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(DummyFormatSExpr) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "((return world))"; - request.Syntax = ESyntax::SExpr; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(DummyFormatSExpr) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "((return world))"; + request.Syntax = ESyntax::SExpr; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(DummyFormatPg) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1::text"; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(DummyFormatPg) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1::text"; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodFormatYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "SELECT\n 1\n;\n"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(GoodFormatYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "SELECT\n 1\n;\n"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodFormatYqlWithWinEOL) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "SELECT\r\n 1\r\n;\r\n"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(GoodFormatYqlWithWinEOL) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "SELECT\r\n 1\r\n;\r\n"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodFormatYqlWithWinEOLInComment) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "--\r\nSELECT\n 1\n;\n\nSELECT\n 2\n;\n"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(GoodFormatYqlWithWinEOLInComment) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "--\r\nSELECT\n 1\n;\n\nSELECT\n 2\n;\n"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(UnparsedFormatYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select1\n"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(UnparsedFormatYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select1\n"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadFormatYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadFormatYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(ContextForBadFormatYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "SELECT\n 'привет',1;"; - request.File = "myFile.sql"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 1); - const auto& pos = res.Checks[0].Issues.back().Position; - UNIT_ASSERT_VALUES_EQUAL(pos.Row, 2); - UNIT_ASSERT_VALUES_EQUAL(pos.Column, 13); - } +Y_UNIT_TEST(ContextForBadFormatYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "SELECT\n 'привет',1;"; + request.File = "myFile.sql"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 1); + const auto& pos = res.Checks[0].Issues.back().Position; + UNIT_ASSERT_VALUES_EQUAL(pos.Row, 2); + UNIT_ASSERT_VALUES_EQUAL(pos.Column, 13); +} - Y_UNIT_TEST(BadFormatYqlHidden) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select\t1 "; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); - UNIT_ASSERT(!res.Checks[0].Success); - UNIT_ASSERT_NO_DIFF(res.Checks[0].Issues.ToString(), - "<main>:1:0: Warning: Format mismatch, expected:\n" - "SELECT\n" - "••••1\n" - ";\n\n" - "but got:\n" - "select→1•\n" - ", code: 3\n"); - } +Y_UNIT_TEST(BadFormatYqlHidden) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select\t1 "; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "format"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "format"); + UNIT_ASSERT(!res.Checks[0].Success); + UNIT_ASSERT_NO_DIFF(res.Checks[0].Issues.ToString(), + "<main>:1:0: Warning: Format mismatch, expected:\n" + "SELECT\n" + "••••1\n" + ";\n\n" + "but got:\n" + "select→1•\n" + ", code: 3\n"); +} - Y_UNIT_TEST(GoodParserSExpr) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "((return world))"; - request.Syntax = ESyntax::SExpr; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodParserSExpr) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "((return world))"; + request.Syntax = ESyntax::SExpr; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadParserSExpr) { - TChecksRequest request = MakeCheckRequest(); - request.Program = ")"; - request.Syntax = ESyntax::SExpr; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadParserSExpr) { + TChecksRequest request = MakeCheckRequest(); + request.Program = ")"; + request.Syntax = ESyntax::SExpr; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(GoodParserPg) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1::text"; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodParserPg) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1::text"; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadParserPg) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "sel"; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadParserPg) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "sel"; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(DummyParserPg) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1::text"; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(DummyParserPg) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1::text"; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodParserYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "SELECT 1"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(GoodParserYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "SELECT 1"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadParserYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "1"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadParserYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "1"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "parser"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "parser"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(DummyTranslatorSExpr) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "((return world))"; - request.Syntax = ESyntax::SExpr; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); - } +Y_UNIT_TEST(DummyTranslatorSExpr) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "((return world))"; + request.Syntax = ESyntax::SExpr; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodTranslatorPg) { - TChecksRequest request = MakeCheckRequest(); - request.ClusterMapping["plato"] = TString(YtProviderName); - request.Program = "select * from plato.\"Input\""; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodTranslatorPg) { + TChecksRequest request = MakeCheckRequest(); + request.ClusterMapping["plato"] = TString(YtProviderName); + request.Program = "select * from plato.\"Input\""; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadTranslatorPg) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select * from \"Input\""; - request.Syntax = ESyntax::PG; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadTranslatorPg) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select * from \"Input\""; + request.Syntax = ESyntax::PG; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(GoodTranslatorYql) { - TChecksRequest request = MakeCheckRequest(); - request.ClusterMapping["plato"] = TString(YtProviderName); - request.Program = "use plato; select * from Input"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodTranslatorYql) { + TChecksRequest request = MakeCheckRequest(); + request.ClusterMapping["plato"] = TString(YtProviderName); + request.Program = "use plato; select * from Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadTranslatorYql) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select ListLengggth([1,2,3])"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadTranslatorYql) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select ListLengggth([1,2,3])"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(AllowYqlExportsForLibrary) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "$a = 1; export $a"; - request.Mode = EMode::Library; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(AllowYqlExportsForLibrary) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "$a = 1; export $a"; + request.Mode = EMode::Library; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(AllowYqlExportsForDefault) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "$a = 1; export $a"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(AllowYqlExportsForDefault) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "$a = 1; export $a"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(DisallowYqlExportsForMain) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "$a = 1; export $a"; - request.Syntax = ESyntax::YQL; - request.Mode = EMode::Main; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(DisallowYqlExportsForMain) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "$a = 1; export $a"; + request.Syntax = ESyntax::YQL; + request.Mode = EMode::Main; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(DisallowYqlExportsForView) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "$a = 1; export $a"; - request.Syntax = ESyntax::YQL; - request.Mode = EMode::View; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(DisallowYqlExportsForView) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "$a = 1; export $a"; + request.Syntax = ESyntax::YQL; + request.Mode = EMode::View; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(GoodYqlView) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1"; - request.Syntax = ESyntax::YQL; - request.Mode = EMode::View; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodYqlView) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1"; + request.Syntax = ESyntax::YQL; + request.Mode = EMode::View; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(BadYqlView) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select 1;select 2"; - request.Syntax = ESyntax::YQL; - request.Mode = EMode::View; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(BadYqlView) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select 1;select 2"; + request.Syntax = ESyntax::YQL; + request.Mode = EMode::View; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(AllChecks) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "SELECT\n 1\n;\n"; - request.Syntax = ESyntax::YQL; - auto res = RunChecks(request); - TSet<TString> passedChecks; - for (const auto& r : res.Checks) { - Cerr << r.CheckName << "\n"; - UNIT_ASSERT(r.Success); - Cerr << r.Issues.ToString(); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - passedChecks.insert(r.CheckName); - } - - UNIT_ASSERT_VALUES_EQUAL(passedChecks, ListChecks()); +Y_UNIT_TEST(AllChecks) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "SELECT\n 1\n;\n"; + request.Syntax = ESyntax::YQL; + auto res = RunChecks(request); + TSet<TString> passedChecks; + for (const auto& r : res.Checks) { + Cerr << r.CheckName << "\n"; + UNIT_ASSERT(r.Success); + Cerr << r.Issues.ToString(); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); + passedChecks.insert(r.CheckName); } - Y_UNIT_TEST(AllChecksByStar) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "SELECT\n 1\n;\n"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "*"}); - auto res = RunChecks(request); - TSet<TString> passedChecks; - for (const auto& r : res.Checks) { - Cerr << r.CheckName << "\n"; - UNIT_ASSERT(r.Success); - Cerr << r.Issues.ToString(); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - passedChecks.insert(r.CheckName); - } - - UNIT_ASSERT_VALUES_EQUAL(passedChecks, ListChecks()); - } + UNIT_ASSERT_VALUES_EQUAL(passedChecks, ListChecks()); +} - Y_UNIT_TEST(NoChecksByStarWithSecondFilter) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "1"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.Include = false, .CheckNameGlob = "*"}); - request.Filters->push_back(TCheckFilter{.Include = true, .CheckNameGlob = "lexer"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); +Y_UNIT_TEST(AllChecksByStar) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "SELECT\n 1\n;\n"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "*"}); + auto res = RunChecks(request); + TSet<TString> passedChecks; + for (const auto& r : res.Checks) { + Cerr << r.CheckName << "\n"; + UNIT_ASSERT(r.Success); + Cerr << r.Issues.ToString(); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); + passedChecks.insert(r.CheckName); } - Y_UNIT_TEST(BadTranslatorYqlWithoutUseMany) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select * from Input"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } + UNIT_ASSERT_VALUES_EQUAL(passedChecks, ListChecks()); +} - Y_UNIT_TEST(GoodTranslatorYqlWithoutUseSingle) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select * from Input"; - request.ClusterMode = EClusterMode::Single; - request.ClusterSystem = YtProviderName; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(NoChecksByStarWithSecondFilter) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "1"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.Include = false, .CheckNameGlob = "*"}); + request.Filters->push_back(TCheckFilter{.Include = true, .CheckNameGlob = "lexer"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "lexer"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); +} - Y_UNIT_TEST(GoodTranslatorYqlWithoutUseUnknown) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select * from Input"; - request.ClusterMode = EClusterMode::Unknown; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(BadTranslatorYqlWithoutUseMany) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select * from Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(BadTranslatorYqlAnotherClusterMany) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select * from foo.Input"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(GoodTranslatorYqlWithoutUseSingle) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select * from Input"; + request.ClusterMode = EClusterMode::Single; + request.ClusterSystem = YtProviderName; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodTranslatorYqlAnotherClusterMany) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "select * from foo.Input"; - request.ClusterSystem = YtProviderName; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodTranslatorYqlWithoutUseUnknown) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select * from Input"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemUpdate) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "update foo set value = 1"; - request.ClusterMode = EClusterMode::Unknown; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(BadTranslatorYqlAnotherClusterMany) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select * from foo.Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} - Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemDelete) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "delete from foo where value = 1"; - request.ClusterMode = EClusterMode::Unknown; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodTranslatorYqlAnotherClusterMany) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "select * from foo.Input"; + request.ClusterSystem = YtProviderName; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemReplaceInto) { - TChecksRequest request = MakeCheckRequest(); - request.Program = "replace into foo select 1"; - request.ClusterMode = EClusterMode::Unknown; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } +Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemUpdate) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "update foo set value = 1"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(TooHighLangVersion) { - TChecksRequest request; - request.LangVer = GetMaxLangVersion(); - request.ClusterMapping["plato"] = TString(YtProviderName); - request.Program = "use plato; select * from Input"; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT(!res.Checks[0].Success); - Cerr << res.Checks[0].Issues.ToString(); - UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); - } +Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemDelete) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "delete from foo where value = 1"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} - Y_UNIT_TEST(UsedFlags) { - TChecksRequest request; - request.Program = R"sql( +Y_UNIT_TEST(GoodTranslatorYqlUnknownSystemReplaceInto) { + TChecksRequest request = MakeCheckRequest(); + request.Program = "replace into foo select 1"; + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); +} + +Y_UNIT_TEST(TooHighLangVersion) { + TChecksRequest request; + request.LangVer = GetMaxLangVersion(); + request.ClusterMapping["plato"] = TString(YtProviderName); + request.Program = "use plato; select * from Input"; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT(!res.Checks[0].Success); + Cerr << res.Checks[0].Issues.ToString(); + UNIT_ASSERT(res.Checks[0].Issues.Size() > 0); +} + +Y_UNIT_TEST(UsedFlags) { + TChecksRequest request; + request.Program = R"sql( $input = AsList( AsStruct(1 AS key, 1001 AS subkey, 'AAA' AS value), ); @@ -645,16 +644,16 @@ Y_UNIT_TEST_SUITE(TLinterTests) { FROM AS_TABLE($input) AS i1 )sql"; - request.ClusterMode = EClusterMode::Unknown; - request.Syntax = ESyntax::YQL; - request.Filters.ConstructInPlace(); - request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); - - auto res = RunChecks(request); - UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); - UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); - UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(),0); - } - + request.ClusterMode = EClusterMode::Unknown; + request.Syntax = ESyntax::YQL; + request.Filters.ConstructInPlace(); + request.Filters->push_back(TCheckFilter{.CheckNameGlob = "translator"}); + + auto res = RunChecks(request); + UNIT_ASSERT_VALUES_EQUAL(res.Checks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].CheckName, "translator"); + UNIT_ASSERT_C(res.Checks[0].Success, res.Checks[0].Issues.ToString()); + UNIT_ASSERT_VALUES_EQUAL(res.Checks[0].Issues.Size(), 0); } + +} // Y_UNIT_TEST_SUITE(TLinterTests) diff --git a/yql/essentials/public/fastcheck/parser.cpp b/yql/essentials/public/fastcheck/parser.cpp index 9ab04728744..e122beb1291 100644 --- a/yql/essentials/public/fastcheck/parser.cpp +++ b/yql/essentials/public/fastcheck/parser.cpp @@ -14,7 +14,7 @@ namespace NFastCheck { namespace { -class TParserRunner : public TCheckRunnerBase { +class TParserRunner: public TCheckRunnerBase { public: TString GetCheckName() const final { return "parser"; @@ -22,12 +22,12 @@ public: TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { - case ESyntax::SExpr: - return RunSExpr(request); - case ESyntax::PG: - return RunPg(request); - case ESyntax::YQL: - return RunYql(request); + case ESyntax::SExpr: + return RunSExpr(request); + case ESyntax::PG: + return RunPg(request); + case ESyntax::YQL: + return RunYql(request); } } @@ -37,15 +37,15 @@ private: return TCheckResponse{ .CheckName = GetCheckName(), .Success = res.IsOk(), - .Issues = res.Issues - }; + .Issues = res.Issues}; } - class TPGParseEventsHandler : public IPGParseEvents { + class TPGParseEventsHandler: public IPGParseEvents { public: TPGParseEventsHandler(TCheckResponse& res) : Res_(res) - {} + { + } void OnResult(const List* raw) final { Y_UNUSED(raw); @@ -61,14 +61,14 @@ private: }; TCheckResponse RunPg(const TChecksRequest& request) { - TCheckResponse res {.CheckName = GetCheckName()}; + TCheckResponse res{.CheckName = GetCheckName()}; TPGParseEventsHandler handler(res); PGParse(request.Program, handler); return res; } TCheckResponse RunYql(const TChecksRequest& request) { - TCheckResponse res {.CheckName = GetCheckName()}; + TCheckResponse res{.CheckName = GetCheckName()}; NSQLTranslation::TTranslationSettings settings; settings.SyntaxVersion = request.SyntaxVersion; settings.AnsiLexer = request.IsAnsiLexer; @@ -92,7 +92,7 @@ private: parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); auto msg = NSQLTranslationV1::SqlAST(parsers, request.Program, request.File, res.Issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS, - settings.AnsiLexer, true, &arena); + settings.AnsiLexer, true, &arena); if (msg) { res.Success = true; } @@ -102,11 +102,11 @@ private: } }; -} +} // namespace std::unique_ptr<ICheckRunner> MakeParserRunner() { return std::make_unique<TParserRunner>(); } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/settings.cpp b/yql/essentials/public/fastcheck/settings.cpp index a26ef6ca43c..a3313c8d94c 100644 --- a/yql/essentials/public/fastcheck/settings.cpp +++ b/yql/essentials/public/fastcheck/settings.cpp @@ -2,17 +2,17 @@ namespace NYql::NFastCheck { - THashSet<TString> TranslationFlags() { - return { - "AnsiOrderByLimitInUnionAll", - "DisableCoalesceJoinKeysOnQualifiedAll", - "AnsiRankForNullableKeys", - "DisableUnorderedSubqueries", - "DisableAnsiOptionalAs", - "FlexibleTypes", - "CompactNamedExprs", - "DistinctOverWindow", - }; - } +THashSet<TString> TranslationFlags() { + return { + "AnsiOrderByLimitInUnionAll", + "DisableCoalesceJoinKeysOnQualifiedAll", + "AnsiRankForNullableKeys", + "DisableUnorderedSubqueries", + "DisableAnsiOptionalAs", + "FlexibleTypes", + "CompactNamedExprs", + "DistinctOverWindow", + }; +} } // namespace NYql::NFastCheck diff --git a/yql/essentials/public/fastcheck/settings.h b/yql/essentials/public/fastcheck/settings.h index 4e63b554fb8..cb8eb8c41a6 100644 --- a/yql/essentials/public/fastcheck/settings.h +++ b/yql/essentials/public/fastcheck/settings.h @@ -5,6 +5,6 @@ namespace NYql::NFastCheck { - THashSet<TString> TranslationFlags(); +THashSet<TString> TranslationFlags(); } // namespace NYql::NFastCheck diff --git a/yql/essentials/public/fastcheck/translator.cpp b/yql/essentials/public/fastcheck/translator.cpp index aa17b6272bb..cadeb4ec296 100644 --- a/yql/essentials/public/fastcheck/translator.cpp +++ b/yql/essentials/public/fastcheck/translator.cpp @@ -16,7 +16,7 @@ namespace NFastCheck { namespace { -class TTranslatorRunner : public TCheckRunnerBase { +class TTranslatorRunner: public TCheckRunnerBase { public: TString GetCheckName() const final { return "translator"; @@ -24,12 +24,12 @@ public: TCheckResponse DoRun(const TChecksRequest& request) final { switch (request.Syntax) { - case ESyntax::SExpr: - return RunSExpr(request); - case ESyntax::PG: - return RunPg(request); - case ESyntax::YQL: - return RunYql(request); + case ESyntax::SExpr: + return RunSExpr(request); + case ESyntax::PG: + return RunPg(request); + case ESyntax::YQL: + return RunYql(request); } } @@ -51,12 +51,11 @@ private: return TCheckResponse{ .CheckName = GetCheckName(), .Success = astRes.IsOk(), - .Issues = astRes.Issues - }; + .Issues = astRes.Issues}; } TCheckResponse RunYql(const TChecksRequest& request) { - TCheckResponse res {.CheckName = GetCheckName()}; + TCheckResponse res{.CheckName = GetCheckName()}; google::protobuf::Arena arena; NSQLTranslation::TTranslationSettings settings; settings.Arena = &arena; @@ -69,17 +68,17 @@ private: settings.Flags = TranslationFlags(); switch (request.Mode) { - case EMode::Default: - settings.AlwaysAllowExports = true; - break; - case EMode::Library: - settings.Mode = NSQLTranslation::ESqlMode::LIBRARY; - break; - case EMode::Main: - break; - case EMode::View: - settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; - break; + case EMode::Default: + settings.AlwaysAllowExports = true; + break; + case EMode::Library: + settings.Mode = NSQLTranslation::ESqlMode::LIBRARY; + break; + case EMode::Main: + break; + case EMode::View: + settings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW; + break; } if (!ParseTranslationSettings(request.Program, settings, res.Issues)) { @@ -102,39 +101,39 @@ private: void FillClusters(const TChecksRequest& request, NSQLTranslation::TTranslationSettings& settings) { if (!request.ClusterSystem.empty()) { Y_ENSURE(AnyOf(Providers, [&](const auto& p) { return p == request.ClusterSystem; }), - "Invalid ClusterSystem value: " + request.ClusterSystem); + "Invalid ClusterSystem value: " + request.ClusterSystem); } switch (request.ClusterMode) { - case EClusterMode::Many: - for (const auto& x : request.ClusterMapping) { - Y_ENSURE(AnyOf(Providers, [&](const auto& p) { return p == x.second; }), - "Invalid system: " + x.second); - } - - settings.ClusterMapping = request.ClusterMapping; - settings.DynamicClusterProvider = request.ClusterSystem; - break; - case EClusterMode::Single: - Y_ENSURE(!request.ClusterSystem.empty(), "Missing ClusterSystem parameter"); - settings.DefaultCluster = "single"; - settings.ClusterMapping["single"] = request.ClusterSystem; - settings.DynamicClusterProvider = request.ClusterSystem; - break; - case EClusterMode::Unknown: - settings.DefaultCluster = "single"; - settings.ClusterMapping["single"] = UnknownProviderName; - settings.DynamicClusterProvider = UnknownProviderName; - break; + case EClusterMode::Many: + for (const auto& x : request.ClusterMapping) { + Y_ENSURE(AnyOf(Providers, [&](const auto& p) { return p == x.second; }), + "Invalid system: " + x.second); + } + + settings.ClusterMapping = request.ClusterMapping; + settings.DynamicClusterProvider = request.ClusterSystem; + break; + case EClusterMode::Single: + Y_ENSURE(!request.ClusterSystem.empty(), "Missing ClusterSystem parameter"); + settings.DefaultCluster = "single"; + settings.ClusterMapping["single"] = request.ClusterSystem; + settings.DynamicClusterProvider = request.ClusterSystem; + break; + case EClusterMode::Unknown: + settings.DefaultCluster = "single"; + settings.ClusterMapping["single"] = UnknownProviderName; + settings.DynamicClusterProvider = UnknownProviderName; + break; } } }; -} +} // namespace std::unique_ptr<ICheckRunner> MakeTranslatorRunner() { return std::make_unique<TTranslatorRunner>(); } -} -} +} // namespace NFastCheck +} // namespace NYql diff --git a/yql/essentials/public/fastcheck/ut/ya.make b/yql/essentials/public/fastcheck/ut/ya.make index 91a2c5c9b40..244dffd1249 100644 --- a/yql/essentials/public/fastcheck/ut/ya.make +++ b/yql/essentials/public/fastcheck/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(yql/essentials/public/fastcheck) +ENABLE(YQL_STYLE_CPP) + SRCS( fastcheck_ut.cpp linter_ut.cpp diff --git a/yql/essentials/public/fastcheck/ya.make b/yql/essentials/public/fastcheck/ya.make index 2b374581d43..01999ded3bd 100644 --- a/yql/essentials/public/fastcheck/ya.make +++ b/yql/essentials/public/fastcheck/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( check_runner.cpp fastcheck.cpp diff --git a/yql/essentials/public/issue/ut/ya.make b/yql/essentials/public/issue/ut/ya.make index 46f35941bcb..28bebef5d25 100644 --- a/yql/essentials/public/issue/ut/ya.make +++ b/yql/essentials/public/issue/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(yql/essentials/public/issue) +ENABLE(YQL_STYLE_CPP) + FORK_SUBTESTS() SRCS( diff --git a/yql/essentials/public/issue/ya.make b/yql/essentials/public/issue/ya.make index f376c8bcda7..5d91d21a7f7 100644 --- a/yql/essentials/public/issue/ya.make +++ b/yql/essentials/public/issue/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( yql_issue.cpp yql_issue_message.cpp diff --git a/yql/essentials/public/issue/yql_issue.cpp b/yql/essentials/public/issue/yql_issue.cpp index 02d8bd2e273..d91ca5af1b3 100644 --- a/yql/essentials/public/issue/yql_issue.cpp +++ b/yql/essentials/public/issue/yql_issue.cpp @@ -15,7 +15,6 @@ #include <util/generic/stack.h> #include <cstdlib> - namespace NYql { void SanitizeNonAscii(TString& s) { @@ -117,12 +116,12 @@ Y_NO_INLINE void Indent(IOutputStream& out, ui32 indentation) { } void ProgramLinesWithErrors( - const TString& programText, - const TVector<TIssue>& errors, - TMap<ui32, TStringBuf>& lines) + const TString& programText, + const TVector<TIssue>& errors, + TMap<ui32, TStringBuf>& lines) { TVector<ui32> rows; - for (const auto& topIssue: errors) { + for (const auto& topIssue : errors) { WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 /*level*/) { for (ui32 row = issue.Position.Row; row <= issue.EndPosition.Row; row++) { rows.push_back(row); @@ -135,7 +134,7 @@ void ProgramLinesWithErrors( auto progIt = prog.begin(), progEnd = prog.end(); ui32 progRow = 1; - for (ui32 row: rows) { + for (ui32 row : rows) { while (progRow < row && progIt != progEnd) { ++progRow; ++progIt; @@ -146,34 +145,30 @@ void ProgramLinesWithErrors( } } -} // namspace +} // namespace -void TIssues::PrintTo(IOutputStream& out, bool oneLine) const -{ +void TIssues::PrintTo(IOutputStream& out, bool oneLine) const { if (oneLine) { bool printWithSpace = false; if (Issues_.size() > 1) { printWithSpace = true; out << "["; } - for (const auto& topIssue: Issues_) { + for (const auto& topIssue : Issues_) { WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { if (level > 0) { out << " subissue: { "; } else { out << (printWithSpace ? " { " : "{ "); } - issue.PrintTo(out, true); - }, - [&](const TIssue&, ui16) { - out << " }"; - }); + issue.PrintTo(out, true); }, + [&](const TIssue&, ui16) { out << " }"; }); } if (Issues_.size() > 1) { out << " ]"; } } else { - for (const auto& topIssue: Issues_) { + for (const auto& topIssue : Issues_) { WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { auto shift = level * 4; Indent(out, shift); @@ -184,17 +179,16 @@ void TIssues::PrintTo(IOutputStream& out, bool oneLine) const } void TIssues::PrintWithProgramTo( - IOutputStream& out, - const TString& programFilename, - const TString& programText, - bool colorize) const -{ + IOutputStream& out, + const TString& programFilename, + const TString& programText, + bool colorize) const { using namespace NColorizer; TMap<ui32, TStringBuf> lines; ProgramLinesWithErrors(programText, Issues_, lines); - for (const TIssue& topIssue: Issues_) { + for (const TIssue& topIssue : Issues_) { WalkThroughIssues(topIssue, false, [&](const TIssue& issue, ui16 level) { auto shift = level * 4; Indent(out, shift); @@ -227,7 +221,7 @@ void TIssues::PrintWithProgramTo( } } - out << ": "<< severityName << ": " << issue.GetMessage(); + out << ": " << severityName << ": " << issue.GetMessage(); if (colorize) { out << Old(); } @@ -270,7 +264,7 @@ TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message) { endPos = message.find(')', startPos + TerminationMessageMarker.size()); if (endPos != TString::npos) { TStringBuf lenText = message.Tail(startPos + TerminationMessageMarker.size()) - .Trunc(endPos - startPos - TerminationMessageMarker.size()); + .Trunc(endPos - startPos - TerminationMessageMarker.size()); try { len = FromString<size_t>(lenText); } catch (const TFromStringException&) { @@ -298,7 +292,7 @@ TMaybe<TPosition> TryParseTerminationMessage(TStringBuf& message) { return Nothing(); } -} // namspace NYql +} // namespace NYql template <> void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos) { @@ -308,8 +302,8 @@ void Out<NYql::TPosition>(IOutputStream& out, const NYql::TPosition& pos) { } } -template<> -void Out<NYql::TRange>(IOutputStream & out, const NYql::TRange & range) { +template <> +void Out<NYql::TRange>(IOutputStream& out, const NYql::TRange& range) { if (range.IsRange()) { out << '[' << range.Position << '-' << range.EndPosition << ']'; } else { diff --git a/yql/essentials/public/issue/yql_issue.h b/yql/essentials/public/issue/yql_issue.h index 7f354b8670b..4ccc6a88ac7 100644 --- a/yql/essentials/public/issue/yql_issue.h +++ b/yql/essentials/public/issue/yql_issue.h @@ -61,7 +61,7 @@ public: { } - template<typename T> + template <typename T> TTextWalker& Advance(const T& buf) { for (char c : buf) { Advance(c); @@ -108,6 +108,7 @@ using TIssuePtr = TIntrusivePtr<TIssue>; class TIssue: public TThrRefBase { TVector<TIntrusivePtr<TIssue>> Children_; TString Message_; + public: TPosition Position; TPosition EndPosition; @@ -135,7 +136,7 @@ public: } inline TRange Range() const { - return{ Position, EndPosition }; + return {Position, EndPosition}; } template <typename T> @@ -148,16 +149,14 @@ public: } inline bool operator==(const TIssue& other) const { - return Position == other.Position && Message_ == other.Message_ - && IssueCode == other.IssueCode; + return Position == other.Position && Message_ == other.Message_ && IssueCode == other.IssueCode; } ui64 Hash() const noexcept { return CombineHashes( CombineHashes( (size_t)CombineHashes(IntHash(Position.Row), IntHash(Position.Column)), - ComputeHash(Position.File) - ), + ComputeHash(Position.File)), (size_t)CombineHashes((size_t)IntHash(static_cast<int>(IssueCode)), ComputeHash(Message_))); } @@ -247,7 +246,8 @@ public: return *this; } - inline TIssues(TIssues&& rhs) : Issues_(std::move(rhs.Issues_)) + inline TIssues(TIssues&& rhs) + : Issues_(std::move(rhs.Issues_)) { } @@ -256,7 +256,8 @@ public: return *this; } - template <typename ... Args> void AddIssue(Args&& ... args) { + template <typename... Args> + void AddIssue(Args&&... args) { Issues_.emplace_back(std::forward<Args>(args)...); } @@ -267,7 +268,7 @@ public: inline void AddIssues(const TPosition& pos, const TIssues& errors) { Issues_.reserve(Issues_.size() + errors.Size()); - for (const auto& e: errors) { + for (const auto& e : errors) { TIssue& issue = Issues_.emplace_back(); *issue.MutableMessage() = e.GetMessage(); // No need to sanitize message, it has already been sanitized. issue.Position = pos; @@ -305,10 +306,10 @@ public: void PrintTo(IOutputStream& out, bool oneLine = false) const; void PrintWithProgramTo( - IOutputStream& out, - const TString& programFilename, - const TString& programText, - bool colorize = true) const; + IOutputStream& out, + const TString& programFilename, + const TString& programText, + bool colorize = true) const; inline TString ToString(bool oneLine = false) const { TStringStream out; @@ -332,12 +333,14 @@ private: TVector<TIssue> Issues_; }; -class TErrorException : public yexception { +class TErrorException: public yexception { const TIssueCode Code_; + public: explicit TErrorException(TIssueCode code) : Code_(code) - {} + { + } TIssueCode GetCode() const { return Code_; } diff --git a/yql/essentials/public/issue/yql_issue_id.h b/yql/essentials/public/issue/yql_issue_id.h index b4889319a77..692fdb6216d 100644 --- a/yql/essentials/public/issue/yql_issue_id.h +++ b/yql/essentials/public/issue/yql_issue_id.h @@ -14,9 +14,9 @@ #include <util/string/subst.h> #ifdef _win_ -#ifdef GetMessage -#undef GetMessage -#endif + #ifdef GetMessage + #undef GetMessage + #endif #endif namespace NYql { @@ -29,7 +29,7 @@ const TIssueCode UNEXPECTED_ERROR = 1; inline TString SeverityToString(ESeverity severity) { auto ret = NYql::TSeverityIds::ESeverityId_Name(severity); - return ret.empty() ? "Unknown" : to_title(ret.substr(2)); //remove prefix "S_" + return ret.empty() ? "Unknown" : to_title(ret.substr(2)); // remove prefix "S_" } template <typename T> @@ -43,7 +43,7 @@ inline TString IssueCodeToString(TIssueCode id) { } } -template<typename TProto, const char* ResourceName> +template <typename TProto, const char* ResourceName> class TIssueId { TProto ProtoIssues_; THashMap<TIssueCode, NYql::TSeverityIds::ESeverityId> IssuesMap_; @@ -56,7 +56,7 @@ class TIssueId { } bool CheckSeverityNameFormat(const TString& name) const { - if (name.size() > 2 && name.substr(0,2) == "S_") { + if (name.size() > 2 && name.substr(0, 2) == "S_") { return true; } return false; @@ -66,14 +66,14 @@ public: ESeverity GetSeverity(TIssueCode id) const { auto it = IssuesMap_.find(id); Y_ENSURE(it != IssuesMap_.end(), "Unknown issue id: " - << id << "(" << IssueCodeToString<TProto>(id) << ")"); + << id << "(" << IssueCodeToString<TProto>(id) << ")"); return it->second; } TString GetMessage(TIssueCode id) const { auto it = IssuesFormatMap_.find(id); Y_ENSURE(it != IssuesFormatMap_.end(), "Unknown issue id: " - << id << "(" << IssueCodeToString<TProto>(id) << ")"); + << id << "(" << IssueCodeToString<TProto>(id) << ")"); return it->second; } @@ -87,14 +87,14 @@ public: for (int i = 0; i < sDesc->value_count(); i++) { const auto& name = sDesc->value(i)->name(); Y_ENSURE(CheckSeverityNameFormat(name), - "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix"); + "Wrong severity name: " << name << ". Severity must starts with \"S_\" prefix"); } for (const auto& x : ProtoIssues_.ids()) { auto rv = IssuesMap_.insert(std::make_pair(x.code(), x.severity())); Y_ENSURE(rv.second, "Duplicate issue code found, code: " - << static_cast<int>(x.code()) - << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) << ")"); } // Check all IssueCodes have mapping to severity @@ -102,27 +102,27 @@ public: for (int i = 0; i < eDesc->value_count(); i++) { auto it = IssuesMap_.find(eDesc->value(i)->number()); Y_ENSURE(it != IssuesMap_.end(), "IssueCode: " - << eDesc->value(i)->name() - << " is not found in protobuf data file"); + << eDesc->value(i)->name() + << " is not found in protobuf data file"); } for (const auto& x : ProtoIssues_.ids()) { auto rv = IssuesFormatMap_.insert(std::make_pair(x.code(), x.format())); Y_ENSURE(rv.second, "Duplicate issue code found, code: " - << static_cast<int>(x.code()) - << "(" << IssueCodeToString<TProto>(x.code()) <<")"); + << static_cast<int>(x.code()) + << "(" << IssueCodeToString<TProto>(x.code()) << ")"); } } }; -template<typename TProto, const char* ResourceName> +template <typename TProto, const char* ResourceName> inline ESeverity GetSeverity(TIssueCode id) { return Singleton<TIssueId<TProto, ResourceName>>()->GetSeverity(id); } -template<typename TProto, const char* ResourceName> +template <typename TProto, const char* ResourceName> inline TString GetMessage(TIssueCode id) { return Singleton<TIssueId<TProto, ResourceName>>()->GetMessage(id); } -} +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_manager.cpp b/yql/essentials/public/issue/yql_issue_manager.cpp index 8ffeed9fb08..1d1ed624282 100644 --- a/yql/essentials/public/issue/yql_issue_manager.cpp +++ b/yql/essentials/public/issue/yql_issue_manager.cpp @@ -55,7 +55,7 @@ void TIssueManager::LeaveScope() { RawIssues_.top().first = new TIssue(); (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); } else { - (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; + (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; } } @@ -93,7 +93,7 @@ TIssuePtr TIssueManager::CheckUniqAndLimit(TIssuePtr issue) { } if (IssueLimit_ && UniqueIssues_[severity].size() == IssueLimit_) { OverflowIssues_[severity] = MakeIntrusive<TIssue>(TStringBuilder() - << "Too many " << SeverityToString(issue->GetSeverity()) << " issues"); + << "Too many " << SeverityToString(issue->GetSeverity()) << " issues"); OverflowIssues_[severity]->Severity = severity; return {}; } @@ -125,7 +125,7 @@ void TIssueManager::RaiseIssue(const TIssue& issue) { (*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO); } else { (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO; - } + } } RawIssues_.top().first->Get()->AddSubIssue(p); } @@ -179,7 +179,7 @@ TIssues TIssueManager::GetIssues() { TIssues TIssueManager::GetCompletedIssues() const { TIssues res; - for (auto& p: OverflowIssues_) { + for (auto& p : OverflowIssues_) { if (p) { res.AddIssue(*p); } @@ -189,7 +189,7 @@ TIssues TIssueManager::GetCompletedIssues() const { } void TIssueManager::AddIssues(const TIssues& issues) { - for (auto& issue: issues) { + for (auto& issue : issues) { if (auto p = CheckUniqAndLimit(issue)) { CompletedIssues_.AddIssue(*p); } @@ -197,7 +197,7 @@ void TIssueManager::AddIssues(const TIssues& issues) { } void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) { - for (auto& issue: issues) { + for (auto& issue : issues) { if (auto p = CheckUniqAndLimit(TIssue(pos, issue.GetMessage()))) { CompletedIssues_.AddIssue(*p); } @@ -205,11 +205,11 @@ void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) { } void TIssueManager::Reset(const TIssues& issues) { - for (auto& p: OverflowIssues_) { + for (auto& p : OverflowIssues_) { p.Drop(); } - for (auto& s: UniqueIssues_) { + for (auto& s : UniqueIssues_) { s.clear(); } CompletedIssues_.Clear(); @@ -224,7 +224,7 @@ void TIssueManager::Reset() { Reset(TIssues()); } -void TIssueManager::AddWarningRule(const TWarningRule &rule) +void TIssueManager::AddWarningRule(const TWarningRule& rule) { WarningPolicy_.AddRule(rule); } diff --git a/yql/essentials/public/issue/yql_issue_manager.h b/yql/essentials/public/issue/yql_issue_manager.h index 6b6ec2c7194..8a5ef8daea1 100644 --- a/yql/essentials/public/issue/yql_issue_manager.h +++ b/yql/essentials/public/issue/yql_issue_manager.h @@ -27,7 +27,7 @@ public: void Reset(const TIssues& issues); void Reset(); - void AddWarningRule(const TWarningRule &rule); + void AddWarningRule(const TWarningRule& rule); void SetWarningToErrorTreatMessage(const TString& msg); void SetIssueCountLimit(size_t limit) { @@ -63,6 +63,7 @@ private: class TIssueScopeGuard: private TNonCopyable { TIssueManager& Manager_; + public: TIssueScopeGuard(TIssueManager& manager, std::function<TIssuePtr()> fn) : Manager_(manager) @@ -80,4 +81,4 @@ public: } }; -} +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_manager_ut.cpp b/yql/essentials/public/issue/yql_issue_manager_ut.cpp index e3308163c34..9c135ecd15f 100644 --- a/yql/essentials/public/issue/yql_issue_manager_ut.cpp +++ b/yql/essentials/public/issue/yql_issue_manager_ut.cpp @@ -11,186 +11,186 @@ static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 co } Y_UNIT_TEST_SUITE(TIssueManagerTest) { - Y_UNIT_TEST(NoErrorNoLevelTest) { - TIssueManager issueManager; - auto completedIssues = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); - } +Y_UNIT_TEST(NoErrorNoLevelTest) { + TIssueManager issueManager; + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); +} - Y_UNIT_TEST(NoErrorOneLevelTest) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - auto completedIssues = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); - } +Y_UNIT_TEST(NoErrorOneLevelTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); +} - Y_UNIT_TEST(NoErrorTwoLevelsTest) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.LeaveScope(); - auto completedIssues = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); - } +Y_UNIT_TEST(NoErrorTwoLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.LeaveScope(); + auto completedIssues = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 0); +} - Y_UNIT_TEST(OneErrorOneLevelTest) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - auto completedIssues1 = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - auto completedIssues2 = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); - issueManager.LeaveScope(); - auto completedIssues3 = issueManager.GetCompletedIssues(); - UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); +Y_UNIT_TEST(OneErrorOneLevelTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + auto completedIssues1 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueOne")); + auto completedIssues2 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues2.Size(), 0); + issueManager.LeaveScope(); + auto completedIssues3 = issueManager.GetCompletedIssues(); + UNIT_ASSERT_VALUES_EQUAL(completedIssues3.Size(), 1); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "IssueOne"); - } + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "IssueOne"); +} - Y_UNIT_TEST(OneErrorTwoLevelsTest) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "B"); +Y_UNIT_TEST(OneErrorTwoLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "B"); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->GetMessage(), "IssueOne"); - } + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues()[0]->GetMessage(), "IssueOne"); +} - Y_UNIT_TEST(MultiErrorsMultiLevelsTest) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1")); - issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2)); - issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3")); - issueManager.LeaveScope(); - issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2")); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); //WarningScope1 - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "WarningScope1"); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->GetMessage(), "ErrorScope3"); - UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->GetMessage(), "ErrorScope2"); - auto ref = R"___(<main>: Error: A +Y_UNIT_TEST(MultiErrorsMultiLevelsTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1")); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2)); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetSubIssues().size(), 0); // WarningScope1 + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "WarningScope1"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues().size(), 2); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[0]->GetSubIssues()[0]->GetMessage(), "ErrorScope3"); + UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->GetMessage(), "ErrorScope2"); + auto ref = R"___(<main>: Error: A <main>: Error: WarningScope1 <main>:1:1: Error: B <main>:2:2: Error: C <main>: Error: ErrorScope3 <main>: Error: ErrorScope2 )___"; - UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); - } + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); +} - Y_UNIT_TEST(TIssueScopeGuardSimpleTest) { - TIssueManager issueManager; - { - TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0)); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1")); - } - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto scopeIssue = issues.begin(); - UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); - auto subIssues = scopeIssue->GetSubIssues(); - UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "ErrorScope1"); +Y_UNIT_TEST(TIssueScopeGuardSimpleTest) { + TIssueManager issueManager; + { + TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "ErrorScope1")); } + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto scopeIssue = issues.begin(); + UNIT_ASSERT_VALUES_EQUAL(scopeIssue->GetMessage(), "A"); + auto subIssues = scopeIssue->GetSubIssues(); + UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->GetMessage(), "ErrorScope1"); +} - Y_UNIT_TEST(FuseScopesTest) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0)); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - issueManager.LeaveScope(); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo")); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - auto ref = R"___(<main>: Error: A +Y_UNIT_TEST(FuseScopesTest) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0)); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueOne")); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueTwo")); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + auto ref = R"___(<main>: Error: A <main>: Error: B, C <main>: Error: D <main>:2:1: Error: IssueOne <main>:2:1: Error: IssueTwo )___"; - UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); - } + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); +} - Y_UNIT_TEST(UniqIssues) { - TIssueManager issueManager; - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo")); - issueManager.RaiseIssue(TIssue(TPosition(2,3), "IssueOne")); - issueManager.RaiseWarning(TIssue(TPosition(2,3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne")); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); - auto ref = R"___(<main>: Error: A +Y_UNIT_TEST(UniqIssues) { + TIssueManager issueManager; + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueOne")); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueTwo")); + issueManager.RaiseIssue(TIssue(TPosition(2, 3), "IssueOne")); + issueManager.RaiseWarning(TIssue(TPosition(2, 3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + issueManager.RaiseIssue(TIssue(TPosition(1, 2), "IssueOne")); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1); + auto ref = R"___(<main>: Error: A <main>:1:1: Error: B <main>:2:1: Error: IssueOne <main>:2:1: Error: IssueTwo <main>:3:2: Error: IssueOne <main>:3:2: Warning: IssueOne, code: 1 )___"; - UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); - } + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); +} - Y_UNIT_TEST(Limits) { - TIssueManager issueManager; - issueManager.SetIssueCountLimit(2); - issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); - issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue1")); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue2")); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue3")); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); - issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); - issueManager.LeaveScope(); - issueManager.LeaveScope(); - auto issues = issueManager.GetIssues(); - UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3); - auto ref = R"___(<main>: Error: Too many Error issues +Y_UNIT_TEST(Limits) { + TIssueManager issueManager; + issueManager.SetIssueCountLimit(2); + issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0)); + issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1)); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue1")); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue2")); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue3")); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING)); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); + issueManager.RaiseIssue(TIssue(TPosition(1, 1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO)); + issueManager.LeaveScope(); + issueManager.LeaveScope(); + auto issues = issueManager.GetIssues(); + UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3); + auto ref = R"___(<main>: Error: Too many Error issues <main>: Warning: Too many Warning issues <main>: Error: A <main>:1:1: Error: B @@ -201,6 +201,6 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) { <main>:1:1: Info: Issue7, code: 2 <main>:1:1: Info: Issue8, code: 2 )___"; - UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); - } + UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref); } +} // Y_UNIT_TEST_SUITE(TIssueManagerTest) diff --git a/yql/essentials/public/issue/yql_issue_message.cpp b/yql/essentials/public/issue/yql_issue_message.cpp index d759edf432f..59742a8b776 100644 --- a/yql/essentials/public/issue/yql_issue_message.cpp +++ b/yql/essentials/public/issue/yql_issue_message.cpp @@ -11,17 +11,13 @@ namespace NYql { using namespace NIssue::NProto; -template -TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage); +template TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage); -template -void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues); +template void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues); -template -void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage); +template void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage); -template -void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message); +template void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message); NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { NIssue::NProto::IssueMessage issueMessage; @@ -29,8 +25,8 @@ NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { return issueMessage; } -} +} // namespace NYql Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>, stream, issues) { stream << JoinSeq("", issues); -}
\ No newline at end of file +} diff --git a/yql/essentials/public/issue/yql_issue_message.h b/yql/essentials/public/issue/yql_issue_message.h index be1d1a45bb8..d60ce09ce95 100644 --- a/yql/essentials/public/issue/yql_issue_message.h +++ b/yql/essentials/public/issue/yql_issue_message.h @@ -9,15 +9,15 @@ namespace NYql { namespace NIssue { namespace NProto { class IssueMessage; -} -} +} // namespace NProto +} // namespace NIssue -template<typename TIssueMessage> +template <typename TIssueMessage> TIssue IssueFromMessage(const TIssueMessage& issueMessage); -template<typename TIssueMessage> +template <typename TIssueMessage> void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues); -template<typename TIssueMessage> +template <typename TIssueMessage> TString IssuesFromMessageAsString(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message) { TIssues issues; IssuesFromMessage(message, issues); @@ -26,9 +26,9 @@ TString IssuesFromMessageAsString(const ::google::protobuf::RepeatedPtrField<TIs NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue); -template<typename TIssueMessage> +template <typename TIssueMessage> void IssueToMessage(const TIssue& topIssue, TIssueMessage* message); -template<typename TIssueMessage> +template <typename TIssueMessage> void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message); -} +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_message_impl.h b/yql/essentials/public/issue/yql_issue_message_impl.h index c59134e183a..a169f8cd303 100644 --- a/yql/essentials/public/issue/yql_issue_message_impl.h +++ b/yql/essentials/public/issue/yql_issue_message_impl.h @@ -10,7 +10,7 @@ namespace NYql { using namespace NIssue::NProto; -template<typename TIssueMessage> +template <typename TIssueMessage> TIssue IssueFromMessage(const TIssueMessage& issueMessage) { TIssue topIssue; TDeque<std::pair<TIssue*, const TIssueMessage*>> queue; @@ -42,17 +42,18 @@ TIssue IssueFromMessage(const TIssueMessage& issueMessage) { return topIssue; } -template<typename TIssueMessage> -void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage> &message, TIssues &issues) { +template <typename TIssueMessage> +void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues) { issues.Clear(); if (message.size()) { issues.Reserve(message.size()); - for (auto &x : message) + for (auto& x : message) { issues.AddIssue(IssueFromMessage(x)); + } } } -template<typename TIssueMessage> +template <typename TIssueMessage> void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { TDeque<std::pair<const TIssue*, TIssueMessage*>> queue; queue.push_front(std::make_pair(&topIssue, issueMessage)); @@ -82,15 +83,16 @@ void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { } } -template<typename TIssueMessage> -void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) { +template <typename TIssueMessage> +void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message) { message->Clear(); - if (!issues) + if (!issues) { return; + } message->Reserve(issues.Size()); - for (const auto &issue : issues) { + for (const auto& issue : issues) { IssueToMessage(issue, message->Add()); } } -} +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_ut.cpp b/yql/essentials/public/issue/yql_issue_ut.cpp index 9076c0ee3da..00f146555d0 100644 --- a/yql/essentials/public/issue/yql_issue_ut.cpp +++ b/yql/essentials/public/issue/yql_issue_ut.cpp @@ -18,123 +18,123 @@ using namespace google::protobuf; using namespace NYql; Y_UNIT_TEST_SUITE(IssueTest) { - Y_UNIT_TEST(Ascii) { - TIssue issue1("тест abc"); - UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); - TIssue issue2("\xFF abc"); - UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); - TIssue issue3(""); - UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); - TIssue issue4("abc"); - UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); - } +Y_UNIT_TEST(Ascii) { + TIssue issue1("тест abc"); + UNIT_ASSERT_VALUES_EQUAL(issue1.GetMessage(), "тест abc"); + TIssue issue2("\xFF abc"); + UNIT_ASSERT_VALUES_EQUAL(issue2.GetMessage(), "? abc"); + TIssue issue3(""); + UNIT_ASSERT_VALUES_EQUAL(issue3.GetMessage(), ""); + TIssue issue4("abc"); + UNIT_ASSERT_VALUES_EQUAL(issue4.GetMessage(), "abc"); } +} // Y_UNIT_TEST_SUITE(IssueTest) Y_UNIT_TEST_SUITE(TextWalkerTest) { - Y_UNIT_TEST(BasicTest) { - TPosition pos; - pos.Row = 1; +Y_UNIT_TEST(BasicTest) { + TPosition pos; + pos.Row = 1; - TTextWalker walker(pos, false); - walker.Advance(TStringBuf("a\r\taa")); + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("a\r\taa")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); - walker.Advance(TStringBuf("\na")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); - } + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); + walker.Advance(TStringBuf("\na")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 2)); +} + +Y_UNIT_TEST(CrLfTest) { + TPosition pos; + pos.Row = 1; + + TTextWalker walker(pos, false); + walker.Advance(TStringBuf("a\raa\r")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(0, 2)); + walker.Advance(TStringBuf("\r\r\ra")); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); + walker.Advance('\r'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 2)); + walker.Advance('\n'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(0, 3)); + walker.Advance('a'); + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); +} - Y_UNIT_TEST(CrLfTest) { +Y_UNIT_TEST(UnicodeTest) { + { TPosition pos; pos.Row = 1; TTextWalker walker(pos, false); - walker.Advance(TStringBuf("a\raa\r")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 1)); - walker.Advance('\n'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(0, 2)); - walker.Advance(TStringBuf("\r\r\ra")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(4, 2)); - walker.Advance('\r'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(5, 2)); - walker.Advance('\n'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(0, 3)); - walker.Advance('a'); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(1, 3)); - } - - Y_UNIT_TEST(UnicodeTest) { - { - TPosition pos; - pos.Row = 1; + walker.Advance(TStringBuf("привет")); - TTextWalker walker(pos, false); - walker.Advance(TStringBuf("привет")); - - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(12, 1)); - } + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(12, 1)); + } - { - TPosition pos; - pos.Row = 1; + { + TPosition pos; + pos.Row = 1; - TTextWalker walker(pos, true); - walker.Advance(TStringBuf("привет")); + TTextWalker walker(pos, true); + walker.Advance(TStringBuf("привет")); - UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(6, 1)); - } + UNIT_ASSERT_VALUES_EQUAL(pos, TPosition(6, 1)); } } +} // Y_UNIT_TEST_SUITE(TextWalkerTest) Y_UNIT_TEST_SUITE(ToOneLineStringTest) { - Y_UNIT_TEST(OneMessageTest) { - TIssues issues; - issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); - UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); - } +Y_UNIT_TEST(OneMessageTest) { + TIssues issues; + issues.AddIssue(TPosition(12, 34, "file.abc"), "error"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error }"); +} - Y_UNIT_TEST(SubIssuesTest) { - TIssue issue(TPosition(12, 34, "file.abc"), "error"); - TIssue subissue("suberror"); - subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror")); - issue.AddSubIssue(MakeIntrusive<TIssue>(subissue)); +Y_UNIT_TEST(SubIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error"); + TIssue subissue("suberror"); + subissue.AddSubIssue(MakeIntrusive<TIssue>("subsuberror")); + issue.AddSubIssue(MakeIntrusive<TIssue>(subissue)); - TIssues issues; - issues.AddIssue(issue); - UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: { <main>: Error: suberror subissue: { <main>: Error: subsuberror } } }"); - } + TIssues issues; + issues.AddIssue(issue); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "{ file.abc:34:12: Error: error subissue: { <main>: Error: suberror subissue: { <main>: Error: subsuberror } } }"); +} - Y_UNIT_TEST(ManyIssuesTest) { - TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); - issue.AddSubIssue(MakeIntrusive<TIssue>("suberror")); - TIssues issues; - issues.AddIssue(issue); - issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); - UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: { <main>: Error: suberror } } { abc.file:2:100: Error: my message } ]"); - } +Y_UNIT_TEST(ManyIssuesTest) { + TIssue issue(TPosition(12, 34, "file.abc"), "error\n"); + issue.AddSubIssue(MakeIntrusive<TIssue>("suberror")); + TIssues issues; + issues.AddIssue(issue); + issues.AddIssue(TPosition(100, 2, "abc.file"), "my\nmessage"); + UNIT_ASSERT_STRINGS_EQUAL(issues.ToOneLineString(), "[ { file.abc:34:12: Error: error subissue: { <main>: Error: suberror } } { abc.file:2:100: Error: my message } ]"); } +} // Y_UNIT_TEST_SUITE(ToOneLineStringTest) Y_UNIT_TEST_SUITE(EscapeNonUtf8) { - Y_UNIT_TEST(Escape) { - const TString nonUtf8String = "\xfe\xfa\xf5\xc2"; - UNIT_ASSERT(!IsUtf(nonUtf8String)); - - // Check that our escaping correctly processes unicode pairs - const TString toNormalize = "Ёлка"; - const TString nfd = WideToUTF8(Normalize<NUnicode::ENormalization::NFD>(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol - const TString nfc = WideToUTF8(Normalize<NUnicode::ENormalization::NFC>(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter - UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); - std::pair<TString, TString> nonUtf8Messages[] = { - { nonUtf8String, "????" }, - { TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????" }, - { nfd, nfd }, - { nfc, nfc }, - { TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd }, - { TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc }, - }; - - for (const auto& [src, dst] : nonUtf8Messages) { - TIssue issue(src); - UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); - } +Y_UNIT_TEST(Escape) { + const TString nonUtf8String = "\xfe\xfa\xf5\xc2"; + UNIT_ASSERT(!IsUtf(nonUtf8String)); + + // Check that our escaping correctly processes unicode pairs + const TString toNormalize = "Ёлка"; + const TString nfd = WideToUTF8(Normalize<NUnicode::ENormalization::NFD>(UTF8ToWide(toNormalize))); // dots over 'ё' will be separate unicode symbol + const TString nfc = WideToUTF8(Normalize<NUnicode::ENormalization::NFC>(UTF8ToWide(toNormalize))); // dots over 'ё' will be with with their letter + UNIT_ASSERT_STRINGS_UNEQUAL(nfc, nfd); + std::pair<TString, TString> nonUtf8Messages[] = { + {nonUtf8String, "????"}, + {TStringBuilder() << nonUtf8String << "Failed to parse file " << nonUtf8String << "עברית" << nonUtf8String, "????Failed to parse file ????עברית????"}, + {nfd, nfd}, + {nfc, nfc}, + {TStringBuilder() << nfc << nonUtf8String << nfd, TStringBuilder() << nfc << "????" << nfd}, + {TStringBuilder() << nfd << nonUtf8String << nfc, TStringBuilder() << nfd << "????" << nfc}, + }; + + for (const auto& [src, dst] : nonUtf8Messages) { + TIssue issue(src); + UNIT_ASSERT_STRINGS_EQUAL(issue.GetMessage(), dst); } } +} // Y_UNIT_TEST_SUITE(EscapeNonUtf8) diff --git a/yql/essentials/public/issue/yql_issue_utils.cpp b/yql/essentials/public/issue/yql_issue_utils.cpp index 2be0b8e57bb..8f441122f72 100644 --- a/yql/essentials/public/issue/yql_issue_utils.cpp +++ b/yql/essentials/public/issue/yql_issue_utils.cpp @@ -43,7 +43,7 @@ TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) { } if (depthBeforeLeaf && !leafs.empty()) { - for (size_t pos: leafs) { + for (size_t pos : leafs) { ui32 depth = depthBeforeLeaf; auto parent = std::get<3>(issues.at(pos)); while (depth && parent) { @@ -64,7 +64,7 @@ TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) { leafs.clear(); TIssue result; - for (auto& i: issues) { + for (auto& i : issues) { const auto srcIssue = std::get<0>(i); auto& targetIssue = std::get<1>(i); const auto level = std::get<2>(i); @@ -102,4 +102,4 @@ TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) { return result; } -} // namspace NYql +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_issue_utils.h b/yql/essentials/public/issue/yql_issue_utils.h index 023ea0edbfb..c820d7c5383 100644 --- a/yql/essentials/public/issue/yql_issue_utils.h +++ b/yql/essentials/public/issue/yql_issue_utils.h @@ -8,16 +8,15 @@ namespace NYql { struct TTruncateIssueOpts { - -#define YQL_TRUNC_DECL_FIELD(type, name, def) \ +#define YQL_TRUNC_DECL_FIELD(type, name, def) \ TTruncateIssueOpts& Set##name(type arg##name)& { \ - name = arg##name; \ - return *this; \ - } \ + name = arg##name; \ + return *this; \ + } \ TTruncateIssueOpts&& Set##name(type arg##name)&& { \ - name = arg##name; \ - return std::move(*this); \ - } \ + name = arg##name; \ + return std::move(*this); \ + } \ type name = def; YQL_TRUNC_DECL_FIELD(ui32, MaxLevels, Max<ui32>()) diff --git a/yql/essentials/public/issue/yql_issue_utils_ut.cpp b/yql/essentials/public/issue/yql_issue_utils_ut.cpp index 657643a6134..685d652ef7e 100644 --- a/yql/essentials/public/issue/yql_issue_utils_ut.cpp +++ b/yql/essentials/public/issue/yql_issue_utils_ut.cpp @@ -9,42 +9,42 @@ using namespace NYql; Y_UNIT_TEST_SUITE(TIssueUtilsTest) { - Y_UNIT_TEST(TruncLevels1) { - auto level0 = MakeIntrusive<TIssue>("level0"); - auto level1 = MakeIntrusive<TIssue>("level1"); - auto level2 = MakeIntrusive<TIssue>("level2"); - auto level30 = MakeIntrusive<TIssue>("level30"); - auto level31 = MakeIntrusive<TIssue>("level31"); - auto level40 = MakeIntrusive<TIssue>("level40"); - auto level41 = MakeIntrusive<TIssue>("level41"); - auto level51 = MakeIntrusive<TIssue>("level51"); - - /* - * * 0 - * | - * * 1 - * | - * * 2 -- - * | | - * * 30 * 31 - * | | - * * 40 * 41 - * | - * * 51 - */ - - level0->AddSubIssue(level1); - level1->AddSubIssue(level2); - level2->AddSubIssue(level30); - level2->AddSubIssue(level31); - level30->AddSubIssue(level40); - level31->AddSubIssue(level41); - level41->AddSubIssue(level51); - - { - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); - const auto expected = -R"___(<main>: Error: level0 +Y_UNIT_TEST(TruncLevels1) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level30 = MakeIntrusive<TIssue>("level30"); + auto level31 = MakeIntrusive<TIssue>("level31"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + auto level51 = MakeIntrusive<TIssue>("level51"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 -- + * | | + * * 30 * 31 + * | | + * * 40 * 41 + * | + * * 51 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level30); + level2->AddSubIssue(level31); + level30->AddSubIssue(level40); + level31->AddSubIssue(level41); + level41->AddSubIssue(level51); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = + R"___(<main>: Error: level0 <main>: Error: level1 <main>: Error: (skipped levels) <main>: Error: level30 @@ -52,109 +52,109 @@ R"___(<main>: Error: level0 <main>: Error: level41 <main>: Error: level51 )___"; - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } - { - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString(); - const auto expected = -R"___(<main>: Error: level0 + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString(); + const auto expected = + R"___(<main>: Error: level0 <main>: Error: level1 <main>: Error: (skipped levels) <main>: Error: level40 <main>: Error: level51 )___"; - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + UNIT_ASSERT_STRINGS_EQUAL(res, expected); } +} - Y_UNIT_TEST(TruncLevels2) { - auto level0 = MakeIntrusive<TIssue>("level0"); - auto level1 = MakeIntrusive<TIssue>("level1"); - auto level2 = MakeIntrusive<TIssue>("level2"); - auto level3 = MakeIntrusive<TIssue>("level3"); - auto level40 = MakeIntrusive<TIssue>("level40"); - auto level41 = MakeIntrusive<TIssue>("level41"); - - /* - * * 0 - * | - * * 1 - * | - * * 2 - * | - * * 3 -- - * | | - * * 40 * 41 - */ - - level0->AddSubIssue(level1); - level1->AddSubIssue(level2); - level2->AddSubIssue(level3); - level3->AddSubIssue(level40); - level3->AddSubIssue(level41); - - { - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); - const auto expected = -R"___(<main>: Error: level0 +Y_UNIT_TEST(TruncLevels2) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level3 = MakeIntrusive<TIssue>("level3"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 + * | + * * 3 -- + * | | + * * 40 * 41 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level3); + level3->AddSubIssue(level40); + level3->AddSubIssue(level41); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = + R"___(<main>: Error: level0 <main>: Error: level1 <main>: Error: (skipped levels) <main>: Error: level3 <main>: Error: level40 <main>: Error: level41 )___"; - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + UNIT_ASSERT_STRINGS_EQUAL(res, expected); } +} - Y_UNIT_TEST(TruncLevels3) { - auto level0 = MakeIntrusive<TIssue>("level0"); - auto level1 = MakeIntrusive<TIssue>("level1"); - auto level2 = MakeIntrusive<TIssue>("level2"); - auto level3 = MakeIntrusive<TIssue>("level3"); - auto level40 = MakeIntrusive<TIssue>("level40"); - auto level41 = MakeIntrusive<TIssue>("level41"); - auto level50 = MakeIntrusive<TIssue>("level50"); - - /* - * * 0 - * | - * * 1 - * | - * * 2 - * | - * * 3 -- - * | | - * * 40 | - * | | - * * 50 * 41 - */ - - level0->AddSubIssue(level1); - level1->AddSubIssue(level2); - level2->AddSubIssue(level3); - level3->AddSubIssue(level40); - level3->AddSubIssue(level41); - level40->AddSubIssue(level50); - - { - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString(); - const auto expected = -R"___(<main>: Error: level0 +Y_UNIT_TEST(TruncLevels3) { + auto level0 = MakeIntrusive<TIssue>("level0"); + auto level1 = MakeIntrusive<TIssue>("level1"); + auto level2 = MakeIntrusive<TIssue>("level2"); + auto level3 = MakeIntrusive<TIssue>("level3"); + auto level40 = MakeIntrusive<TIssue>("level40"); + auto level41 = MakeIntrusive<TIssue>("level41"); + auto level50 = MakeIntrusive<TIssue>("level50"); + + /* + * * 0 + * | + * * 1 + * | + * * 2 + * | + * * 3 -- + * | | + * * 40 | + * | | + * * 50 * 41 + */ + + level0->AddSubIssue(level1); + level1->AddSubIssue(level2); + level2->AddSubIssue(level3); + level3->AddSubIssue(level40); + level3->AddSubIssue(level41); + level40->AddSubIssue(level50); + + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString(); + const auto expected = + R"___(<main>: Error: level0 <main>: Error: level1 <main>: Error: level2 <main>: Error: (skipped levels) <main>: Error: level41 <main>: Error: level50 )___"; - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } - { - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); - const auto expected = -R"___(<main>: Error: level0 + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString(); + const auto expected = + R"___(<main>: Error: level0 <main>: Error: level1 <main>: Error: (skipped levels) <main>: Error: level3 @@ -162,13 +162,13 @@ R"___(<main>: Error: level0 <main>: Error: level40 <main>: Error: level50 )___"; - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + UNIT_ASSERT_STRINGS_EQUAL(res, expected); + } - { - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString(); - const auto expected = -R"___(<main>: Error: level0 + { + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString(); + const auto expected = + R"___(<main>: Error: level0 <main>: Error: (skipped levels) <main>: Error: level2 <main>: Error: level3 @@ -176,25 +176,24 @@ R"___(<main>: Error: level0 <main>: Error: level50 <main>: Error: level41 )___"; - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + UNIT_ASSERT_STRINGS_EQUAL(res, expected); } +} - Y_UNIT_TEST(KeepSeverity) { - const auto templ = -R"___(<main>: {severity}: level0, code: 1 +Y_UNIT_TEST(KeepSeverity) { + const auto templ = + R"___(<main>: {severity}: level0, code: 1 <main>: {severity}: level1, code: 1 )___"; - for (auto severity: {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) { - - auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity)); - auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity)); + for (auto severity : {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) { + auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity)); + auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity)); - level0->AddSubIssue(level1); + level0->AddSubIssue(level1); - const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString(); - const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity)); - UNIT_ASSERT_STRINGS_EQUAL(res, expected); - } + const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString(); + const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity)); + UNIT_ASSERT_STRINGS_EQUAL(res, expected); } } +} // Y_UNIT_TEST_SUITE(TIssueUtilsTest) diff --git a/yql/essentials/public/issue/yql_warning.cpp b/yql/essentials/public/issue/yql_warning.cpp index 80d4686d1e5..82e350b4939 100644 --- a/yql/essentials/public/issue/yql_warning.cpp +++ b/yql/essentials/public/issue/yql_warning.cpp @@ -3,7 +3,6 @@ #include <util/string/cast.h> #include <util/string/join.h> - namespace NYql { TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, const TString& action, @@ -33,7 +32,8 @@ TWarningRule::EParseResult TWarningRule::ParseFrom(const TString& codePattern, c TWarningPolicy::TWarningPolicy(bool isReplay) : IsReplay_(isReplay) -{} +{ +} void TWarningPolicy::AddRule(const TWarningRule& rule) { @@ -65,8 +65,7 @@ void TWarningPolicy::AddRule(const TWarningRule& rule) } } -EWarningAction TWarningPolicy::GetAction(TIssueCode code) const -{ +EWarningAction TWarningPolicy::GetAction(TIssueCode code) const { auto it = Overrides_.find(code); return (it == Overrides_.end()) ? BaseAction_ : it->second; } @@ -78,4 +77,4 @@ void TWarningPolicy::Clear() Rules_.clear(); } -} +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_warning.h b/yql/essentials/public/issue/yql_warning.h index e21972d1d17..16cf21dc82c 100644 --- a/yql/essentials/public/issue/yql_warning.h +++ b/yql/essentials/public/issue/yql_warning.h @@ -6,7 +6,7 @@ #include <util/generic/string.h> #if defined(_win_) -#undef ERROR + #undef ERROR #endif namespace NYql { @@ -19,12 +19,19 @@ enum class EWarningAction { class TWarningRule { public: - const TString& GetPattern() const { return IssueCodePattern_; } - EWarningAction GetAction() const { return Action_; } - - enum class EParseResult { PARSE_OK, PARSE_PATTERN_FAIL, PARSE_ACTION_FAIL }; + const TString& GetPattern() const { + return IssueCodePattern_; + } + EWarningAction GetAction() const { + return Action_; + } + + enum class EParseResult { PARSE_OK, + PARSE_PATTERN_FAIL, + PARSE_ACTION_FAIL }; static EParseResult ParseFrom(const TString& codePattern, const TString& action, TWarningRule& result, TString& errorMessage); + private: TString IssueCodePattern_; EWarningAction Action_ = EWarningAction::DEFAULT; @@ -40,15 +47,17 @@ public: EWarningAction GetAction(TIssueCode code) const; - const TWarningRules& GetRules() const { return Rules_; } + const TWarningRules& GetRules() const { + return Rules_; + } void Clear(); private: - const bool IsReplay_; + const bool IsReplay_; TWarningRules Rules_; EWarningAction BaseAction_ = EWarningAction::DEFAULT; THashMap<TIssueCode, EWarningAction> Overrides_; }; -} +} // namespace NYql diff --git a/yql/essentials/public/issue/yql_warning_ut.cpp b/yql/essentials/public/issue/yql_warning_ut.cpp index 5edf7ed3631..c224e9e4a22 100644 --- a/yql/essentials/public/issue/yql_warning_ut.cpp +++ b/yql/essentials/public/issue/yql_warning_ut.cpp @@ -6,31 +6,30 @@ using namespace NYql; Y_UNIT_TEST_SUITE(TWarningRuleTest) { - Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) { - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); - } - - Y_UNIT_TEST(InvalidActionIsDetected) { - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); - } - - Y_UNIT_TEST(InvalidPatternIsDetected) { - TWarningRule rule; - TString errorMessage; - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); - } +Y_UNIT_TEST(AllValidActionsAndPatternsAreParsed) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "disable", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*", "error", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("0", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_OK); } +Y_UNIT_TEST(InvalidActionIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "wrong", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("1234", "", rule, errorMessage), TWarningRule::EParseResult::PARSE_ACTION_FAIL); +} + +Y_UNIT_TEST(InvalidPatternIsDetected) { + TWarningRule rule; + TString errorMessage; + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("-1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("*1", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); + UNIT_ASSERT_VALUES_EQUAL(TWarningRule::ParseFrom("default", "default", rule, errorMessage), TWarningRule::EParseResult::PARSE_PATTERN_FAIL); +} +} // Y_UNIT_TEST_SUITE(TWarningRuleTest) void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& action) { @@ -40,55 +39,52 @@ void AddRule(TWarningPolicy& policy, const TString& codePattern, const TString& policy.AddRule(rule); } - Y_UNIT_TEST_SUITE(TWarningPolicyTest) { - Y_UNIT_TEST(BasicIntegerRules) { - TWarningPolicy policy; - AddRule(policy, "123", "error"); - AddRule(policy, "456", "error"); - AddRule(policy, "456", "disable"); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT); - } +Y_UNIT_TEST(BasicIntegerRules) { + TWarningPolicy policy; + AddRule(policy, "123", "error"); + AddRule(policy, "456", "error"); + AddRule(policy, "456", "disable"); + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(111), EWarningAction::DEFAULT); +} - Y_UNIT_TEST(AsteriskRules) { - TWarningPolicy policy; - AddRule(policy, "*", "error"); - AddRule(policy, "456", "disable"); - - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); +Y_UNIT_TEST(AsteriskRules) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "disable"); - AddRule(policy, "*", "default"); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DISABLE); - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); + AddRule(policy, "*", "default"); - } + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); +} - Y_UNIT_TEST(ReactionOnPull) { - TWarningPolicy policy; - AddRule(policy, "*", "error"); - AddRule(policy, "456", "default"); - AddRule(policy, "999", "disable"); +Y_UNIT_TEST(ReactionOnPull) { + TWarningPolicy policy; + AddRule(policy, "*", "error"); + AddRule(policy, "456", "default"); + AddRule(policy, "999", "disable"); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DISABLE); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::ERROR); - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); - policy.Clear(); - UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0); + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 3); + policy.Clear(); + UNIT_ASSERT_VALUES_EQUAL(policy.GetRules().size(), 0); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); - UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); - } + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(999), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(456), EWarningAction::DEFAULT); + UNIT_ASSERT_VALUES_EQUAL(policy.GetAction(123), EWarningAction::DEFAULT); } +} // Y_UNIT_TEST_SUITE(TWarningPolicyTest) diff --git a/yql/essentials/public/langver/ut/ya.make b/yql/essentials/public/langver/ut/ya.make index ab179a9c34a..092ac834a94 100644 --- a/yql/essentials/public/langver/ut/ya.make +++ b/yql/essentials/public/langver/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(yql/essentials/public/langver) +ENABLE(YQL_STYLE_CPP) + SRCS( yql_langver_ut.cpp ) diff --git a/yql/essentials/public/langver/ut/yql_langver_ut.cpp b/yql/essentials/public/langver/ut/yql_langver_ut.cpp index 67f7586e643..82151bca563 100644 --- a/yql/essentials/public/langver/ut/yql_langver_ut.cpp +++ b/yql/essentials/public/langver/ut/yql_langver_ut.cpp @@ -5,126 +5,126 @@ namespace NYql { Y_UNIT_TEST_SUITE(TLangVerTests) { - Y_UNIT_TEST(IsValidMin) { - UNIT_ASSERT(IsValidLangVersion(MinLangVersion)); - } - - Y_UNIT_TEST(Parse) { - TLangVersion v; - UNIT_ASSERT(!ParseLangVersion("",v)); - UNIT_ASSERT(!ParseLangVersion("2025.01X",v)); - UNIT_ASSERT(!ParseLangVersion("2025-01",v)); - UNIT_ASSERT(!ParseLangVersion("99999.99",v)); - UNIT_ASSERT(ParseLangVersion("2025.01",v)); - UNIT_ASSERT_VALUES_EQUAL(v, MakeLangVersion(2025,1)); - UNIT_ASSERT(ParseLangVersion("9999.99",v)); - UNIT_ASSERT_VALUES_EQUAL(v, MakeLangVersion(9999,99)); - } - - Y_UNIT_TEST(Format) { - TLangVersionBuffer b; - TStringBuf s; - UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(99999, 1), b, s)); - UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(999, 1), b, s)); - UNIT_ASSERT(FormatLangVersion(MakeLangVersion(2025, 1), b, s)); - UNIT_ASSERT_VALUES_EQUAL(s, "2025.01"); - UNIT_ASSERT_VALUES_EQUAL(b[s.Size()], 0); - UNIT_ASSERT(FormatLangVersion(MakeLangVersion(2025, 12), b, s)); - UNIT_ASSERT_VALUES_EQUAL(s, "2025.12"); - UNIT_ASSERT_VALUES_EQUAL(b[s.Size()], 0); - } - - Y_UNIT_TEST(FormatString) { - UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(99999, 1))); - UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(999, 1))); - UNIT_ASSERT_VALUES_EQUAL(FormatLangVersion(MakeLangVersion(2025, 1)), "2025.01"); - UNIT_ASSERT_VALUES_EQUAL(FormatLangVersion(MakeLangVersion(2025, 12)), "2025.12"); - } - - Y_UNIT_TEST(Deprecated) { - UNIT_ASSERT(IsDeprecatedLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2027,1))); - UNIT_ASSERT(!IsDeprecatedLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2025,1))); - UNIT_ASSERT(!IsDeprecatedLangVersion(MakeLangVersion(2025,4),MakeLangVersion(2028,1))); - } - - Y_UNIT_TEST(Unsupported) { - UNIT_ASSERT(!IsUnsupportedLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2025,1))); - UNIT_ASSERT(!IsUnsupportedLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2027,1))); - UNIT_ASSERT(IsUnsupportedLangVersion(MakeLangVersion(2025,4),MakeLangVersion(2028,1))); - UNIT_ASSERT(IsUnsupportedLangVersion(MakeLangVersion(2025,5),MakeLangVersion(2029,1))); - } - - Y_UNIT_TEST(Available) { - UNIT_ASSERT(IsAvailableLangVersion(MakeLangVersion(2025,2),MakeLangVersion(2025,2))); - UNIT_ASSERT(!IsAvailableLangVersion(MakeLangVersion(2025,3),MakeLangVersion(2025,2))); - } - - Y_UNIT_TEST(MaxReleasedLangVersionIsValid) { - UNIT_ASSERT(IsValidLangVersion(GetMaxReleasedLangVersion())); - } - - Y_UNIT_TEST(MaxLangVersionIsValid) { - UNIT_ASSERT(IsValidLangVersion(GetMaxLangVersion())); - } - - Y_UNIT_TEST(MaxVersionIsAboveThanReleased) { - UNIT_ASSERT(GetMaxLangVersion() > GetMaxReleasedLangVersion()); - } - - Y_UNIT_TEST(BackwardCompatibleFeatureAvailable_All) { - UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, MinLangVersion, - EBackportCompatibleFeaturesMode::All)); - UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxReleasedLangVersion(), - EBackportCompatibleFeaturesMode::All)); - UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxLangVersion(), - EBackportCompatibleFeaturesMode::All)); - } - - Y_UNIT_TEST(BackwardCompatibleFeatureAvailable_Released) { - UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, MinLangVersion, - EBackportCompatibleFeaturesMode::Released)); - UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxReleasedLangVersion(), - EBackportCompatibleFeaturesMode::Released)); - UNIT_ASSERT(!IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxLangVersion(), - EBackportCompatibleFeaturesMode::Released)); - } - - Y_UNIT_TEST(BackwardCompatibleFeatureAvailable_None) { - UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, MinLangVersion, - EBackportCompatibleFeaturesMode::None)); - UNIT_ASSERT(!IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxReleasedLangVersion(), - EBackportCompatibleFeaturesMode::None)); - UNIT_ASSERT(!IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxLangVersion(), - EBackportCompatibleFeaturesMode::None)); - } - - Y_UNIT_TEST(EnumerateAllValid) { - const auto max = GetMaxLangVersion(); - const auto maxReleased = GetMaxReleasedLangVersion(); - bool hasMin = false; - bool hasMax = false; - bool hasMaxReleased = false; - EnumerateLangVersions([&](TLangVersion ver) { - UNIT_ASSERT(IsValidLangVersion(ver)); - UNIT_ASSERT(ver >= MinLangVersion); - UNIT_ASSERT(ver <= max); - if (ver == MinLangVersion) { - hasMin = true; - } - - if (ver == max) { - hasMax = true; - } - - if (ver == maxReleased) { - hasMaxReleased = true; - } - }); - - UNIT_ASSERT(hasMin); - UNIT_ASSERT(hasMax); - UNIT_ASSERT(hasMaxReleased); - } +Y_UNIT_TEST(IsValidMin) { + UNIT_ASSERT(IsValidLangVersion(MinLangVersion)); } +Y_UNIT_TEST(Parse) { + TLangVersion v; + UNIT_ASSERT(!ParseLangVersion("", v)); + UNIT_ASSERT(!ParseLangVersion("2025.01X", v)); + UNIT_ASSERT(!ParseLangVersion("2025-01", v)); + UNIT_ASSERT(!ParseLangVersion("99999.99", v)); + UNIT_ASSERT(ParseLangVersion("2025.01", v)); + UNIT_ASSERT_VALUES_EQUAL(v, MakeLangVersion(2025, 1)); + UNIT_ASSERT(ParseLangVersion("9999.99", v)); + UNIT_ASSERT_VALUES_EQUAL(v, MakeLangVersion(9999, 99)); } + +Y_UNIT_TEST(Format) { + TLangVersionBuffer b; + TStringBuf s; + UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(99999, 1), b, s)); + UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(999, 1), b, s)); + UNIT_ASSERT(FormatLangVersion(MakeLangVersion(2025, 1), b, s)); + UNIT_ASSERT_VALUES_EQUAL(s, "2025.01"); + UNIT_ASSERT_VALUES_EQUAL(b[s.Size()], 0); + UNIT_ASSERT(FormatLangVersion(MakeLangVersion(2025, 12), b, s)); + UNIT_ASSERT_VALUES_EQUAL(s, "2025.12"); + UNIT_ASSERT_VALUES_EQUAL(b[s.Size()], 0); +} + +Y_UNIT_TEST(FormatString) { + UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(99999, 1))); + UNIT_ASSERT(!FormatLangVersion(MakeLangVersion(999, 1))); + UNIT_ASSERT_VALUES_EQUAL(FormatLangVersion(MakeLangVersion(2025, 1)), "2025.01"); + UNIT_ASSERT_VALUES_EQUAL(FormatLangVersion(MakeLangVersion(2025, 12)), "2025.12"); +} + +Y_UNIT_TEST(Deprecated) { + UNIT_ASSERT(IsDeprecatedLangVersion(MakeLangVersion(2025, 2), MakeLangVersion(2027, 1))); + UNIT_ASSERT(!IsDeprecatedLangVersion(MakeLangVersion(2025, 3), MakeLangVersion(2025, 1))); + UNIT_ASSERT(!IsDeprecatedLangVersion(MakeLangVersion(2025, 4), MakeLangVersion(2028, 1))); +} + +Y_UNIT_TEST(Unsupported) { + UNIT_ASSERT(!IsUnsupportedLangVersion(MakeLangVersion(2025, 2), MakeLangVersion(2025, 1))); + UNIT_ASSERT(!IsUnsupportedLangVersion(MakeLangVersion(2025, 3), MakeLangVersion(2027, 1))); + UNIT_ASSERT(IsUnsupportedLangVersion(MakeLangVersion(2025, 4), MakeLangVersion(2028, 1))); + UNIT_ASSERT(IsUnsupportedLangVersion(MakeLangVersion(2025, 5), MakeLangVersion(2029, 1))); +} + +Y_UNIT_TEST(Available) { + UNIT_ASSERT(IsAvailableLangVersion(MakeLangVersion(2025, 2), MakeLangVersion(2025, 2))); + UNIT_ASSERT(!IsAvailableLangVersion(MakeLangVersion(2025, 3), MakeLangVersion(2025, 2))); +} + +Y_UNIT_TEST(MaxReleasedLangVersionIsValid) { + UNIT_ASSERT(IsValidLangVersion(GetMaxReleasedLangVersion())); +} + +Y_UNIT_TEST(MaxLangVersionIsValid) { + UNIT_ASSERT(IsValidLangVersion(GetMaxLangVersion())); +} + +Y_UNIT_TEST(MaxVersionIsAboveThanReleased) { + UNIT_ASSERT(GetMaxLangVersion() > GetMaxReleasedLangVersion()); +} + +Y_UNIT_TEST(BackwardCompatibleFeatureAvailable_All) { + UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, MinLangVersion, + EBackportCompatibleFeaturesMode::All)); + UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxReleasedLangVersion(), + EBackportCompatibleFeaturesMode::All)); + UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxLangVersion(), + EBackportCompatibleFeaturesMode::All)); +} + +Y_UNIT_TEST(BackwardCompatibleFeatureAvailable_Released) { + UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, MinLangVersion, + EBackportCompatibleFeaturesMode::Released)); + UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxReleasedLangVersion(), + EBackportCompatibleFeaturesMode::Released)); + UNIT_ASSERT(!IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxLangVersion(), + EBackportCompatibleFeaturesMode::Released)); +} + +Y_UNIT_TEST(BackwardCompatibleFeatureAvailable_None) { + UNIT_ASSERT(IsBackwardCompatibleFeatureAvailable(MinLangVersion, MinLangVersion, + EBackportCompatibleFeaturesMode::None)); + UNIT_ASSERT(!IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxReleasedLangVersion(), + EBackportCompatibleFeaturesMode::None)); + UNIT_ASSERT(!IsBackwardCompatibleFeatureAvailable(MinLangVersion, GetMaxLangVersion(), + EBackportCompatibleFeaturesMode::None)); +} + +Y_UNIT_TEST(EnumerateAllValid) { + const auto max = GetMaxLangVersion(); + const auto maxReleased = GetMaxReleasedLangVersion(); + bool hasMin = false; + bool hasMax = false; + bool hasMaxReleased = false; + EnumerateLangVersions([&](TLangVersion ver) { + UNIT_ASSERT(IsValidLangVersion(ver)); + UNIT_ASSERT(ver >= MinLangVersion); + UNIT_ASSERT(ver <= max); + if (ver == MinLangVersion) { + hasMin = true; + } + + if (ver == max) { + hasMax = true; + } + + if (ver == maxReleased) { + hasMaxReleased = true; + } + }); + + UNIT_ASSERT(hasMin); + UNIT_ASSERT(hasMax); + UNIT_ASSERT(hasMaxReleased); +} +} // Y_UNIT_TEST_SUITE(TLangVerTests) + +} // namespace NYql diff --git a/yql/essentials/public/langver/ya.make b/yql/essentials/public/langver/ya.make index b00ca55b9c0..77a7b1116c1 100644 --- a/yql/essentials/public/langver/ya.make +++ b/yql/essentials/public/langver/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( yql_langver.cpp ) diff --git a/yql/essentials/public/langver/yql_langver.cpp b/yql/essentials/public/langver/yql_langver.cpp index 5f91735eaa9..6360ecf3c37 100644 --- a/yql/essentials/public/langver/yql_langver.cpp +++ b/yql/essentials/public/langver/yql_langver.cpp @@ -11,11 +11,11 @@ namespace { constexpr TLangVersion MaxReleasedLangVersion = MakeLangVersion(2025, 3); -const std::pair<ui32,ui32> Versions[] = { +const std::pair<ui32, ui32> Versions[] = { #include "yql_langver_list.inc" }; -} +} // namespace bool IsValidLangVersion(TLangVersion ver) { for (size_t i = 0; i < Y_ARRAY_SIZE(Versions); ++i) { @@ -94,14 +94,14 @@ TLangVersion GetMaxLangVersion() { } bool IsBackwardCompatibleFeatureAvailable(TLangVersion currentVer, TLangVersion featureVer, - EBackportCompatibleFeaturesMode mode) { + EBackportCompatibleFeaturesMode mode) { switch (mode) { - case EBackportCompatibleFeaturesMode::All: - return true; - case EBackportCompatibleFeaturesMode::Released: - return IsAvailableLangVersion(featureVer, GetMaxReleasedLangVersion()); - case EBackportCompatibleFeaturesMode::None: - return IsAvailableLangVersion(featureVer, currentVer); + case EBackportCompatibleFeaturesMode::All: + return true; + case EBackportCompatibleFeaturesMode::Released: + return IsAvailableLangVersion(featureVer, GetMaxReleasedLangVersion()); + case EBackportCompatibleFeaturesMode::None: + return IsAvailableLangVersion(featureVer, currentVer); } } @@ -111,4 +111,4 @@ void EnumerateLangVersions(const std::function<void(TLangVersion)>& callback) { } } -} +} // namespace NYql diff --git a/yql/essentials/public/langver/yql_langver.h b/yql/essentials/public/langver/yql_langver.h index c646ce9ac08..2799746e18f 100644 --- a/yql/essentials/public/langver/yql_langver.h +++ b/yql/essentials/public/langver/yql_langver.h @@ -70,6 +70,6 @@ enum class EBackportCompatibleFeaturesMode { }; bool IsBackwardCompatibleFeatureAvailable(TLangVersion currentVer, TLangVersion featureVer, - EBackportCompatibleFeaturesMode mode); + EBackportCompatibleFeaturesMode mode); -} +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/compile_mkql.cpp b/yql/essentials/public/purecalc/common/compile_mkql.cpp index 7bb95e20754..2a7500927f9 100644 --- a/yql/essentials/public/purecalc/common/compile_mkql.cpp +++ b/yql/essentials/public/purecalc/common/compile_mkql.cpp @@ -90,10 +90,10 @@ NCommon::IMkqlCallableCompiler::TCompiler MakeFolderPathCallableCompiler(const T }; } -} +} // namespace NKikimr::NMiniKQL::TRuntimeNode CompileMkql(const TExprNode::TPtr& exprRoot, TExprContext& exprCtx, - const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, const NKikimr::NMiniKQL::TTypeEnvironment& env, const TUserDataTable& userData, NCommon::TMemoizedTypesMap* typeMemoization) + const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, const NKikimr::NMiniKQL::TTypeEnvironment& env, const TUserDataTable& userData, NCommon::TMemoizedTypesMap* typeMemoization) { NCommon::TMkqlCommonCallableCompiler compiler; @@ -106,11 +106,11 @@ NKikimr::NMiniKQL::TRuntimeNode CompileMkql(const TExprNode::TPtr& exprRoot, TEx // Prepare build context NKikimr::NMiniKQL::TProgramBuilder pgmBuilder(env, funcRegistry); - NCommon::TMkqlBuildContext buildCtx(compiler, pgmBuilder, exprCtx, /*lambdaId*/0, /*args*/{}, typeMemoization); + NCommon::TMkqlBuildContext buildCtx(compiler, pgmBuilder, exprCtx, /*lambdaId*/ 0, /*args*/ {}, typeMemoization); // Build the root MKQL node return NCommon::MkqlBuildExpr(*exprRoot, buildCtx); } -} // NYql::NPureCalc +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/compile_mkql.h b/yql/essentials/public/purecalc/common/compile_mkql.h index caba3baa2b9..09991e3d590 100644 --- a/yql/essentials/public/purecalc/common/compile_mkql.h +++ b/yql/essentials/public/purecalc/common/compile_mkql.h @@ -6,15 +6,14 @@ #include <yql/essentials/ast/yql_expr.h> #include <yql/essentials/core/yql_user_data.h> - namespace NYql::NPureCalc { - /** - * Compile expr to mkql byte-code - */ +/** + * Compile expr to mkql byte-code + */ - NKikimr::NMiniKQL::TRuntimeNode CompileMkql(const TExprNode::TPtr& exprRoot, TExprContext& exprCtx, - const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, const NKikimr::NMiniKQL::TTypeEnvironment& env, const TUserDataTable& userData, - NCommon::TMemoizedTypesMap* typeMemoization = nullptr); +NKikimr::NMiniKQL::TRuntimeNode CompileMkql(const TExprNode::TPtr& exprRoot, TExprContext& exprCtx, + const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, const NKikimr::NMiniKQL::TTypeEnvironment& env, const TUserDataTable& userData, + NCommon::TMemoizedTypesMap* typeMemoization = nullptr); -} +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/fwd.h b/yql/essentials/public/purecalc/common/fwd.h index 22df90a6b29..dc77d14b96b 100644 --- a/yql/essentials/public/purecalc/common/fwd.h +++ b/yql/essentials/public/purecalc/common/fwd.h @@ -4,53 +4,53 @@ #include <memory> namespace NYql::NPureCalc { - class TCompileError; +class TCompileError; - template <typename> - class IConsumer; +template <typename> +class IConsumer; - template <typename> - class IStream; +template <typename> +class IStream; - class IProgramFactory; +class IProgramFactory; - class IWorkerFactory; +class IWorkerFactory; - class IPullStreamWorkerFactory; +class IPullStreamWorkerFactory; - class IPullListWorkerFactory; +class IPullListWorkerFactory; - class IPushStreamWorkerFactory; +class IPushStreamWorkerFactory; - class IWorker; +class IWorker; - class IPullStreamWorker; +class IPullStreamWorker; - class IPullListWorker; +class IPullListWorker; - class IPushStreamWorker; +class IPushStreamWorker; - class TInputSpecBase; +class TInputSpecBase; - class TOutputSpecBase; +class TOutputSpecBase; - class IProgram; +class IProgram; - template <typename, typename, typename> - class TProgramCommon; +template <typename, typename, typename> +class TProgramCommon; - template <typename, typename> - class TPullStreamProgram; +template <typename, typename> +class TPullStreamProgram; - template <typename, typename> - class TPullListProgram; +template <typename, typename> +class TPullListProgram; - template <typename, typename> - class TPushStreamProgram; +template <typename, typename> +class TPushStreamProgram; - using IProgramFactoryPtr = TIntrusivePtr<IProgramFactory>; - using IWorkerFactoryPtr = std::shared_ptr<IWorkerFactory>; - using IPullStreamWorkerFactoryPtr = std::shared_ptr<IPullStreamWorkerFactory>; - using IPullListWorkerFactoryPtr = std::shared_ptr<IPullListWorkerFactory>; - using IPushStreamWorkerFactoryPtr = std::shared_ptr<IPushStreamWorkerFactory>; -} +using IProgramFactoryPtr = TIntrusivePtr<IProgramFactory>; +using IWorkerFactoryPtr = std::shared_ptr<IWorkerFactory>; +using IPullStreamWorkerFactoryPtr = std::shared_ptr<IPullStreamWorkerFactory>; +using IPullListWorkerFactoryPtr = std::shared_ptr<IPullListWorkerFactory>; +using IPushStreamWorkerFactoryPtr = std::shared_ptr<IPushStreamWorkerFactory>; +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/inspect_input.cpp b/yql/essentials/public/purecalc/common/inspect_input.cpp index 9ca56da5dec..7717aac2e04 100644 --- a/yql/essentials/public/purecalc/common/inspect_input.cpp +++ b/yql/essentials/public/purecalc/common/inspect_input.cpp @@ -3,31 +3,31 @@ #include <yql/essentials/core/yql_expr_type_annotation.h> namespace NYql::NPureCalc { - bool TryFetchInputIndexFromSelf(const TExprNode& node, TExprContext& ctx, ui32 inputsCount, ui32& result) { - TIssueScopeGuard issueSope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(node.Pos()), TStringBuilder() << "At function: " << node.Content()); - }); +bool TryFetchInputIndexFromSelf(const TExprNode& node, TExprContext& ctx, ui32 inputsCount, ui32& result) { + TIssueScopeGuard issueSope(ctx.IssueManager, [&]() { + return MakeIntrusive<TIssue>(ctx.GetPosition(node.Pos()), TStringBuilder() << "At function: " << node.Content()); + }); - if (!EnsureArgsCount(node, 1, ctx)) { - return false; - } - - if (!EnsureAtom(*node.Child(0), ctx)) { - return false; - } + if (!EnsureArgsCount(node, 1, ctx)) { + return false; + } - if (!TryFromString(node.Child(0)->Content(), result)) { - auto message = TStringBuilder() << "Index " << TString{node.Child(0)->Content()}.Quote() << " isn't UI32"; - ctx.AddError(TIssue(ctx.GetPosition(node.Child(0)->Pos()), std::move(message))); - return false; - } + if (!EnsureAtom(*node.Child(0), ctx)) { + return false; + } - if (result >= inputsCount) { - auto message = TStringBuilder() << "Invalid input index: " << result << " is out of range [0;" << inputsCount << ")"; - ctx.AddError(TIssue(ctx.GetPosition(node.Child(0)->Pos()), std::move(message))); - return false; - } + if (!TryFromString(node.Child(0)->Content(), result)) { + auto message = TStringBuilder() << "Index " << TString{node.Child(0)->Content()}.Quote() << " isn't UI32"; + ctx.AddError(TIssue(ctx.GetPosition(node.Child(0)->Pos()), std::move(message))); + return false; + } - return true; + if (result >= inputsCount) { + auto message = TStringBuilder() << "Invalid input index: " << result << " is out of range [0;" << inputsCount << ")"; + ctx.AddError(TIssue(ctx.GetPosition(node.Child(0)->Pos()), std::move(message))); + return false; } + + return true; } +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/inspect_input.h b/yql/essentials/public/purecalc/common/inspect_input.h index 558144865da..46a6a0b4fbe 100644 --- a/yql/essentials/public/purecalc/common/inspect_input.h +++ b/yql/essentials/public/purecalc/common/inspect_input.h @@ -3,5 +3,5 @@ #include <yql/essentials/ast/yql_expr.h> namespace NYql::NPureCalc { - bool TryFetchInputIndexFromSelf(const TExprNode&, TExprContext&, ui32, ui32&); -} +bool TryFetchInputIndexFromSelf(const TExprNode&, TExprContext&, ui32, ui32&); +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/interface.h b/yql/essentials/public/purecalc/common/interface.h index 23aa93bdf23..36741ed24d3 100644 --- a/yql/essentials/public/purecalc/common/interface.h +++ b/yql/essentials/public/purecalc/common/interface.h @@ -24,970 +24,970 @@ class ITimeProvider; namespace NKikimr { - namespace NMiniKQL { - class TScopedAlloc; - class IComputationGraph; - class IFunctionRegistry; - class TTypeEnvironment; - class TType; - class TStructType; +namespace NMiniKQL { +class TScopedAlloc; +class IComputationGraph; +class IFunctionRegistry; +class TTypeEnvironment; +class TType; +class TStructType; +} // namespace NMiniKQL +} // namespace NKikimr + +namespace NYql { +namespace NPureCalc { +/** + * SQL or s-expression translation error. + */ +class TCompileError: public yexception { +private: + TString Yql_; + TString Issues_; + +public: + // TODO: maybe accept an actual list of issues here? + // See https://a.yandex-team.ru/arc/review/439403/details#comment-778237 + TCompileError(TString yql, TString issues) + : Yql_(std::move(yql)) + , Issues_(std::move(issues)) + { + } + +public: + /** + * Get the sql query which caused the error (if there is one available). + */ + const TString& GetYql() const { + return Yql_; + } + + /** + * Get detailed description for all errors and warnings that happened during sql translation. + */ + const TString& GetIssues() const { + return Issues_; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A generic input stream of objects. + */ +template <typename T> +class IStream { +public: + virtual ~IStream() = default; + +public: + /** + * Pops and returns a next value in the stream. If the stream is finished, should return some sentinel object. + * + * Depending on return type, this function may not transfer object ownership to a user. + * Thus, the stream may manage the returned object * itself. + * That is, the returned object's lifetime may be bound to the input stream lifetime; it may be destroyed + * upon calling Fetch() or upon destroying the stream, whichever happens first. + */ + virtual T Fetch() = 0; +}; + +/** + * Create a new stream which applies the given functor to the elements of the original stream. + */ +template <typename TOld, typename TNew, typename TFunctor> +inline THolder<IStream<TNew>> MapStream(THolder<IStream<TOld>> stream, TFunctor functor) { + return THolder(new NPrivate::TMappingStream<TNew, TOld, TFunctor>(std::move(stream), std::move(functor))); +}; + +/** + * Convert stream of objects into a stream of potentially incompatible objects. + * + * This conversion applies static cast to the output of the original stream. Use with caution! + */ +/// @{ +template < + typename TNew, typename TOld, + std::enable_if_t<!std::is_same<TNew, TOld>::value>* = nullptr> +inline THolder<IStream<TNew>> ConvertStreamUnsafe(THolder<IStream<TOld>> stream) { + return MapStream<TOld, TNew>(std::move(stream), [](TOld x) -> TNew { return static_cast<TNew>(x); }); +} +template <typename T> +inline THolder<IStream<T>> ConvertStreamUnsafe(THolder<IStream<T>> stream) { + return stream; +} +/// @} + +/** + * Convert stream of objects into a stream of compatible objects. + * + * Note: each conversion adds one level of indirection so avoid them if possible. + */ +template <typename TNew, typename TOld, std::enable_if_t<std::is_convertible<TOld, TNew>::value>* = nullptr> +inline THolder<IStream<TNew>> ConvertStream(THolder<IStream<TOld>> stream) { + return ConvertStreamUnsafe<TNew, TOld>(std::move(stream)); } -namespace NYql { - namespace NPureCalc { - /** - * SQL or s-expression translation error. - */ - class TCompileError: public yexception { - private: - TString Yql_; - TString Issues_; - - public: - // TODO: maybe accept an actual list of issues here? - // See https://a.yandex-team.ru/arc/review/439403/details#comment-778237 - TCompileError(TString yql, TString issues) - : Yql_(std::move(yql)) - , Issues_(std::move(issues)) - { - } - - public: - /** - * Get the sql query which caused the error (if there is one available). - */ - const TString& GetYql() const { - return Yql_; - } - - /** - * Get detailed description for all errors and warnings that happened during sql translation. - */ - const TString& GetIssues() const { - return Issues_; - } - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * A generic input stream of objects. - */ - template <typename T> - class IStream { - public: - virtual ~IStream() = default; - - public: - /** - * Pops and returns a next value in the stream. If the stream is finished, should return some sentinel object. - * - * Depending on return type, this function may not transfer object ownership to a user. - * Thus, the stream may manage the returned object * itself. - * That is, the returned object's lifetime may be bound to the input stream lifetime; it may be destroyed - * upon calling Fetch() or upon destroying the stream, whichever happens first. - */ - virtual T Fetch() = 0; - }; - - /** - * Create a new stream which applies the given functor to the elements of the original stream. - */ - template <typename TOld, typename TNew, typename TFunctor> - inline THolder<IStream<TNew>> MapStream(THolder<IStream<TOld>> stream, TFunctor functor) { - return THolder(new NPrivate::TMappingStream<TNew, TOld, TFunctor>(std::move(stream), std::move(functor))); - }; - - /** - * Convert stream of objects into a stream of potentially incompatible objects. - * - * This conversion applies static cast to the output of the original stream. Use with caution! - */ - /// @{ - template < - typename TNew, typename TOld, - std::enable_if_t<!std::is_same<TNew, TOld>::value>* = nullptr> - inline THolder<IStream<TNew>> ConvertStreamUnsafe(THolder<IStream<TOld>> stream) { - return MapStream<TOld, TNew>(std::move(stream), [](TOld x) -> TNew { return static_cast<TNew>(x); }); - } - template <typename T> - inline THolder<IStream<T>> ConvertStreamUnsafe(THolder<IStream<T>> stream) { - return stream; - } - /// @} - - /** - * Convert stream of objects into a stream of compatible objects. - * - * Note: each conversion adds one level of indirection so avoid them if possible. - */ - template <typename TNew, typename TOld, std::enable_if_t<std::is_convertible<TOld, TNew>::value>* = nullptr> - inline THolder<IStream<TNew>> ConvertStream(THolder<IStream<TOld>> stream) { - return ConvertStreamUnsafe<TNew, TOld>(std::move(stream)); - } +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A generic push consumer. + */ +template <typename T> +class IConsumer { +public: + virtual ~IConsumer() = default; + +public: + /** + * Feed an object to consumer. + * + * Depending on argument type, the consumer may not take ownership of the passed object; + * in that case it is the caller responsibility to manage the object lifetime after passing it to this method. + * + * The passed object can be destroyed after the consumer returns from this function; the consumer should + * not store pointer to the passed object or the passed object itself without taking all necessary precautions + * to ensure that the pointer or the object stays valid after returning. + */ + virtual void OnObject(T) = 0; + + /** + * Close the consumer and run finalization logic. Calling OnObject after calling this function is an error. + */ + virtual void OnFinish() = 0; +}; + +/** + * Create a new consumer which applies the given functor to objects before . + */ +template <typename TOld, typename TNew, typename TFunctor> +inline THolder<IConsumer<TNew>> MapConsumer(THolder<IConsumer<TOld>> stream, TFunctor functor) { + return THolder(new NPrivate::TMappingConsumer<TNew, TOld, TFunctor>(std::move(stream), std::move(functor))); +}; + +/** + * Convert consumer of objects into a consumer of potentially incompatible objects. + * + * This conversion applies static cast to the input value. Use with caution. + */ +/// @{ +template < + typename TNew, typename TOld, + std::enable_if_t<!std::is_same<TNew, TOld>::value>* = nullptr> +inline THolder<IConsumer<TNew>> ConvertConsumerUnsafe(THolder<IConsumer<TOld>> consumer) { + return MapConsumer<TOld, TNew>(std::move(consumer), [](TNew x) -> TOld { return static_cast<TOld>(x); }); +} +template <typename T> +inline THolder<IConsumer<T>> ConvertConsumerUnsafe(THolder<IConsumer<T>> consumer) { + return consumer; +} +/// @} + +/** + * Convert consumer of objects into a consumer of compatible objects. + * + * Note: each conversion adds one level of indirection so avoid them if possible. + */ +template <typename TNew, typename TOld, std::enable_if_t<std::is_convertible<TNew, TOld>::value>* = nullptr> +inline THolder<IConsumer<TNew>> ConvertConsumer(THolder<IConsumer<TOld>> consumer) { + return ConvertConsumerUnsafe<TNew, TOld>(std::move(consumer)); +} - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * A generic push consumer. - */ - template <typename T> - class IConsumer { - public: - virtual ~IConsumer() = default; - - public: - /** - * Feed an object to consumer. - * - * Depending on argument type, the consumer may not take ownership of the passed object; - * in that case it is the caller responsibility to manage the object lifetime after passing it to this method. - * - * The passed object can be destroyed after the consumer returns from this function; the consumer should - * not store pointer to the passed object or the passed object itself without taking all necessary precautions - * to ensure that the pointer or the object stays valid after returning. - */ - virtual void OnObject(T) = 0; - - /** - * Close the consumer and run finalization logic. Calling OnObject after calling this function is an error. - */ - virtual void OnFinish() = 0; - }; - - /** - * Create a new consumer which applies the given functor to objects before . - */ - template <typename TOld, typename TNew, typename TFunctor> - inline THolder<IConsumer<TNew>> MapConsumer(THolder<IConsumer<TOld>> stream, TFunctor functor) { - return THolder(new NPrivate::TMappingConsumer<TNew, TOld, TFunctor>(std::move(stream), std::move(functor))); - }; - - - /** - * Convert consumer of objects into a consumer of potentially incompatible objects. - * - * This conversion applies static cast to the input value. Use with caution. - */ - /// @{ - template < - typename TNew, typename TOld, - std::enable_if_t<!std::is_same<TNew, TOld>::value>* = nullptr> - inline THolder<IConsumer<TNew>> ConvertConsumerUnsafe(THolder<IConsumer<TOld>> consumer) { - return MapConsumer<TOld, TNew>(std::move(consumer), [](TNew x) -> TOld { return static_cast<TOld>(x); }); - } - template <typename T> - inline THolder<IConsumer<T>> ConvertConsumerUnsafe(THolder<IConsumer<T>> consumer) { - return consumer; - } - /// @} - - /** - * Convert consumer of objects into a consumer of compatible objects. - * - * Note: each conversion adds one level of indirection so avoid them if possible. - */ - template <typename TNew, typename TOld, std::enable_if_t<std::is_convertible<TNew, TOld>::value>* = nullptr> - inline THolder<IConsumer<TNew>> ConvertConsumer(THolder<IConsumer<TOld>> consumer) { - return ConvertConsumerUnsafe<TNew, TOld>(std::move(consumer)); - } +/** + * Create a consumer which holds a non-owning pointer to the given consumer + * and passes all messages to the latter. + */ +template <typename T, typename C> +THolder<NPrivate::TNonOwningConsumer<T, C>> MakeNonOwningConsumer(C consumer) { + return MakeHolder<NPrivate::TNonOwningConsumer<T, C>>(consumer); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Logging options. + */ +struct TLoggingOptions final { +public: + /// Logging level for messages generated during compilation. + ELogPriority LogLevel; + + /// Where to write log messages. + IOutputStream* LogDestination; + +public: + TLoggingOptions(); + /** + * Set a new logging level. + * + * @return reference to self, to allow method chaining. + */ + TLoggingOptions& SetLogLevel(ELogPriority); + + /** + * Set a new logging destination. + * + * @return reference to self, to allow method chaining. + */ + TLoggingOptions& SetLogDestination(IOutputStream*); +}; + +/** + * General options for program factory. + */ +struct TProgramFactoryOptions final { +public: + /// Path to a directory with compiled UDFs. Leave empty to disable loading external UDFs. + TString UdfsDir; + + /// List of available external resources, e.g. files, UDFs, libraries. + TVector<NUserData::TUserData> UserData; + + /// LLVM settings. Assign "OFF" to disable LLVM, empty string for default settings. + TString LLVMSettings; + + /// Block engine settings. Assign "force" to unconditionally enable + /// it, "disable" for turn it off and "auto" to left the final + /// decision to the platform heuristics. + TString BlockEngineSettings; + + /// Output stream to dump the compiled and optimized expressions. + IOutputStream* ExprOutputStream; + + /// Provider for generic counters which can be used to export statistics from UDFs. + NKikimr::NUdf::ICountersProvider* CountersProvider; + + /// YT Type V3 flags for Skiff/Yson serialization. + ui64 NativeYtTypeFlags; + + /// Seed for deterministic time provider + TMaybe<ui64> DeterministicTimeProviderSeed; + + /// Use special system columns to support tables naming (supports non empty ``TablePath()``/``TableName()``) + bool UseSystemColumns; + + /// Reuse allocated workers + bool UseWorkerPool; + + /// Use Antlr4 parser (for migration) + bool UseAntlr4; + + /// Language version + TLangVersion LangVer; + +public: + TProgramFactoryOptions(); + +public: + /** + * Set language version for queries + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetLanguageVersion(TLangVersion langver); + + /** + * Set a new path to a directory with UDFs. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetUDFsDir(TStringBuf); + + /** + * Add a new library to the UserData list. + * + * @param disposition where the resource resides, e.g. on filesystem, in memory, etc. + * NB: URL disposition is not supported. + * @param name name of the resource. + * @param content depending on disposition, either path to the resource or its content. + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& AddLibrary(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content); + + /** + * Add a new file to the UserData list. + * + * @param disposition where the resource resides, e.g. on filesystem, in memory, etc. + * NB: URL disposition is not supported. + * @param name name of the resource. + * @param content depending on disposition, either path to the resource or its content. + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& AddFile(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content); + + /** + * Add a new UDF to the UserData list. + * + * @param disposition where the resource resides, e.g. on filesystem, in memory, etc. + * NB: URL disposition is not supported. + * @param name name of the resource. + * @param content depending on disposition, either path to the resource or its content. + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& AddUDF(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content); + + /** + * Set new LLVM settings. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetLLVMSettings(TStringBuf llvm_settings); + + /** + * Set new block engine settings. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetBlockEngineSettings(TStringBuf blockEngineSettings); + + /** + * Set the stream to dump the compiled and optimized expressions. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetExprOutputStream(IOutputStream* exprOutputStream); + + /** + * Set new counters provider. Passed pointer should stay alive for as long as the processor factory + * stays alive. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetCountersProvider(NKikimr::NUdf::ICountersProvider* countersProvider); + + /** + * Set new YT Type V3 mode. Deprecated method. Use SetNativeYtTypeFlags instead + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetUseNativeYtTypes(bool useNativeTypes); + + /** + * Set YT Type V3 flags. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetNativeYtTypeFlags(ui64 nativeTypeFlags); + + /** + * Set seed for deterministic time provider. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetDeterministicTimeProviderSeed(TMaybe<ui64> seed); + + /** + * Set new flag whether to allow using system columns or not. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetUseSystemColumns(bool useSystemColumns); + + /** + * Set new flag whether to allow reusing workers or not. + * + * @return reference to self, to allow method chaining. + */ + TProgramFactoryOptions& SetUseWorkerPool(bool useWorkerPool); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * What exactly are we parsing: SQL or an s-expression. + */ +enum class ETranslationMode { + SQL /* "SQL" */, + SExpr /* "s-expression" */, + Mkql /* "mkql" */, + PG /* PostgreSQL */ +}; + +/** + * A facility for compiling sql and s-expressions and making programs from them. + */ +class IProgramFactory: public TThrRefBase { +protected: + virtual IPullStreamWorkerFactoryPtr MakePullStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) = 0; + virtual IPullListWorkerFactoryPtr MakePullListWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) = 0; + virtual IPushStreamWorkerFactoryPtr MakePushStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) = 0; + +public: + /** + * Add new udf module. It's not specified whether adding new modules will affect existing programs + * (theoretical answer is 'no'). + */ + virtual void AddUdfModule(const TStringBuf&, NKikimr::NUdf::TUniquePtr<NKikimr::NUdf::IUdfModule>&&) = 0; + // TODO: support setting udf modules via factory options. + + /** + * Set new counters provider, override one that was specified via factory options. Note that existing + * programs will still reference the previous provider. + */ + virtual void SetCountersProvider(NKikimr::NUdf::ICountersProvider*) = 0; + // TODO: support setting providers via factory options. + + template <typename TInputSpec, typename TOutputSpec> + THolder<TPullStreamProgram<TInputSpec, TOutputSpec>> MakePullStreamProgram( + TInputSpec inputSpec, TOutputSpec outputSpec, TString query, ETranslationMode mode = ETranslationMode::SQL, ui16 syntaxVersion = 1) { + auto workerFactory = MakePullStreamWorkerFactory(inputSpec, outputSpec, std::move(query), mode, syntaxVersion); + return MakeHolder<TPullStreamProgram<TInputSpec, TOutputSpec>>(std::move(inputSpec), std::move(outputSpec), workerFactory); + } - /** - * Create a consumer which holds a non-owning pointer to the given consumer - * and passes all messages to the latter. - */ - template <typename T, typename C> - THolder<NPrivate::TNonOwningConsumer<T, C>> MakeNonOwningConsumer(C consumer) { - return MakeHolder<NPrivate::TNonOwningConsumer<T, C>>(consumer); + template <typename TInputSpec, typename TOutputSpec> + THolder<TPullListProgram<TInputSpec, TOutputSpec>> MakePullListProgram( + TInputSpec inputSpec, TOutputSpec outputSpec, TString query, ETranslationMode mode = ETranslationMode::SQL, ui16 syntaxVersion = 1) { + auto workerFactory = MakePullListWorkerFactory(inputSpec, outputSpec, std::move(query), mode, syntaxVersion); + return MakeHolder<TPullListProgram<TInputSpec, TOutputSpec>>(std::move(inputSpec), std::move(outputSpec), workerFactory); + } + + template <typename TInputSpec, typename TOutputSpec> + THolder<TPushStreamProgram<TInputSpec, TOutputSpec>> MakePushStreamProgram( + TInputSpec inputSpec, TOutputSpec outputSpec, TString query, ETranslationMode mode = ETranslationMode::SQL, ui16 syntaxVersion = 1) { + auto workerFactory = MakePushStreamWorkerFactory(inputSpec, outputSpec, std::move(query), mode, syntaxVersion); + return MakeHolder<TPushStreamProgram<TInputSpec, TOutputSpec>>(std::move(inputSpec), std::move(outputSpec), workerFactory); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * A facility for creating workers. Despite being a part of a public API, worker factory is not used directly. + */ +class IWorkerFactory: public std::enable_shared_from_this<IWorkerFactory> { +public: + virtual ~IWorkerFactory() = default; + /** + * Get input column names for specified input that are actually used in the query. + */ + virtual const THashSet<TString>& GetUsedColumns(ui32) const = 0; + /** + * Overload for single-input programs. + */ + virtual const THashSet<TString>& GetUsedColumns() const = 0; + + /** + * Make input type schema for specified input as deduced by program optimizer. This schema is equivalent + * to one provided by input spec up to the order of the fields in structures. + */ + virtual NYT::TNode MakeInputSchema(ui32) const = 0; + /** + * Overload for single-input programs. + */ + virtual NYT::TNode MakeInputSchema() const = 0; + + /** + * Make output type schema as deduced by program optimizer. If output spec provides its own schema, than + * this schema is equivalent to one provided by output spec up to the order of the fields in structures. + */ + /// @{ + /** + * Overload for single-table output programs (i.e. output type is struct). + */ + virtual NYT::TNode MakeOutputSchema() const = 0; + /** + * Overload for multi-table output programs (i.e. output type is variant over tuple). + */ + virtual NYT::TNode MakeOutputSchema(ui32) const = 0; + /** + * Overload for multi-table output programs (i.e. output type is variant over struct). + */ + virtual NYT::TNode MakeOutputSchema(TStringBuf) const = 0; + /// @} + + /** + * Make full output schema. For single-output programs returns struct type, for multi-output programs + * returns variant type. + * + * Warning: calling this function may result in extended memory usage for large number of output tables. + */ + virtual NYT::TNode MakeFullOutputSchema() const = 0; + + /** + * Get compilation issues + */ + virtual TIssues GetIssues() const = 0; + + /** + * Get precompiled mkql program + */ + virtual TString GetCompiledProgram() = 0; + + /** + * Return a worker to the factory for possible reuse + */ + virtual void ReturnWorker(IWorker* worker) = 0; +}; + +class TReleaseWorker { +public: + template <class T> + static inline void Destroy(T* t) noexcept { + t->Release(); + } +}; + +template <class T> +using TWorkerHolder = THolder<T, TReleaseWorker>; + +/** + * Factory for generating pull stream workers. + */ +class IPullStreamWorkerFactory: public IWorkerFactory { +public: + /** + * Create a new pull stream worker. + */ + virtual TWorkerHolder<IPullStreamWorker> MakeWorker() = 0; +}; + +/** + * Factory for generating pull list workers. + */ +class IPullListWorkerFactory: public IWorkerFactory { +public: + /** + * Create a new pull list worker. + */ + virtual TWorkerHolder<IPullListWorker> MakeWorker() = 0; +}; + +/** + * Factory for generating push stream workers. + */ +class IPushStreamWorkerFactory: public IWorkerFactory { +public: + /** + * Create a new push stream worker. + */ + virtual TWorkerHolder<IPushStreamWorker> MakeWorker() = 0; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Worker is a central part of any program instance. It contains current computation state + * (called computation graph) and objects required to work with it, including an allocator for unboxed values. + * + * Usually, users do not interact with workers directly. They use program instance entry points such as streams + * and consumers instead. The only case when one would have to to interact with workers is when implementing + * custom io-specification. + */ +class IWorker { +protected: + friend class TReleaseWorker; + /** + * Cleanup the worker and return to a worker factory for reuse + */ + virtual void Release() = 0; + +public: + virtual ~IWorker() = default; + +public: + /** + * Number of inputs for this program. + */ + virtual ui32 GetInputsCount() const = 0; + + /** + * MiniKQL input struct type of specified input for this program. Type is equivalent to the deduced input + * schema (see IWorker::MakeInputSchema()) + * + * If ``original`` is set to ``true``, returns type without virtual system columns. + */ + virtual const NKikimr::NMiniKQL::TStructType* GetInputType(ui32, bool original = false) const = 0; + /** + * Overload for single-input programs. + */ + virtual const NKikimr::NMiniKQL::TStructType* GetInputType(bool original = false) const = 0; + + /** + * MiniKQL input struct type of the specified input for this program. + * The returned type is the actual type of the specified input node. + */ + virtual const NKikimr::NMiniKQL::TStructType* GetRawInputType(ui32) const = 0; + /** + * Overload for single-input programs. + */ + virtual const NKikimr::NMiniKQL::TStructType* GetRawInputType() const = 0; + + /** + * MiniKQL output struct type for this program. The returned type is equivalent to the deduced output + * schema (see IWorker::MakeFullOutputSchema()). + */ + virtual const NKikimr::NMiniKQL::TType* GetOutputType() const = 0; + + /** + * MiniKQL output struct type for this program. The returned type is + * the actual type of the root node. + */ + virtual const NKikimr::NMiniKQL::TType* GetRawOutputType() const = 0; + + /** + * Make input type schema for specified input as deduced by program optimizer. This schema is equivalent + * to one provided by input spec up to the order of the fields in structures. + */ + virtual NYT::TNode MakeInputSchema(ui32) const = 0; + /** + * Overload for single-input programs. + */ + virtual NYT::TNode MakeInputSchema() const = 0; + + /** + * Make output type schema as deduced by program optimizer. If output spec provides its own schema, than + * this schema is equivalent to one provided by output spec up to the order of the fields in structures. + */ + /// @{ + /** + * Overload for single-table output programs (i.e. output type is struct). + */ + virtual NYT::TNode MakeOutputSchema() const = 0; + /** + * Overload for multi-table output programs (i.e. output type is variant over tuple). + */ + virtual NYT::TNode MakeOutputSchema(ui32) const = 0; + /** + * Overload for multi-table output programs (i.e. output type is variant over struct). + */ + virtual NYT::TNode MakeOutputSchema(TStringBuf) const = 0; + /// @} + + /** + * Generates full output schema. For single-output programs returns struct type, for multi-output programs + * returns variant type. + * + * Warning: calling this function may result in extended memory usage for large number of output tables. + */ + virtual NYT::TNode MakeFullOutputSchema() const = 0; + + /** + * Get scoped alloc used in this worker. + */ + virtual NKikimr::NMiniKQL::TScopedAlloc& GetScopedAlloc() = 0; + + /** + * Get computation graph. + */ + virtual NKikimr::NMiniKQL::IComputationGraph& GetGraph() = 0; + + /** + * Get function registry for this worker. + */ + virtual const NKikimr::NMiniKQL::IFunctionRegistry& GetFunctionRegistry() const = 0; + + /** + * Get type environment for this worker. + */ + virtual NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnvironment() = 0; + + /** + * Get llvm settings for this worker. + */ + virtual const TString& GetLLVMSettings() const = 0; + + /** + * Get YT Type V3 flags + */ + virtual ui64 GetNativeYtTypeFlags() const = 0; + + /** + * Get time provider + */ + virtual ITimeProvider* GetTimeProvider() const = 0; + + /** + * Release all input data from worker state + */ + virtual void Invalidate() = 0; +}; + +/** + * Worker which operates in pull stream mode. + */ +class IPullStreamWorker: public IWorker { +public: + /** + * Set input computation graph node for specified input. The passed unboxed value should be a stream of + * structs. It should be created via the allocator associated with this very worker. + * This function can only be called once for each input. + */ + virtual void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) = 0; + + /** + * Get the output computation graph node. The returned node will be a stream of structs or variants. + * This function cannot be called before setting an input value. + */ + virtual NKikimr::NUdf::TUnboxedValue& GetOutput() = 0; +}; + +/** + * Worker which operates in pull list mode. + */ +class IPullListWorker: public IWorker { +public: + /** + * Set input computation graph node for specified input. The passed unboxed value should be a list of + * structs. It should be created via the allocator associated with this very worker. + * This function can only be called once for each index. + */ + virtual void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) = 0; + + /** + * Get the output computation graph node. The returned node will be a list of structs or variants. + * This function cannot be called before setting an input value. + */ + virtual NKikimr::NUdf::TUnboxedValue& GetOutput() = 0; + + /** + * Get iterator over the output list. + */ + virtual NKikimr::NUdf::TUnboxedValue& GetOutputIterator() = 0; + + /** + * Reset iterator to the beginning of the output list. After calling this function, GetOutputIterator() + * will return a fresh iterator; all previously returned iterators will become invalid. + */ + virtual void ResetOutputIterator() = 0; +}; + +/** + * Worker which operates in push stream mode. + */ +class IPushStreamWorker: public IWorker { +public: + /** + * Set a consumer where the worker will relay its output. This function can only be called once. + */ + virtual void SetConsumer(THolder<IConsumer<const NKikimr::NUdf::TUnboxedValue*>>) = 0; + + /** + * Push new value to the graph, than feed all new output to the consumer. Values cannot be pushed before + * assigning a consumer. + */ + virtual void Push(NKikimr::NUdf::TUnboxedValue&&) = 0; + + /** + * Send finish event and clear the computation graph. No new values will be accepted. + */ + virtual void OnFinish() = 0; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Input specifications describe format for program input. They carry information about input data schema + * as well as the knowledge about how to convert input structures into unboxed values (data format which can be + * processed by the YQL runtime). + * + * Input spec defines the arguments of the program's Apply method. For example, a program + * with the protobuf input spec will accept a stream of protobuf messages while a program with the + * yson spec will accept an input stream (binary or text one). + * + * See documentation for input and output spec traits for hints on how to implement a custom specs. + */ +class TInputSpecBase { +protected: + mutable TVector<THashMap<TString, NYT::TNode>> AllVirtualColumns_; + +public: + virtual ~TInputSpecBase() = default; + +public: + /** + * Get input data schemas in YQL format (NB: not a YT format). Each item of the returned vector must + * describe a structure. + * + * Format of each item is approximately this one: + * + * @code + * [ + * 'StructType', + * [ + * ["Field1Name", ["DataType", "Int32"]], + * ["Field2Name", ["DataType", "String"]], + * ... + * ] + * ] + * @endcode + */ + virtual const TVector<NYT::TNode>& GetSchemas() const = 0; + // TODO: make a neat schema builder + + /** + * Get virtual columns for each input. + * + * Key of each mapping is column name, value is data schema in YQL format. + */ + const TVector<THashMap<TString, NYT::TNode>>& GetAllVirtualColumns() const { + if (AllVirtualColumns_.empty()) { + AllVirtualColumns_ = TVector<THashMap<TString, NYT::TNode>>(GetSchemas().size()); } - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * Logging options. - */ - struct TLoggingOptions final { - public: - /// Logging level for messages generated during compilation. - ELogPriority LogLevel; - - /// Where to write log messages. - IOutputStream* LogDestination; - - public: - TLoggingOptions(); - /** - * Set a new logging level. - * - * @return reference to self, to allow method chaining. - */ - TLoggingOptions& SetLogLevel(ELogPriority); - - /** - * Set a new logging destination. - * - * @return reference to self, to allow method chaining. - */ - TLoggingOptions& SetLogDestination(IOutputStream*); - }; - - /** - * General options for program factory. - */ - struct TProgramFactoryOptions final { - public: - /// Path to a directory with compiled UDFs. Leave empty to disable loading external UDFs. - TString UdfsDir; - - /// List of available external resources, e.g. files, UDFs, libraries. - TVector<NUserData::TUserData> UserData; - - /// LLVM settings. Assign "OFF" to disable LLVM, empty string for default settings. - TString LLVMSettings; - - /// Block engine settings. Assign "force" to unconditionally enable - /// it, "disable" for turn it off and "auto" to left the final - /// decision to the platform heuristics. - TString BlockEngineSettings; - - /// Output stream to dump the compiled and optimized expressions. - IOutputStream* ExprOutputStream; - - /// Provider for generic counters which can be used to export statistics from UDFs. - NKikimr::NUdf::ICountersProvider* CountersProvider; - - /// YT Type V3 flags for Skiff/Yson serialization. - ui64 NativeYtTypeFlags; - - /// Seed for deterministic time provider - TMaybe<ui64> DeterministicTimeProviderSeed; - - /// Use special system columns to support tables naming (supports non empty ``TablePath()``/``TableName()``) - bool UseSystemColumns; - - /// Reuse allocated workers - bool UseWorkerPool; - - /// Use Antlr4 parser (for migration) - bool UseAntlr4; - - /// Language version - TLangVersion LangVer; - - public: - TProgramFactoryOptions(); - - public: - /** - * Set language version for queries - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetLanguageVersion(TLangVersion langver); - - /** - * Set a new path to a directory with UDFs. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetUDFsDir(TStringBuf); - - /** - * Add a new library to the UserData list. - * - * @param disposition where the resource resides, e.g. on filesystem, in memory, etc. - * NB: URL disposition is not supported. - * @param name name of the resource. - * @param content depending on disposition, either path to the resource or its content. - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& AddLibrary(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content); - - /** - * Add a new file to the UserData list. - * - * @param disposition where the resource resides, e.g. on filesystem, in memory, etc. - * NB: URL disposition is not supported. - * @param name name of the resource. - * @param content depending on disposition, either path to the resource or its content. - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& AddFile(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content); - - /** - * Add a new UDF to the UserData list. - * - * @param disposition where the resource resides, e.g. on filesystem, in memory, etc. - * NB: URL disposition is not supported. - * @param name name of the resource. - * @param content depending on disposition, either path to the resource or its content. - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& AddUDF(NUserData::EDisposition disposition, TStringBuf name, TStringBuf content); - - /** - * Set new LLVM settings. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetLLVMSettings(TStringBuf llvm_settings); - - /** - * Set new block engine settings. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetBlockEngineSettings(TStringBuf blockEngineSettings); - - /** - * Set the stream to dump the compiled and optimized expressions. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetExprOutputStream(IOutputStream* exprOutputStream); - - /** - * Set new counters provider. Passed pointer should stay alive for as long as the processor factory - * stays alive. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetCountersProvider(NKikimr::NUdf::ICountersProvider* countersProvider); - - /** - * Set new YT Type V3 mode. Deprecated method. Use SetNativeYtTypeFlags instead - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetUseNativeYtTypes(bool useNativeTypes); - - /** - * Set YT Type V3 flags. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetNativeYtTypeFlags(ui64 nativeTypeFlags); - - /** - * Set seed for deterministic time provider. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetDeterministicTimeProviderSeed(TMaybe<ui64> seed); - - /** - * Set new flag whether to allow using system columns or not. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetUseSystemColumns(bool useSystemColumns); - - /** - * Set new flag whether to allow reusing workers or not. - * - * @return reference to self, to allow method chaining. - */ - TProgramFactoryOptions& SetUseWorkerPool(bool useWorkerPool); - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * What exactly are we parsing: SQL or an s-expression. - */ - enum class ETranslationMode { - SQL /* "SQL" */, - SExpr /* "s-expression" */, - Mkql /* "mkql" */, - PG /* PostgreSQL */ - }; - - /** - * A facility for compiling sql and s-expressions and making programs from them. - */ - class IProgramFactory: public TThrRefBase { - protected: - virtual IPullStreamWorkerFactoryPtr MakePullStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) = 0; - virtual IPullListWorkerFactoryPtr MakePullListWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) = 0; - virtual IPushStreamWorkerFactoryPtr MakePushStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) = 0; - - public: - /** - * Add new udf module. It's not specified whether adding new modules will affect existing programs - * (theoretical answer is 'no'). - */ - virtual void AddUdfModule(const TStringBuf&, NKikimr::NUdf::TUniquePtr<NKikimr::NUdf::IUdfModule>&&) = 0; - // TODO: support setting udf modules via factory options. - - /** - * Set new counters provider, override one that was specified via factory options. Note that existing - * programs will still reference the previous provider. - */ - virtual void SetCountersProvider(NKikimr::NUdf::ICountersProvider*) = 0; - // TODO: support setting providers via factory options. - - template <typename TInputSpec, typename TOutputSpec> - THolder<TPullStreamProgram<TInputSpec, TOutputSpec>> MakePullStreamProgram( - TInputSpec inputSpec, TOutputSpec outputSpec, TString query, ETranslationMode mode = ETranslationMode::SQL, ui16 syntaxVersion = 1 - ) { - auto workerFactory = MakePullStreamWorkerFactory(inputSpec, outputSpec, std::move(query), mode, syntaxVersion); - return MakeHolder<TPullStreamProgram<TInputSpec, TOutputSpec>>(std::move(inputSpec), std::move(outputSpec), workerFactory); - } - - template <typename TInputSpec, typename TOutputSpec> - THolder<TPullListProgram<TInputSpec, TOutputSpec>> MakePullListProgram( - TInputSpec inputSpec, TOutputSpec outputSpec, TString query, ETranslationMode mode = ETranslationMode::SQL, ui16 syntaxVersion = 1 - ) { - auto workerFactory = MakePullListWorkerFactory(inputSpec, outputSpec, std::move(query), mode, syntaxVersion); - return MakeHolder<TPullListProgram<TInputSpec, TOutputSpec>>(std::move(inputSpec), std::move(outputSpec), workerFactory); - } - - template <typename TInputSpec, typename TOutputSpec> - THolder<TPushStreamProgram<TInputSpec, TOutputSpec>> MakePushStreamProgram( - TInputSpec inputSpec, TOutputSpec outputSpec, TString query, ETranslationMode mode = ETranslationMode::SQL, ui16 syntaxVersion = 1 - ) { - auto workerFactory = MakePushStreamWorkerFactory(inputSpec, outputSpec, std::move(query), mode, syntaxVersion); - return MakeHolder<TPushStreamProgram<TInputSpec, TOutputSpec>>(std::move(inputSpec), std::move(outputSpec), workerFactory); - } - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * A facility for creating workers. Despite being a part of a public API, worker factory is not used directly. - */ - class IWorkerFactory: public std::enable_shared_from_this<IWorkerFactory> { - public: - virtual ~IWorkerFactory() = default; - /** - * Get input column names for specified input that are actually used in the query. - */ - virtual const THashSet<TString>& GetUsedColumns(ui32) const = 0; - /** - * Overload for single-input programs. - */ - virtual const THashSet<TString>& GetUsedColumns() const = 0; - - /** - * Make input type schema for specified input as deduced by program optimizer. This schema is equivalent - * to one provided by input spec up to the order of the fields in structures. - */ - virtual NYT::TNode MakeInputSchema(ui32) const = 0; - /** - * Overload for single-input programs. - */ - virtual NYT::TNode MakeInputSchema() const = 0; - - /** - * Make output type schema as deduced by program optimizer. If output spec provides its own schema, than - * this schema is equivalent to one provided by output spec up to the order of the fields in structures. - */ - /// @{ - /** - * Overload for single-table output programs (i.e. output type is struct). - */ - virtual NYT::TNode MakeOutputSchema() const = 0; - /** - * Overload for multi-table output programs (i.e. output type is variant over tuple). - */ - virtual NYT::TNode MakeOutputSchema(ui32) const = 0; - /** - * Overload for multi-table output programs (i.e. output type is variant over struct). - */ - virtual NYT::TNode MakeOutputSchema(TStringBuf) const = 0; - /// @} - - /** - * Make full output schema. For single-output programs returns struct type, for multi-output programs - * returns variant type. - * - * Warning: calling this function may result in extended memory usage for large number of output tables. - */ - virtual NYT::TNode MakeFullOutputSchema() const = 0; - - /** - * Get compilation issues - */ - virtual TIssues GetIssues() const = 0; - - /** - * Get precompiled mkql program - */ - virtual TString GetCompiledProgram() = 0; - - /** - * Return a worker to the factory for possible reuse - */ - virtual void ReturnWorker(IWorker* worker) = 0; - }; - - class TReleaseWorker { - public: - template <class T> - static inline void Destroy(T* t) noexcept { - t->Release(); - } - }; - - template <class T> - using TWorkerHolder = THolder<T, TReleaseWorker>; - - /** - * Factory for generating pull stream workers. - */ - class IPullStreamWorkerFactory: public IWorkerFactory { - public: - /** - * Create a new pull stream worker. - */ - virtual TWorkerHolder<IPullStreamWorker> MakeWorker() = 0; - }; - - /** - * Factory for generating pull list workers. - */ - class IPullListWorkerFactory: public IWorkerFactory { - public: - /** - * Create a new pull list worker. - */ - virtual TWorkerHolder<IPullListWorker> MakeWorker() = 0; - }; - - /** - * Factory for generating push stream workers. - */ - class IPushStreamWorkerFactory: public IWorkerFactory { - public: - /** - * Create a new push stream worker. - */ - virtual TWorkerHolder<IPushStreamWorker> MakeWorker() = 0; - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * Worker is a central part of any program instance. It contains current computation state - * (called computation graph) and objects required to work with it, including an allocator for unboxed values. - * - * Usually, users do not interact with workers directly. They use program instance entry points such as streams - * and consumers instead. The only case when one would have to to interact with workers is when implementing - * custom io-specification. - */ - class IWorker { - protected: - friend class TReleaseWorker; - /** - * Cleanup the worker and return to a worker factory for reuse - */ - virtual void Release() = 0; - - public: - virtual ~IWorker() = default; - - public: - /** - * Number of inputs for this program. - */ - virtual ui32 GetInputsCount() const = 0; - - /** - * MiniKQL input struct type of specified input for this program. Type is equivalent to the deduced input - * schema (see IWorker::MakeInputSchema()) - * - * If ``original`` is set to ``true``, returns type without virtual system columns. - */ - virtual const NKikimr::NMiniKQL::TStructType* GetInputType(ui32, bool original = false) const = 0; - /** - * Overload for single-input programs. - */ - virtual const NKikimr::NMiniKQL::TStructType* GetInputType(bool original = false) const = 0; - - /** - * MiniKQL input struct type of the specified input for this program. - * The returned type is the actual type of the specified input node. - */ - virtual const NKikimr::NMiniKQL::TStructType* GetRawInputType(ui32) const = 0; - /** - * Overload for single-input programs. - */ - virtual const NKikimr::NMiniKQL::TStructType* GetRawInputType() const = 0; - - /** - * MiniKQL output struct type for this program. The returned type is equivalent to the deduced output - * schema (see IWorker::MakeFullOutputSchema()). - */ - virtual const NKikimr::NMiniKQL::TType* GetOutputType() const = 0; - - /** - * MiniKQL output struct type for this program. The returned type is - * the actual type of the root node. - */ - virtual const NKikimr::NMiniKQL::TType* GetRawOutputType() const = 0; - - /** - * Make input type schema for specified input as deduced by program optimizer. This schema is equivalent - * to one provided by input spec up to the order of the fields in structures. - */ - virtual NYT::TNode MakeInputSchema(ui32) const = 0; - /** - * Overload for single-input programs. - */ - virtual NYT::TNode MakeInputSchema() const = 0; - - /** - * Make output type schema as deduced by program optimizer. If output spec provides its own schema, than - * this schema is equivalent to one provided by output spec up to the order of the fields in structures. - */ - /// @{ - /** - * Overload for single-table output programs (i.e. output type is struct). - */ - virtual NYT::TNode MakeOutputSchema() const = 0; - /** - * Overload for multi-table output programs (i.e. output type is variant over tuple). - */ - virtual NYT::TNode MakeOutputSchema(ui32) const = 0; - /** - * Overload for multi-table output programs (i.e. output type is variant over struct). - */ - virtual NYT::TNode MakeOutputSchema(TStringBuf) const = 0; - /// @} - - /** - * Generates full output schema. For single-output programs returns struct type, for multi-output programs - * returns variant type. - * - * Warning: calling this function may result in extended memory usage for large number of output tables. - */ - virtual NYT::TNode MakeFullOutputSchema() const = 0; - - /** - * Get scoped alloc used in this worker. - */ - virtual NKikimr::NMiniKQL::TScopedAlloc& GetScopedAlloc() = 0; - - /** - * Get computation graph. - */ - virtual NKikimr::NMiniKQL::IComputationGraph& GetGraph() = 0; - - /** - * Get function registry for this worker. - */ - virtual const NKikimr::NMiniKQL::IFunctionRegistry& GetFunctionRegistry() const = 0; - - /** - * Get type environment for this worker. - */ - virtual NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnvironment() = 0; - - /** - * Get llvm settings for this worker. - */ - virtual const TString& GetLLVMSettings() const = 0; - - /** - * Get YT Type V3 flags - */ - virtual ui64 GetNativeYtTypeFlags() const = 0; - - /** - * Get time provider - */ - virtual ITimeProvider* GetTimeProvider() const = 0; - - /** - * Release all input data from worker state - */ - virtual void Invalidate() = 0; - }; - - /** - * Worker which operates in pull stream mode. - */ - class IPullStreamWorker: public IWorker { - public: - /** - * Set input computation graph node for specified input. The passed unboxed value should be a stream of - * structs. It should be created via the allocator associated with this very worker. - * This function can only be called once for each input. - */ - virtual void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) = 0; - - /** - * Get the output computation graph node. The returned node will be a stream of structs or variants. - * This function cannot be called before setting an input value. - */ - virtual NKikimr::NUdf::TUnboxedValue& GetOutput() = 0; - }; - - /** - * Worker which operates in pull list mode. - */ - class IPullListWorker: public IWorker { - public: - /** - * Set input computation graph node for specified input. The passed unboxed value should be a list of - * structs. It should be created via the allocator associated with this very worker. - * This function can only be called once for each index. - */ - virtual void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) = 0; - - /** - * Get the output computation graph node. The returned node will be a list of structs or variants. - * This function cannot be called before setting an input value. - */ - virtual NKikimr::NUdf::TUnboxedValue& GetOutput() = 0; - - /** - * Get iterator over the output list. - */ - virtual NKikimr::NUdf::TUnboxedValue& GetOutputIterator() = 0; - - /** - * Reset iterator to the beginning of the output list. After calling this function, GetOutputIterator() - * will return a fresh iterator; all previously returned iterators will become invalid. - */ - virtual void ResetOutputIterator() = 0; - }; - - /** - * Worker which operates in push stream mode. - */ - class IPushStreamWorker: public IWorker { - public: - /** - * Set a consumer where the worker will relay its output. This function can only be called once. - */ - virtual void SetConsumer(THolder<IConsumer<const NKikimr::NUdf::TUnboxedValue*>>) = 0; - - /** - * Push new value to the graph, than feed all new output to the consumer. Values cannot be pushed before - * assigning a consumer. - */ - virtual void Push(NKikimr::NUdf::TUnboxedValue&&) = 0; - - /** - * Send finish event and clear the computation graph. No new values will be accepted. - */ - virtual void OnFinish() = 0; - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * Input specifications describe format for program input. They carry information about input data schema - * as well as the knowledge about how to convert input structures into unboxed values (data format which can be - * processed by the YQL runtime). - * - * Input spec defines the arguments of the program's Apply method. For example, a program - * with the protobuf input spec will accept a stream of protobuf messages while a program with the - * yson spec will accept an input stream (binary or text one). - * - * See documentation for input and output spec traits for hints on how to implement a custom specs. - */ - class TInputSpecBase { - protected: - mutable TVector<THashMap<TString, NYT::TNode>> AllVirtualColumns_; - - public: - virtual ~TInputSpecBase() = default; - - public: - /** - * Get input data schemas in YQL format (NB: not a YT format). Each item of the returned vector must - * describe a structure. - * - * Format of each item is approximately this one: - * - * @code - * [ - * 'StructType', - * [ - * ["Field1Name", ["DataType", "Int32"]], - * ["Field2Name", ["DataType", "String"]], - * ... - * ] - * ] - * @endcode - */ - virtual const TVector<NYT::TNode>& GetSchemas() const = 0; - // TODO: make a neat schema builder - - /** - * Get virtual columns for each input. - * - * Key of each mapping is column name, value is data schema in YQL format. - */ - const TVector<THashMap<TString, NYT::TNode>>& GetAllVirtualColumns() const { - if (AllVirtualColumns_.empty()) { - AllVirtualColumns_ = TVector<THashMap<TString, NYT::TNode>>(GetSchemas().size()); - } - - return AllVirtualColumns_; - } - - virtual bool ProvidesBlocks() const { return false; } - }; - - /** - * Output specifications describe format for program output. Like input specifications, they cary knowledge - * about program output type and how to convert unboxed values into that type. - */ - class TOutputSpecBase { - private: - TMaybe<THashSet<TString>> OutputColumnsFilter_; - - public: - virtual ~TOutputSpecBase() = default; - - public: - /** - * Get output data schema in YQL format (NB: not a YT format). The returned value must describe a structure - * or a variant made of structures for fulti-table outputs (note: not all specs support multi-table output). - * - * See docs for the input spec's GetSchemas(). - * - * Also TNode entity could be returned (NYT::TNode::CreateEntity()), - * in which case output schema would be inferred from query and could be - * obtained by Program::GetOutputSchema() call. - */ - virtual const NYT::TNode& GetSchema() const = 0; - - /** - * Get an output columns filter. - * - * Output columns filter is a set of column names that should be left in the output. All columns that are - * not in this set will not be calculated. Depending on the output schema, they will be either removed - * completely (for optional columns) or filled with defaults (for required columns). - */ - const TMaybe<THashSet<TString>>& GetOutputColumnsFilter() const { - return OutputColumnsFilter_; - } - - /** - * Set new output columns filter. - */ - void SetOutputColumnsFilter(const TMaybe<THashSet<TString>>& outputColumnsFilter) { - OutputColumnsFilter_ = outputColumnsFilter; - } - - virtual bool AcceptsBlocks() const { return false; } - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * Input spec traits provide information on how to process program input. - * - * Each input spec should create a template specialization for this class, in which it should provide several - * static variables and functions. - * - * For example, a hypothetical example of implementing a JSON input spec would look like this: - * - * @code - * class TJsonInputSpec: public TInputSpecBase { - * // whatever magic you require for this spec - * }; - * - * template <> - * class TInputSpecTraits<TJsonInputSpec> { - * // write here four constants, one typedef and three static functions described below - * }; - * @endcode - * - * @tparam T input spec type. - */ - template <typename T> - struct TInputSpecTraits { - /// Safety flag which should be set to false in all template specializations of this class. Attempt to - /// build a program using a spec with `IsPartial=true` will result in compilation error. - static const constexpr bool IsPartial = true; - - /// Indicates whether this spec supports pull stream mode. - static const constexpr bool SupportPullStreamMode = false; - /// Indicates whether this spec supports pull list mode. - static const constexpr bool SupportPullListMode = false; - /// Indicates whether this spec supports push stream mode. - static const constexpr bool SupportPushStreamMode = false; - - /// For push mode, indicates the return type of the builder's Process function. - using TConsumerType = void; - - /// For pull stream mode, should take an input spec, a pull stream worker and whatever the user passed - /// to the program's Apply function, create an unboxed values with a custom stream implementations - /// and pass it to the worker's SetInput function for each input. - template <typename ...A> - static void PreparePullStreamWorker(const T&, IPullStreamWorker*, A&&...) { - Y_UNREACHABLE(); - } - - /// For pull list mode, should take an input spec, a pull list worker and whatever the user passed - /// to the program's Apply function, create an unboxed values with a custom list implementations - /// and pass it to the worker's SetInput function for each input. - template <typename ...A> - static void PreparePullListWorker(const T&, IPullListWorker*, A&&...) { - Y_UNREACHABLE(); - } - - /// For push stream mode, should take an input spec and a worker and create a consumer which will - /// be returned to the user. The consumer should keep the worker alive until its own destruction. - /// The return type of this function should exactly match the one defined in ConsumerType typedef. - static TConsumerType MakeConsumer(const T&, TWorkerHolder<IPushStreamWorker>) { - Y_UNREACHABLE(); - } - }; - - /** - * Output spec traits provide information on how to process program output. Like with input specs, each output - * spec requires an appropriate template specialization of this class. - * - * @tparam T output spec type. - */ - template <typename T> - struct TOutputSpecTraits { - /// Safety flag which should be set to false in all template specializations of this class. Attempt to - /// build a program using a spec with `IsPartial=false` will result in compilation error. - static const constexpr bool IsPartial = true; - - /// Indicates whether this spec supports pull stream mode. - static const constexpr bool SupportPullStreamMode = false; - /// Indicates whether this spec supports pull list mode. - static const constexpr bool SupportPullListMode = false; - /// Indicates whether this spec supports push stream mode. - static const constexpr bool SupportPushStreamMode = false; - - /// For pull stream mode, indicates the return type of the program's Apply function. - using TPullStreamReturnType = void; - - /// For pull list mode, indicates the return type of the program's Apply function. - using TPullListReturnType = void; - - /// For pull stream mode, should take an output spec and a worker and build a stream which will be returned - /// to the user. The return type of this function must match the one specified in the PullStreamReturnType. - static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const T&, TWorkerHolder<IPullStreamWorker>) { - Y_UNREACHABLE(); - } - - /// For pull list mode, should take an output spec and a worker and build a list which will be returned - /// to the user. The return type of this function must match the one specified in the PullListReturnType. - static TPullListReturnType ConvertPullListWorkerToOutputType(const T&, TWorkerHolder<IPullListWorker>) { - Y_UNREACHABLE(); - } - - /// For push stream mode, should take an output spec, a worker and whatever arguments the user passed - /// to the program's Apply function, create a consumer for unboxed values and pass it to the worker's - /// SetConsumer function. - template <typename ...A> - static void SetConsumerToWorker(const T&, IPushStreamWorker*, A&&...) { - Y_UNREACHABLE(); - } - }; - - //////////////////////////////////////////////////////////////////////////////////////////////////// + return AllVirtualColumns_; + } + + virtual bool ProvidesBlocks() const { + return false; + } +}; + +/** + * Output specifications describe format for program output. Like input specifications, they cary knowledge + * about program output type and how to convert unboxed values into that type. + */ +class TOutputSpecBase { +private: + TMaybe<THashSet<TString>> OutputColumnsFilter_; + +public: + virtual ~TOutputSpecBase() = default; + +public: + /** + * Get output data schema in YQL format (NB: not a YT format). The returned value must describe a structure + * or a variant made of structures for fulti-table outputs (note: not all specs support multi-table output). + * + * See docs for the input spec's GetSchemas(). + * + * Also TNode entity could be returned (NYT::TNode::CreateEntity()), + * in which case output schema would be inferred from query and could be + * obtained by Program::GetOutputSchema() call. + */ + virtual const NYT::TNode& GetSchema() const = 0; + + /** + * Get an output columns filter. + * + * Output columns filter is a set of column names that should be left in the output. All columns that are + * not in this set will not be calculated. Depending on the output schema, they will be either removed + * completely (for optional columns) or filled with defaults (for required columns). + */ + const TMaybe<THashSet<TString>>& GetOutputColumnsFilter() const { + return OutputColumnsFilter_; + } + + /** + * Set new output columns filter. + */ + void SetOutputColumnsFilter(const TMaybe<THashSet<TString>>& outputColumnsFilter) { + OutputColumnsFilter_ = outputColumnsFilter; + } + + virtual bool AcceptsBlocks() const { + return false; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** + * Input spec traits provide information on how to process program input. + * + * Each input spec should create a template specialization for this class, in which it should provide several + * static variables and functions. + * + * For example, a hypothetical example of implementing a JSON input spec would look like this: + * + * @code + * class TJsonInputSpec: public TInputSpecBase { + * // whatever magic you require for this spec + * }; + * + * template <> + * class TInputSpecTraits<TJsonInputSpec> { + * // write here four constants, one typedef and three static functions described below + * }; + * @endcode + * + * @tparam T input spec type. + */ +template <typename T> +struct TInputSpecTraits { + /// Safety flag which should be set to false in all template specializations of this class. Attempt to + /// build a program using a spec with `IsPartial=true` will result in compilation error. + static const constexpr bool IsPartial = true; + + /// Indicates whether this spec supports pull stream mode. + static const constexpr bool SupportPullStreamMode = false; + /// Indicates whether this spec supports pull list mode. + static const constexpr bool SupportPullListMode = false; + /// Indicates whether this spec supports push stream mode. + static const constexpr bool SupportPushStreamMode = false; + + /// For push mode, indicates the return type of the builder's Process function. + using TConsumerType = void; + + /// For pull stream mode, should take an input spec, a pull stream worker and whatever the user passed + /// to the program's Apply function, create an unboxed values with a custom stream implementations + /// and pass it to the worker's SetInput function for each input. + template <typename... A> + static void PreparePullStreamWorker(const T&, IPullStreamWorker*, A&&...) { + Y_UNREACHABLE(); + } + + /// For pull list mode, should take an input spec, a pull list worker and whatever the user passed + /// to the program's Apply function, create an unboxed values with a custom list implementations + /// and pass it to the worker's SetInput function for each input. + template <typename... A> + static void PreparePullListWorker(const T&, IPullListWorker*, A&&...) { + Y_UNREACHABLE(); + } + + /// For push stream mode, should take an input spec and a worker and create a consumer which will + /// be returned to the user. The consumer should keep the worker alive until its own destruction. + /// The return type of this function should exactly match the one defined in ConsumerType typedef. + static TConsumerType MakeConsumer(const T&, TWorkerHolder<IPushStreamWorker>) { + Y_UNREACHABLE(); + } +}; + +/** + * Output spec traits provide information on how to process program output. Like with input specs, each output + * spec requires an appropriate template specialization of this class. + * + * @tparam T output spec type. + */ +template <typename T> +struct TOutputSpecTraits { + /// Safety flag which should be set to false in all template specializations of this class. Attempt to + /// build a program using a spec with `IsPartial=false` will result in compilation error. + static const constexpr bool IsPartial = true; + + /// Indicates whether this spec supports pull stream mode. + static const constexpr bool SupportPullStreamMode = false; + /// Indicates whether this spec supports pull list mode. + static const constexpr bool SupportPullListMode = false; + /// Indicates whether this spec supports push stream mode. + static const constexpr bool SupportPushStreamMode = false; + + /// For pull stream mode, indicates the return type of the program's Apply function. + using TPullStreamReturnType = void; + + /// For pull list mode, indicates the return type of the program's Apply function. + using TPullListReturnType = void; + + /// For pull stream mode, should take an output spec and a worker and build a stream which will be returned + /// to the user. The return type of this function must match the one specified in the PullStreamReturnType. + static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const T&, TWorkerHolder<IPullStreamWorker>) { + Y_UNREACHABLE(); + } + + /// For pull list mode, should take an output spec and a worker and build a list which will be returned + /// to the user. The return type of this function must match the one specified in the PullListReturnType. + static TPullListReturnType ConvertPullListWorkerToOutputType(const T&, TWorkerHolder<IPullListWorker>) { + Y_UNREACHABLE(); + } + + /// For push stream mode, should take an output spec, a worker and whatever arguments the user passed + /// to the program's Apply function, create a consumer for unboxed values and pass it to the worker's + /// SetConsumer function. + template <typename... A> + static void SetConsumerToWorker(const T&, IPushStreamWorker*, A&&...) { + Y_UNREACHABLE(); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// #define NOT_SPEC_MSG(spec_type) "passed class should be derived from " spec_type " spec base" #define PARTIAL_SPEC_MSG(spec_type) "this " spec_type " spec does not define its traits. Make sure you've passed " \ @@ -996,204 +996,203 @@ namespace NYql { "a spec traits template specialization" #define UNSUPPORTED_MODE_MSG(spec_type, mode) "this " spec_type " spec does not support " mode " mode" - class IProgram { - public: - virtual ~IProgram() = default; - - public: - virtual const TInputSpecBase& GetInputSpecBase() const = 0; - virtual const TOutputSpecBase& GetOutputSpecBase() const = 0; - virtual const THashSet<TString>& GetUsedColumns(ui32) const = 0; - virtual const THashSet<TString>& GetUsedColumns() const = 0; - virtual NYT::TNode MakeInputSchema(ui32) const = 0; - virtual NYT::TNode MakeInputSchema() const = 0; - virtual NYT::TNode MakeOutputSchema() const = 0; - virtual NYT::TNode MakeOutputSchema(ui32) const = 0; - virtual NYT::TNode MakeOutputSchema(TStringBuf) const = 0; - virtual NYT::TNode MakeFullOutputSchema() const = 0; - virtual TIssues GetIssues() const = 0; - virtual TString GetCompiledProgram() = 0; - - inline void MergeUsedColumns(THashSet<TString>& columns, ui32 inputIndex) { - const auto& usedColumns = GetUsedColumns(inputIndex); - columns.insert(usedColumns.begin(), usedColumns.end()); - } - - inline void MergeUsedColumns(THashSet<TString>& columns) { - const auto& usedColumns = GetUsedColumns(); - columns.insert(usedColumns.begin(), usedColumns.end()); - } - }; - - template <typename TInputSpec, typename TOutputSpec, typename WorkerFactory> - class TProgramCommon: public IProgram { - static_assert(std::is_base_of<TInputSpecBase, TInputSpec>::value, NOT_SPEC_MSG("input")); - static_assert(std::is_base_of<TOutputSpecBase, TOutputSpec>::value, NOT_SPEC_MSG("output")); - - protected: - TInputSpec InputSpec_; - TOutputSpec OutputSpec_; - std::shared_ptr<WorkerFactory> WorkerFactory_; - - public: - explicit TProgramCommon( - TInputSpec inputSpec, - TOutputSpec outputSpec, - std::shared_ptr<WorkerFactory> workerFactory - ) - : InputSpec_(inputSpec) - , OutputSpec_(outputSpec) - , WorkerFactory_(std::move(workerFactory)) - { - } - - public: - const TInputSpec& GetInputSpec() const { - return InputSpec_; - } - - const TOutputSpec& GetOutputSpec() const { - return OutputSpec_; - } - - const TInputSpecBase& GetInputSpecBase() const override { - return InputSpec_; - } - - const TOutputSpecBase& GetOutputSpecBase() const override { - return OutputSpec_; - } - - const THashSet<TString>& GetUsedColumns(ui32 inputIndex) const override { - return WorkerFactory_->GetUsedColumns(inputIndex); - } - - const THashSet<TString>& GetUsedColumns() const override { - return WorkerFactory_->GetUsedColumns(); - } - - NYT::TNode MakeInputSchema(ui32 inputIndex) const override { - return WorkerFactory_->MakeInputSchema(inputIndex); - } - - NYT::TNode MakeInputSchema() const override { - return WorkerFactory_->MakeInputSchema(); - } - - NYT::TNode MakeOutputSchema() const override { - return WorkerFactory_->MakeOutputSchema(); - } - - NYT::TNode MakeOutputSchema(ui32 outputIndex) const override { - return WorkerFactory_->MakeOutputSchema(outputIndex); - } - - NYT::TNode MakeOutputSchema(TStringBuf outputName) const override { - return WorkerFactory_->MakeOutputSchema(outputName); - } - - NYT::TNode MakeFullOutputSchema() const override { - return WorkerFactory_->MakeFullOutputSchema(); - } - - TIssues GetIssues() const override { - return WorkerFactory_->GetIssues(); - } - - TString GetCompiledProgram() override { - return WorkerFactory_->GetCompiledProgram(); - } - }; - - template <typename TInputSpec, typename TOutputSpec> - class TPullStreamProgram final: public TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory> { - using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::WorkerFactory_; - using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::InputSpec_; - using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::OutputSpec_; - - public: - using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::TProgramCommon; - - public: - template <typename ...T> - typename TOutputSpecTraits<TOutputSpec>::TPullStreamReturnType Apply(T&& ... t) { - static_assert(!TInputSpecTraits<TInputSpec>::IsPartial, PARTIAL_SPEC_MSG("input")); - static_assert(!TOutputSpecTraits<TOutputSpec>::IsPartial, PARTIAL_SPEC_MSG("output")); - static_assert(TInputSpecTraits<TInputSpec>::SupportPullStreamMode, UNSUPPORTED_MODE_MSG("input", "pull stream")); - static_assert(TOutputSpecTraits<TOutputSpec>::SupportPullStreamMode, UNSUPPORTED_MODE_MSG("output", "pull stream")); - - auto worker = WorkerFactory_->MakeWorker(); - TInputSpecTraits<TInputSpec>::PreparePullStreamWorker(InputSpec_, worker.Get(), std::forward<T>(t)...); - return TOutputSpecTraits<TOutputSpec>::ConvertPullStreamWorkerToOutputType(OutputSpec_, std::move(worker)); - } - }; - - template <typename TInputSpec, typename TOutputSpec> - class TPullListProgram final: public TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory> { - using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::WorkerFactory_; - using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::InputSpec_; - using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::OutputSpec_; - - public: - using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::TProgramCommon; - - public: - template <typename ...T> - typename TOutputSpecTraits<TOutputSpec>::TPullListReturnType Apply(T&& ... t) { - static_assert(!TInputSpecTraits<TInputSpec>::IsPartial, PARTIAL_SPEC_MSG("input")); - static_assert(!TOutputSpecTraits<TOutputSpec>::IsPartial, PARTIAL_SPEC_MSG("output")); - static_assert(TInputSpecTraits<TInputSpec>::SupportPullListMode, UNSUPPORTED_MODE_MSG("input", "pull list")); - static_assert(TOutputSpecTraits<TOutputSpec>::SupportPullListMode, UNSUPPORTED_MODE_MSG("output", "pull list")); - - auto worker = WorkerFactory_->MakeWorker(); - TInputSpecTraits<TInputSpec>::PreparePullListWorker(InputSpec_, worker.Get(), std::forward<T>(t)...); - return TOutputSpecTraits<TOutputSpec>::ConvertPullListWorkerToOutputType(OutputSpec_, std::move(worker)); - } - }; - - template <typename TInputSpec, typename TOutputSpec> - class TPushStreamProgram final: public TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory> { - using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::WorkerFactory_; - using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::InputSpec_; - using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::OutputSpec_; - - public: - using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::TProgramCommon; - - public: - template <typename ...T> - typename TInputSpecTraits<TInputSpec>::TConsumerType Apply(T&& ... t) { - static_assert(!TInputSpecTraits<TInputSpec>::IsPartial, PARTIAL_SPEC_MSG("input")); - static_assert(!TOutputSpecTraits<TOutputSpec>::IsPartial, PARTIAL_SPEC_MSG("output")); - static_assert(TInputSpecTraits<TInputSpec>::SupportPushStreamMode, UNSUPPORTED_MODE_MSG("input", "push stream")); - static_assert(TOutputSpecTraits<TOutputSpec>::SupportPushStreamMode, UNSUPPORTED_MODE_MSG("output", "push stream")); - - auto worker = WorkerFactory_->MakeWorker(); - TOutputSpecTraits<TOutputSpec>::SetConsumerToWorker(OutputSpec_, worker.Get(), std::forward<T>(t)...); - return TInputSpecTraits<TInputSpec>::MakeConsumer(InputSpec_, std::move(worker)); - } - }; +class IProgram { +public: + virtual ~IProgram() = default; + +public: + virtual const TInputSpecBase& GetInputSpecBase() const = 0; + virtual const TOutputSpecBase& GetOutputSpecBase() const = 0; + virtual const THashSet<TString>& GetUsedColumns(ui32) const = 0; + virtual const THashSet<TString>& GetUsedColumns() const = 0; + virtual NYT::TNode MakeInputSchema(ui32) const = 0; + virtual NYT::TNode MakeInputSchema() const = 0; + virtual NYT::TNode MakeOutputSchema() const = 0; + virtual NYT::TNode MakeOutputSchema(ui32) const = 0; + virtual NYT::TNode MakeOutputSchema(TStringBuf) const = 0; + virtual NYT::TNode MakeFullOutputSchema() const = 0; + virtual TIssues GetIssues() const = 0; + virtual TString GetCompiledProgram() = 0; + + inline void MergeUsedColumns(THashSet<TString>& columns, ui32 inputIndex) { + const auto& usedColumns = GetUsedColumns(inputIndex); + columns.insert(usedColumns.begin(), usedColumns.end()); + } + + inline void MergeUsedColumns(THashSet<TString>& columns) { + const auto& usedColumns = GetUsedColumns(); + columns.insert(usedColumns.begin(), usedColumns.end()); + } +}; + +template <typename TInputSpec, typename TOutputSpec, typename WorkerFactory> +class TProgramCommon: public IProgram { + static_assert(std::is_base_of<TInputSpecBase, TInputSpec>::value, NOT_SPEC_MSG("input")); + static_assert(std::is_base_of<TOutputSpecBase, TOutputSpec>::value, NOT_SPEC_MSG("output")); + +protected: + TInputSpec InputSpec_; + TOutputSpec OutputSpec_; + std::shared_ptr<WorkerFactory> WorkerFactory_; + +public: + explicit TProgramCommon( + TInputSpec inputSpec, + TOutputSpec outputSpec, + std::shared_ptr<WorkerFactory> workerFactory) + : InputSpec_(inputSpec) + , OutputSpec_(outputSpec) + , WorkerFactory_(std::move(workerFactory)) + { + } + +public: + const TInputSpec& GetInputSpec() const { + return InputSpec_; + } + + const TOutputSpec& GetOutputSpec() const { + return OutputSpec_; + } + + const TInputSpecBase& GetInputSpecBase() const override { + return InputSpec_; + } + + const TOutputSpecBase& GetOutputSpecBase() const override { + return OutputSpec_; + } + + const THashSet<TString>& GetUsedColumns(ui32 inputIndex) const override { + return WorkerFactory_->GetUsedColumns(inputIndex); + } + + const THashSet<TString>& GetUsedColumns() const override { + return WorkerFactory_->GetUsedColumns(); + } + + NYT::TNode MakeInputSchema(ui32 inputIndex) const override { + return WorkerFactory_->MakeInputSchema(inputIndex); + } + + NYT::TNode MakeInputSchema() const override { + return WorkerFactory_->MakeInputSchema(); + } + + NYT::TNode MakeOutputSchema() const override { + return WorkerFactory_->MakeOutputSchema(); + } + + NYT::TNode MakeOutputSchema(ui32 outputIndex) const override { + return WorkerFactory_->MakeOutputSchema(outputIndex); + } + + NYT::TNode MakeOutputSchema(TStringBuf outputName) const override { + return WorkerFactory_->MakeOutputSchema(outputName); + } + + NYT::TNode MakeFullOutputSchema() const override { + return WorkerFactory_->MakeFullOutputSchema(); + } + + TIssues GetIssues() const override { + return WorkerFactory_->GetIssues(); + } + + TString GetCompiledProgram() override { + return WorkerFactory_->GetCompiledProgram(); + } +}; + +template <typename TInputSpec, typename TOutputSpec> +class TPullStreamProgram final: public TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory> { + using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::WorkerFactory_; + using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::InputSpec_; + using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::OutputSpec_; + +public: + using TProgramCommon<TInputSpec, TOutputSpec, IPullStreamWorkerFactory>::TProgramCommon; + +public: + template <typename... T> + typename TOutputSpecTraits<TOutputSpec>::TPullStreamReturnType Apply(T&&... t) { + static_assert(!TInputSpecTraits<TInputSpec>::IsPartial, PARTIAL_SPEC_MSG("input")); + static_assert(!TOutputSpecTraits<TOutputSpec>::IsPartial, PARTIAL_SPEC_MSG("output")); + static_assert(TInputSpecTraits<TInputSpec>::SupportPullStreamMode, UNSUPPORTED_MODE_MSG("input", "pull stream")); + static_assert(TOutputSpecTraits<TOutputSpec>::SupportPullStreamMode, UNSUPPORTED_MODE_MSG("output", "pull stream")); + + auto worker = WorkerFactory_->MakeWorker(); + TInputSpecTraits<TInputSpec>::PreparePullStreamWorker(InputSpec_, worker.Get(), std::forward<T>(t)...); + return TOutputSpecTraits<TOutputSpec>::ConvertPullStreamWorkerToOutputType(OutputSpec_, std::move(worker)); + } +}; + +template <typename TInputSpec, typename TOutputSpec> +class TPullListProgram final: public TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory> { + using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::WorkerFactory_; + using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::InputSpec_; + using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::OutputSpec_; + +public: + using TProgramCommon<TInputSpec, TOutputSpec, IPullListWorkerFactory>::TProgramCommon; + +public: + template <typename... T> + typename TOutputSpecTraits<TOutputSpec>::TPullListReturnType Apply(T&&... t) { + static_assert(!TInputSpecTraits<TInputSpec>::IsPartial, PARTIAL_SPEC_MSG("input")); + static_assert(!TOutputSpecTraits<TOutputSpec>::IsPartial, PARTIAL_SPEC_MSG("output")); + static_assert(TInputSpecTraits<TInputSpec>::SupportPullListMode, UNSUPPORTED_MODE_MSG("input", "pull list")); + static_assert(TOutputSpecTraits<TOutputSpec>::SupportPullListMode, UNSUPPORTED_MODE_MSG("output", "pull list")); + + auto worker = WorkerFactory_->MakeWorker(); + TInputSpecTraits<TInputSpec>::PreparePullListWorker(InputSpec_, worker.Get(), std::forward<T>(t)...); + return TOutputSpecTraits<TOutputSpec>::ConvertPullListWorkerToOutputType(OutputSpec_, std::move(worker)); + } +}; + +template <typename TInputSpec, typename TOutputSpec> +class TPushStreamProgram final: public TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory> { + using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::WorkerFactory_; + using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::InputSpec_; + using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::OutputSpec_; + +public: + using TProgramCommon<TInputSpec, TOutputSpec, IPushStreamWorkerFactory>::TProgramCommon; + +public: + template <typename... T> + typename TInputSpecTraits<TInputSpec>::TConsumerType Apply(T&&... t) { + static_assert(!TInputSpecTraits<TInputSpec>::IsPartial, PARTIAL_SPEC_MSG("input")); + static_assert(!TOutputSpecTraits<TOutputSpec>::IsPartial, PARTIAL_SPEC_MSG("output")); + static_assert(TInputSpecTraits<TInputSpec>::SupportPushStreamMode, UNSUPPORTED_MODE_MSG("input", "push stream")); + static_assert(TOutputSpecTraits<TOutputSpec>::SupportPushStreamMode, UNSUPPORTED_MODE_MSG("output", "push stream")); + + auto worker = WorkerFactory_->MakeWorker(); + TOutputSpecTraits<TOutputSpec>::SetConsumerToWorker(OutputSpec_, worker.Get(), std::forward<T>(t)...); + return TInputSpecTraits<TInputSpec>::MakeConsumer(InputSpec_, std::move(worker)); + } +}; #undef NOT_SPEC_MSG #undef PARTIAL_SPEC_MSG #undef UNSUPPORTED_MODE_MSG - //////////////////////////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////////////////////////// - /** - * Configure global logging facilities. Affects all YQL modules. - */ - void ConfigureLogging(const TLoggingOptions& = {}); +/** + * Configure global logging facilities. Affects all YQL modules. + */ +void ConfigureLogging(const TLoggingOptions& = {}); - /** - * Create a new program factory. - * Custom logging initialization could be preformed by a call to the ConfigureLogging method beforehand. - * If the ConfigureLogging method has not been called the default logging initialization will be performed. - */ - IProgramFactoryPtr MakeProgramFactory(const TProgramFactoryOptions& = {}); - } -} +/** + * Create a new program factory. + * Custom logging initialization could be preformed by a call to the ConfigureLogging method beforehand. + * If the ConfigureLogging method has not been called the default logging initialization will be performed. + */ +IProgramFactoryPtr MakeProgramFactory(const TProgramFactoryOptions& = {}); +} // namespace NPureCalc +} // namespace NYql Y_DECLARE_OUT_SPEC(inline, NYql::NPureCalc::TCompileError, stream, value) { - stream << value.AsStrBuf() << Endl << "Issues:" << Endl << value.GetIssues() << Endl << Endl << "Yql:" << Endl <<value.GetYql(); + stream << value.AsStrBuf() << Endl << "Issues:" << Endl << value.GetIssues() << Endl << Endl << "Yql:" << Endl << value.GetYql(); } diff --git a/yql/essentials/public/purecalc/common/logger_init.cpp b/yql/essentials/public/purecalc/common/logger_init.cpp index 0c1e53d0338..ec3f66140bb 100644 --- a/yql/essentials/public/purecalc/common/logger_init.cpp +++ b/yql/essentials/public/purecalc/common/logger_init.cpp @@ -8,25 +8,25 @@ namespace NYql { namespace NPureCalc { namespace { - std::atomic_bool Initialized; -} +std::atomic_bool Initialized; +} // namespace - void InitLogging(const TLoggingOptions& options) { - NLog::InitLogger(options.LogDestination); - auto& logger = NLog::YqlLogger(); - logger.SetDefaultPriority(options.LogLevel); - for (int i = 0; i < NLog::EComponentHelpers::ToInt(NLog::EComponent::MaxValue); ++i) { - logger.SetComponentLevel((NLog::EComponent) i, (NLog::ELevel) options.LogLevel); - } - Initialized = true; +void InitLogging(const TLoggingOptions& options) { + NLog::InitLogger(options.LogDestination); + auto& logger = NLog::YqlLogger(); + logger.SetDefaultPriority(options.LogLevel); + for (int i = 0; i < NLog::EComponentHelpers::ToInt(NLog::EComponent::MaxValue); ++i) { + logger.SetComponentLevel((NLog::EComponent)i, (NLog::ELevel)options.LogLevel); } + Initialized = true; +} - void EnsureLoggingInitialized() { - if (Initialized.load()) { - return; - } - InitLogging(TLoggingOptions()); +void EnsureLoggingInitialized() { + if (Initialized.load()) { + return; } - -} + InitLogging(TLoggingOptions()); } + +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/logger_init.h b/yql/essentials/public/purecalc/common/logger_init.h index 039cbd44118..349d74b0025 100644 --- a/yql/essentials/public/purecalc/common/logger_init.h +++ b/yql/essentials/public/purecalc/common/logger_init.h @@ -3,8 +3,8 @@ #include "interface.h" namespace NYql { - namespace NPureCalc { - void InitLogging(const TLoggingOptions& options); - void EnsureLoggingInitialized(); - } -} +namespace NPureCalc { +void InitLogging(const TLoggingOptions& options); +void EnsureLoggingInitialized(); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/names.cpp b/yql/essentials/public/purecalc/common/names.cpp index 5e8412a7b22..b678969daf6 100644 --- a/yql/essentials/public/purecalc/common/names.cpp +++ b/yql/essentials/public/purecalc/common/names.cpp @@ -3,17 +3,17 @@ #include <util/generic/strbuf.h> namespace NYql::NPureCalc { - const TStringBuf PurecalcSysColumnsPrefix = "_yql_sys_"; - const TStringBuf PurecalcSysColumnTablePath = "_yql_sys_tablepath"; - const TStringBuf PurecalcBlockColumnLength = "_yql_block_length"; +const TStringBuf PurecalcSysColumnsPrefix = "_yql_sys_"; +const TStringBuf PurecalcSysColumnTablePath = "_yql_sys_tablepath"; +const TStringBuf PurecalcBlockColumnLength = "_yql_block_length"; - const TStringBuf PurecalcDefaultCluster = "view"; - const TStringBuf PurecalcDefaultService = "data"; +const TStringBuf PurecalcDefaultCluster = "view"; +const TStringBuf PurecalcDefaultService = "data"; - const TStringBuf PurecalcInputCallableName = "Self"; - const TStringBuf PurecalcInputTablePrefix = "Input"; +const TStringBuf PurecalcInputCallableName = "Self"; +const TStringBuf PurecalcInputTablePrefix = "Input"; - const TStringBuf PurecalcBlockInputCallableName = "BlockSelf"; +const TStringBuf PurecalcBlockInputCallableName = "BlockSelf"; - const TStringBuf PurecalcUdfModulePrefix = "<purecalc>::"; -} +const TStringBuf PurecalcUdfModulePrefix = "<purecalc>::"; +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/names.h b/yql/essentials/public/purecalc/common/names.h index b19c15ca4fe..fccfe7caad8 100644 --- a/yql/essentials/public/purecalc/common/names.h +++ b/yql/essentials/public/purecalc/common/names.h @@ -3,17 +3,17 @@ #include <util/generic/fwd.h> namespace NYql::NPureCalc { - extern const TStringBuf PurecalcSysColumnsPrefix; - extern const TStringBuf PurecalcSysColumnTablePath; - extern const TStringBuf PurecalcBlockColumnLength; +extern const TStringBuf PurecalcSysColumnsPrefix; +extern const TStringBuf PurecalcSysColumnTablePath; +extern const TStringBuf PurecalcBlockColumnLength; - extern const TStringBuf PurecalcDefaultCluster; - extern const TStringBuf PurecalcDefaultService; +extern const TStringBuf PurecalcDefaultCluster; +extern const TStringBuf PurecalcDefaultService; - extern const TStringBuf PurecalcInputCallableName; - extern const TStringBuf PurecalcInputTablePrefix; +extern const TStringBuf PurecalcInputCallableName; +extern const TStringBuf PurecalcInputTablePrefix; - extern const TStringBuf PurecalcBlockInputCallableName; +extern const TStringBuf PurecalcBlockInputCallableName; - extern const TStringBuf PurecalcUdfModulePrefix; -} +extern const TStringBuf PurecalcUdfModulePrefix; +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/processor_mode.h b/yql/essentials/public/purecalc/common/processor_mode.h index 9bec87cadc9..49368c81c11 100644 --- a/yql/essentials/public/purecalc/common/processor_mode.h +++ b/yql/essentials/public/purecalc/common/processor_mode.h @@ -1,11 +1,11 @@ #pragma once namespace NYql { - namespace NPureCalc { - enum class EProcessorMode { - PullList, - PullStream, - PushStream - }; - } -} +namespace NPureCalc { +enum class EProcessorMode { + PullList, + PullStream, + PushStream +}; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/program_factory.cpp b/yql/essentials/public/purecalc/common/program_factory.cpp index 320d1b6aa8a..1096f6acc4f 100644 --- a/yql/essentials/public/purecalc/common/program_factory.cpp +++ b/yql/essentials/public/purecalc/common/program_factory.cpp @@ -33,11 +33,10 @@ TProgramFactory::TProgramFactory(const TProgramFactoryOptions& options) } TVector<TString> UDFsPaths; - for (const auto& item: Options_.UserData) { + for (const auto& item : Options_.UserData) { if ( item.Type == NUserData::EType::UDF && - item.Disposition == NUserData::EDisposition::FILESYSTEM - ) { + item.Disposition == NUserData::EDisposition::FILESYSTEM) { UDFsPaths.push_back(item.Content); } } @@ -47,7 +46,8 @@ TProgramFactory::TProgramFactory(const TProgramFactoryOptions& options) } FuncRegistry_ = NKikimr::NMiniKQL::CreateFunctionRegistry( - &NYql::NBacktrace::KikimrBackTrace, NKikimr::NMiniKQL::CreateBuiltinRegistry(), false, UDFsPaths)->Clone(); + &NYql::NBacktrace::KikimrBackTrace, NKikimr::NMiniKQL::CreateBuiltinRegistry(), false, UDFsPaths) + ->Clone(); NKikimr::NMiniKQL::FillStaticModules(*FuncRegistry_); } @@ -57,11 +57,9 @@ TProgramFactory::~TProgramFactory() { void TProgramFactory::AddUdfModule( const TStringBuf& moduleName, - NKikimr::NUdf::TUniquePtr<NKikimr::NUdf::IUdfModule>&& module -) { + NKikimr::NUdf::TUniquePtr<NKikimr::NUdf::IUdfModule>&& module) { FuncRegistry_->AddModule( - TString::Join(PurecalcUdfModulePrefix, moduleName), moduleName, std::move(module) - ); + TString::Join(PurecalcUdfModulePrefix, moduleName), moduleName, std::move(module)); } void TProgramFactory::SetCountersProvider(NKikimr::NUdf::ICountersProvider* provider) { @@ -73,8 +71,7 @@ IPullStreamWorkerFactoryPtr TProgramFactory::MakePullStreamWorkerFactory( const TOutputSpecBase& outputSpec, TString query, ETranslationMode mode, - ui16 syntaxVersion -) { + ui16 syntaxVersion) { return std::make_shared<TPullStreamWorkerFactory>(TWorkerFactoryOptions( TIntrusivePtr<TProgramFactory>(this), inputSpec, @@ -95,8 +92,7 @@ IPullStreamWorkerFactoryPtr TProgramFactory::MakePullStreamWorkerFactory( Options_.DeterministicTimeProviderSeed, Options_.UseSystemColumns, Options_.UseWorkerPool, - Options_.UseAntlr4 - )); + Options_.UseAntlr4)); } IPullListWorkerFactoryPtr TProgramFactory::MakePullListWorkerFactory( @@ -104,8 +100,7 @@ IPullListWorkerFactoryPtr TProgramFactory::MakePullListWorkerFactory( const TOutputSpecBase& outputSpec, TString query, ETranslationMode mode, - ui16 syntaxVersion -) { + ui16 syntaxVersion) { return std::make_shared<TPullListWorkerFactory>(TWorkerFactoryOptions( TIntrusivePtr<TProgramFactory>(this), inputSpec, @@ -126,8 +121,7 @@ IPullListWorkerFactoryPtr TProgramFactory::MakePullListWorkerFactory( Options_.DeterministicTimeProviderSeed, Options_.UseSystemColumns, Options_.UseWorkerPool, - Options_.UseAntlr4 - )); + Options_.UseAntlr4)); } IPushStreamWorkerFactoryPtr TProgramFactory::MakePushStreamWorkerFactory( @@ -135,8 +129,7 @@ IPushStreamWorkerFactoryPtr TProgramFactory::MakePushStreamWorkerFactory( const TOutputSpecBase& outputSpec, TString query, ETranslationMode mode, - ui16 syntaxVersion -) { + ui16 syntaxVersion) { if (inputSpec.GetSchemas().size() > 1) { ythrow yexception() << "push stream mode doesn't support several inputs"; } @@ -161,6 +154,5 @@ IPushStreamWorkerFactoryPtr TProgramFactory::MakePushStreamWorkerFactory( Options_.DeterministicTimeProviderSeed, Options_.UseSystemColumns, Options_.UseWorkerPool, - Options_.UseAntlr4 - )); + Options_.UseAntlr4)); } diff --git a/yql/essentials/public/purecalc/common/program_factory.h b/yql/essentials/public/purecalc/common/program_factory.h index 278d3e05a6a..e46de24d9ee 100644 --- a/yql/essentials/public/purecalc/common/program_factory.h +++ b/yql/essentials/public/purecalc/common/program_factory.h @@ -15,34 +15,33 @@ #include <util/generic/strbuf.h> namespace NYql { - namespace NPureCalc { - class TProgramFactory: public IProgramFactory { - private: - TProgramFactoryOptions Options_; - TExprContext ExprContext_; - TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry_; - IModuleResolver::TPtr ModuleResolver_; - TUserDataTable UserData_; - EBlockEngineMode BlockEngineMode_; - IOutputStream* ExprOutputStream_; - THashMap<TString, TString> Modules_; - NKikimr::NUdf::ICountersProvider* CountersProvider_; - - public: - explicit TProgramFactory(const TProgramFactoryOptions&); - ~TProgramFactory() override; - - public: - void AddUdfModule( - const TStringBuf& moduleName, - NKikimr::NUdf::TUniquePtr<NKikimr::NUdf::IUdfModule>&& module - ) override; - - void SetCountersProvider(NKikimr::NUdf::ICountersProvider* provider) override; - - IPullStreamWorkerFactoryPtr MakePullStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) override; - IPullListWorkerFactoryPtr MakePullListWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) override; - IPushStreamWorkerFactoryPtr MakePushStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) override; - }; - } -} +namespace NPureCalc { +class TProgramFactory: public IProgramFactory { +private: + TProgramFactoryOptions Options_; + TExprContext ExprContext_; + TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry_; + IModuleResolver::TPtr ModuleResolver_; + TUserDataTable UserData_; + EBlockEngineMode BlockEngineMode_; + IOutputStream* ExprOutputStream_; + THashMap<TString, TString> Modules_; + NKikimr::NUdf::ICountersProvider* CountersProvider_; + +public: + explicit TProgramFactory(const TProgramFactoryOptions&); + ~TProgramFactory() override; + +public: + void AddUdfModule( + const TStringBuf& moduleName, + NKikimr::NUdf::TUniquePtr<NKikimr::NUdf::IUdfModule>&& module) override; + + void SetCountersProvider(NKikimr::NUdf::ICountersProvider* provider) override; + + IPullStreamWorkerFactoryPtr MakePullStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) override; + IPullListWorkerFactoryPtr MakePullListWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) override; + IPushStreamWorkerFactoryPtr MakePushStreamWorkerFactory(const TInputSpecBase&, const TOutputSpecBase&, TString, ETranslationMode, ui16) override; +}; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/transformations/align_output_schema.cpp b/yql/essentials/public/purecalc/common/transformations/align_output_schema.cpp index f11195bf756..47806d6846e 100644 --- a/yql/essentials/public/purecalc/common/transformations/align_output_schema.cpp +++ b/yql/essentials/public/purecalc/common/transformations/align_output_schema.cpp @@ -10,117 +10,115 @@ using namespace NYql; using namespace NYql::NPureCalc; namespace { - class TOutputAligner : public TSyncTransformerBase { - private: - const TTypeAnnotationNode* OutputStruct_; - bool AcceptsBlocks_; - EProcessorMode ProcessorMode_; - const TTypeAnnotationContext& TypeCtx_; - - public: - explicit TOutputAligner( - const TTypeAnnotationNode* outputStruct, - bool acceptsBlocks, - EProcessorMode processorMode, - const TTypeAnnotationContext& typeCtx - ) - : OutputStruct_(outputStruct) - , AcceptsBlocks_(acceptsBlocks) - , ProcessorMode_(processorMode) - , TypeCtx_(typeCtx) - { - } - - public: - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - output = input; - - const auto* expectedType = MakeExpectedType(ctx); - const auto* expectedItemType = MakeExpectedItemType(); - const auto* actualType = MakeActualType(input); - const auto* actualItemType = MakeActualItemType(input); - - // XXX: Tweak the obtained expression type, is the spec supports blocks: - // 1. Remove "_yql_block_length" attribute, since it's for internal usage. - // 2. Strip block container from the type to store its internal type. - if (AcceptsBlocks_) { - Y_ENSURE(actualItemType->GetKind() == ETypeAnnotationKind::Struct); - actualItemType = UnwrapBlockStruct(actualItemType->Cast<TStructExprType>(), ctx); - if (ProcessorMode_ == EProcessorMode::PullList) { - actualType = ctx.MakeType<TListExprType>(actualItemType); - } else { - actualType = ctx.MakeType<TStreamExprType>(actualItemType); - } - } - - if (!ValidateOutputType(actualItemType, expectedItemType, ctx, TypeCtx_)) { - return TStatus::Error; - } - - if (!expectedType) { - return TStatus::Ok; - } - - auto status = TryConvertTo(output, *actualType, *expectedType, ctx, TypeCtx_); - - if (status.Level == IGraphTransformer::TStatus::Repeat) { - status = IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); +class TOutputAligner: public TSyncTransformerBase { +private: + const TTypeAnnotationNode* OutputStruct_; + bool AcceptsBlocks_; + EProcessorMode ProcessorMode_; + const TTypeAnnotationContext& TypeCtx_; + +public: + explicit TOutputAligner( + const TTypeAnnotationNode* outputStruct, + bool acceptsBlocks, + EProcessorMode processorMode, + const TTypeAnnotationContext& typeCtx) + : OutputStruct_(outputStruct) + , AcceptsBlocks_(acceptsBlocks) + , ProcessorMode_(processorMode) + , TypeCtx_(typeCtx) + { + } + +public: + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + output = input; + + const auto* expectedType = MakeExpectedType(ctx); + const auto* expectedItemType = MakeExpectedItemType(); + const auto* actualType = MakeActualType(input); + const auto* actualItemType = MakeActualItemType(input); + + // XXX: Tweak the obtained expression type, is the spec supports blocks: + // 1. Remove "_yql_block_length" attribute, since it's for internal usage. + // 2. Strip block container from the type to store its internal type. + if (AcceptsBlocks_) { + Y_ENSURE(actualItemType->GetKind() == ETypeAnnotationKind::Struct); + actualItemType = UnwrapBlockStruct(actualItemType->Cast<TStructExprType>(), ctx); + if (ProcessorMode_ == EProcessorMode::PullList) { + actualType = ctx.MakeType<TListExprType>(actualItemType); + } else { + actualType = ctx.MakeType<TStreamExprType>(actualItemType); } - - return status; } - void Rewind() final { + if (!ValidateOutputType(actualItemType, expectedItemType, ctx, TypeCtx_)) { + return TStatus::Error; } - private: - const TTypeAnnotationNode* MakeExpectedType(TExprContext& ctx) { - if (!OutputStruct_) { - return nullptr; - } + if (!expectedType) { + return TStatus::Ok; + } - switch (ProcessorMode_) { - case EProcessorMode::PullList: - return ctx.MakeType<TListExprType>(OutputStruct_); - case EProcessorMode::PullStream: - case EProcessorMode::PushStream: - return ctx.MakeType<TStreamExprType>(OutputStruct_); - } + auto status = TryConvertTo(output, *actualType, *expectedType, ctx, TypeCtx_); - Y_ABORT("Unexpected"); + if (status.Level == IGraphTransformer::TStatus::Repeat) { + status = IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); } - const TTypeAnnotationNode* MakeExpectedItemType() { - return OutputStruct_; + return status; + } + + void Rewind() final { + } + +private: + const TTypeAnnotationNode* MakeExpectedType(TExprContext& ctx) { + if (!OutputStruct_) { + return nullptr; } - const TTypeAnnotationNode* MakeActualType(TExprNode::TPtr& input) { - return input->GetTypeAnn(); + switch (ProcessorMode_) { + case EProcessorMode::PullList: + return ctx.MakeType<TListExprType>(OutputStruct_); + case EProcessorMode::PullStream: + case EProcessorMode::PushStream: + return ctx.MakeType<TStreamExprType>(OutputStruct_); } - const TTypeAnnotationNode* MakeActualItemType(TExprNode::TPtr& input) { - auto actualType = MakeActualType(input); - switch (actualType->GetKind()) { - case ETypeAnnotationKind::Stream: - Y_ENSURE(ProcessorMode_ != EProcessorMode::PullList, - "processor mode mismatches the actual container type"); - return actualType->Cast<TStreamExprType>()->GetItemType(); - case ETypeAnnotationKind::List: - Y_ENSURE(ProcessorMode_ == EProcessorMode::PullList, - "processor mode mismatches the actual container type"); - return actualType->Cast<TListExprType>()->GetItemType(); - default: - Y_ABORT("unexpected return type"); - } + Y_ABORT("Unexpected"); + } + + const TTypeAnnotationNode* MakeExpectedItemType() { + return OutputStruct_; + } + + const TTypeAnnotationNode* MakeActualType(TExprNode::TPtr& input) { + return input->GetTypeAnn(); + } + + const TTypeAnnotationNode* MakeActualItemType(TExprNode::TPtr& input) { + auto actualType = MakeActualType(input); + switch (actualType->GetKind()) { + case ETypeAnnotationKind::Stream: + Y_ENSURE(ProcessorMode_ != EProcessorMode::PullList, + "processor mode mismatches the actual container type"); + return actualType->Cast<TStreamExprType>()->GetItemType(); + case ETypeAnnotationKind::List: + Y_ENSURE(ProcessorMode_ == EProcessorMode::PullList, + "processor mode mismatches the actual container type"); + return actualType->Cast<TListExprType>()->GetItemType(); + default: + Y_ABORT("unexpected return type"); } - }; -} + } +}; +} // namespace TAutoPtr<IGraphTransformer> NYql::NPureCalc::MakeOutputAligner( const TTypeAnnotationNode* outputStruct, bool acceptsBlocks, EProcessorMode processorMode, - const TTypeAnnotationContext& typeCtx -) { + const TTypeAnnotationContext& typeCtx) { return new TOutputAligner(outputStruct, acceptsBlocks, processorMode, typeCtx); } diff --git a/yql/essentials/public/purecalc/common/transformations/align_output_schema.h b/yql/essentials/public/purecalc/common/transformations/align_output_schema.h index a1bf1717ce1..4cbcdcd81fa 100644 --- a/yql/essentials/public/purecalc/common/transformations/align_output_schema.h +++ b/yql/essentials/public/purecalc/common/transformations/align_output_schema.h @@ -6,23 +6,22 @@ #include <yql/essentials/core/yql_type_annotation.h> namespace NYql { - struct TTypeAnnotationContext; +struct TTypeAnnotationContext; - namespace NPureCalc { - /** - * A transformer which converts an output type of the expression to the given type or reports an error. - * - * @param outputStruct destination output struct type. - * @param acceptsBlocks indicates, whether the output type need to be - * preprocessed. - * @param processorMode specifies the top-most container of the result. - * @return a graph transformer for type alignment. - */ - TAutoPtr<IGraphTransformer> MakeOutputAligner( - const TTypeAnnotationNode* outputStruct, - bool acceptsBlocks, - EProcessorMode processorMode, - const TTypeAnnotationContext& typeCtx - ); - } -} +namespace NPureCalc { +/** + * A transformer which converts an output type of the expression to the given type or reports an error. + * + * @param outputStruct destination output struct type. + * @param acceptsBlocks indicates, whether the output type need to be + * preprocessed. + * @param processorMode specifies the top-most container of the result. + * @return a graph transformer for type alignment. + */ +TAutoPtr<IGraphTransformer> MakeOutputAligner( + const TTypeAnnotationNode* outputStruct, + bool acceptsBlocks, + EProcessorMode processorMode, + const TTypeAnnotationContext& typeCtx); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/transformations/extract_used_columns.cpp b/yql/essentials/public/purecalc/common/transformations/extract_used_columns.cpp index 9ff7a0df638..4190ce1fad9 100644 --- a/yql/essentials/public/purecalc/common/transformations/extract_used_columns.cpp +++ b/yql/essentials/public/purecalc/common/transformations/extract_used_columns.cpp @@ -9,88 +9,86 @@ using namespace NYql; using namespace NYql::NPureCalc; namespace { - class TUsedColumnsExtractor : public TSyncTransformerBase { - private: - TVector<THashSet<TString>>* const Destination_; - const TVector<THashSet<TString>>& AllColumns_; - TString NodeName_; - - bool CalculatedUsedFields_ = false; - - public: - TUsedColumnsExtractor( - TVector<THashSet<TString>>* destination, - const TVector<THashSet<TString>>& allColumns, - TString nodeName - ) - : Destination_(destination) - , AllColumns_(allColumns) - , NodeName_(std::move(nodeName)) - { +class TUsedColumnsExtractor: public TSyncTransformerBase { +private: + TVector<THashSet<TString>>* const Destination_; + const TVector<THashSet<TString>>& AllColumns_; + TString NodeName_; + + bool CalculatedUsedFields_ = false; + +public: + TUsedColumnsExtractor( + TVector<THashSet<TString>>* destination, + const TVector<THashSet<TString>>& allColumns, + TString nodeName) + : Destination_(destination) + , AllColumns_(allColumns) + , NodeName_(std::move(nodeName)) + { + } + + TUsedColumnsExtractor(TVector<THashSet<TString>>*, TVector<THashSet<TString>>&&, TString) = delete; + +public: + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + output = input; + + if (CalculatedUsedFields_) { + return IGraphTransformer::TStatus::Ok; } - TUsedColumnsExtractor(TVector<THashSet<TString>>*, TVector<THashSet<TString>>&&, TString) = delete; - - public: - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - output = input; + bool hasError = false; - if (CalculatedUsedFields_) { - return IGraphTransformer::TStatus::Ok; - } + *Destination_ = AllColumns_; - bool hasError = false; - - *Destination_ = AllColumns_; - - VisitExpr(input, [&](const TExprNode::TPtr& inputExpr) { - NNodes::TExprBase node(inputExpr); - if (auto maybeExtract = node.Maybe<NNodes::TCoExtractMembers>()) { - auto extract = maybeExtract.Cast(); - const auto& arg = extract.Input().Ref(); - if (arg.IsCallable(NodeName_)) { - ui32 inputIndex; - if (!TryFetchInputIndexFromSelf(arg, ctx, AllColumns_.size(), inputIndex)) { - hasError = true; - return false; - } - - YQL_ENSURE(inputIndex < AllColumns_.size()); - - auto& destinationColumnsSet = (*Destination_)[inputIndex]; - const auto& allColumnsSet = AllColumns_[inputIndex]; - - destinationColumnsSet.clear(); - for (const auto& columnAtom : extract.Members()) { - TString name = TString(columnAtom.Value()); - YQL_ENSURE(allColumnsSet.contains(name), "unexpected column in the input struct"); - destinationColumnsSet.insert(name); - } + VisitExpr(input, [&](const TExprNode::TPtr& inputExpr) { + NNodes::TExprBase node(inputExpr); + if (auto maybeExtract = node.Maybe<NNodes::TCoExtractMembers>()) { + auto extract = maybeExtract.Cast(); + const auto& arg = extract.Input().Ref(); + if (arg.IsCallable(NodeName_)) { + ui32 inputIndex; + if (!TryFetchInputIndexFromSelf(arg, ctx, AllColumns_.size(), inputIndex)) { + hasError = true; + return false; } - } - return true; - }); + YQL_ENSURE(inputIndex < AllColumns_.size()); + + auto& destinationColumnsSet = (*Destination_)[inputIndex]; + const auto& allColumnsSet = AllColumns_[inputIndex]; - if (hasError) { - return IGraphTransformer::TStatus::Error; + destinationColumnsSet.clear(); + for (const auto& columnAtom : extract.Members()) { + TString name = TString(columnAtom.Value()); + YQL_ENSURE(allColumnsSet.contains(name), "unexpected column in the input struct"); + destinationColumnsSet.insert(name); + } + } } - CalculatedUsedFields_ = true; + return true; + }); - return IGraphTransformer::TStatus::Ok; + if (hasError) { + return IGraphTransformer::TStatus::Error; } - void Rewind() final { - CalculatedUsedFields_ = false; - } - }; -} + CalculatedUsedFields_ = true; + + return IGraphTransformer::TStatus::Ok; + } + + void Rewind() final { + CalculatedUsedFields_ = false; + } +}; +} // namespace TAutoPtr<IGraphTransformer> NYql::NPureCalc::MakeUsedColumnsExtractor( TVector<THashSet<TString>>* destination, const TVector<THashSet<TString>>& allColumns, - const TString& nodeName -) { + const TString& nodeName) { return new TUsedColumnsExtractor(destination, allColumns, nodeName); } diff --git a/yql/essentials/public/purecalc/common/transformations/extract_used_columns.h b/yql/essentials/public/purecalc/common/transformations/extract_used_columns.h index d0850e28b59..09f82d04e6d 100644 --- a/yql/essentials/public/purecalc/common/transformations/extract_used_columns.h +++ b/yql/essentials/public/purecalc/common/transformations/extract_used_columns.h @@ -9,21 +9,20 @@ #include <util/generic/string.h> namespace NYql { - namespace NPureCalc { - /** - * Make transformation which builds sets of input columns from the given expression. - * - * @param destination a vector of string sets which will be populated with column names sets when - * transformation pipeline is launched. This pointer should contain a valid - * TVector<THashSet> instance. The transformation will overwrite its contents. - * @param allColumns vector of sets with all available columns for each input. - * @param nodeName name of the callable used to get input data, e.g. `Self`. - * @return an extractor which scans an input structs contents and populates destination. - */ - TAutoPtr<IGraphTransformer> MakeUsedColumnsExtractor( - TVector<THashSet<TString>>* destination, - const TVector<THashSet<TString>>& allColumns, - const TString& nodeName = TString{PurecalcInputCallableName} - ); - } -} +namespace NPureCalc { +/** + * Make transformation which builds sets of input columns from the given expression. + * + * @param destination a vector of string sets which will be populated with column names sets when + * transformation pipeline is launched. This pointer should contain a valid + * TVector<THashSet> instance. The transformation will overwrite its contents. + * @param allColumns vector of sets with all available columns for each input. + * @param nodeName name of the callable used to get input data, e.g. `Self`. + * @return an extractor which scans an input structs contents and populates destination. + */ +TAutoPtr<IGraphTransformer> MakeUsedColumnsExtractor( + TVector<THashSet<TString>>* destination, + const TVector<THashSet<TString>>& allColumns, + const TString& nodeName = TString{PurecalcInputCallableName}); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/transformations/output_columns_filter.cpp b/yql/essentials/public/purecalc/common/transformations/output_columns_filter.cpp index 04181db7c83..38dd2f074ba 100644 --- a/yql/essentials/public/purecalc/common/transformations/output_columns_filter.cpp +++ b/yql/essentials/public/purecalc/common/transformations/output_columns_filter.cpp @@ -6,94 +6,102 @@ using namespace NYql; using namespace NYql::NPureCalc; namespace { - class TOutputColumnsFilter: public TSyncTransformerBase { - private: - TMaybe<THashSet<TString>> Filter_; - bool Fired_; - - public: - explicit TOutputColumnsFilter(TMaybe<THashSet<TString>> filter) - : Filter_(std::move(filter)) - , Fired_(false) - { +class TOutputColumnsFilter: public TSyncTransformerBase { +private: + TMaybe<THashSet<TString>> Filter_; + bool Fired_; + +public: + explicit TOutputColumnsFilter(TMaybe<THashSet<TString>> filter) + : Filter_(std::move(filter)) + , Fired_(false) + { + } + +public: + void Rewind() override { + Fired_ = false; + } + + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + output = input; + + if (Fired_ || Filter_.Empty()) { + return IGraphTransformer::TStatus::Ok; } - public: - void Rewind() override { - Fired_ = false; + const TTypeAnnotationNode* returnType = output->GetTypeAnn(); + const TTypeAnnotationNode* returnItemType = nullptr; + switch (returnType->GetKind()) { + case ETypeAnnotationKind::Stream: + returnItemType = returnType->Cast<TStreamExprType>()->GetItemType(); + break; + case ETypeAnnotationKind::List: + returnItemType = returnType->Cast<TListExprType>()->GetItemType(); + break; + default: + Y_ABORT("unexpected return type"); } - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - output = input; - - if (Fired_ || Filter_.Empty()) { - return IGraphTransformer::TStatus::Ok; - } - - const TTypeAnnotationNode* returnType = output->GetTypeAnn(); - const TTypeAnnotationNode* returnItemType = nullptr; - switch (returnType->GetKind()) { - case ETypeAnnotationKind::Stream: - returnItemType = returnType->Cast<TStreamExprType>()->GetItemType(); - break; - case ETypeAnnotationKind::List: - returnItemType = returnType->Cast<TListExprType>()->GetItemType(); - break; - default: - Y_ABORT("unexpected return type"); - } - - if (returnItemType->GetKind() != ETypeAnnotationKind::Struct) { - ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), "columns filter only supported for single-output programs")); - } + if (returnItemType->GetKind() != ETypeAnnotationKind::Struct) { + ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), "columns filter only supported for single-output programs")); + } - const auto* returnItemStruct = returnItemType->Cast<TStructExprType>(); - - auto arg = ctx.NewArgument(TPositionHandle(), "row"); - TExprNode::TListType asStructItems; - for (const auto& x : returnItemStruct->GetItems()) { - TExprNode::TPtr value; - if (Filter_->contains(x->GetName())) { - value = ctx.Builder({}) - .Callable("Member") - .Add(0, arg) - .Atom(1, x->GetName()) - .Seal() - .Build(); - } else { - auto type = x->GetItemType(); - value = ctx.Builder({}) - .Callable(type->GetKind() == ETypeAnnotationKind::Optional ? "Nothing" : "Default") - .Add(0, ExpandType({}, *type, ctx)) - .Seal() - .Build(); - } - - auto item = ctx.Builder({}) - .List() - .Atom(0, x->GetName()) - .Add(1, value) - .Seal() - .Build(); - - asStructItems.push_back(item); + const auto* returnItemStruct = returnItemType->Cast<TStructExprType>(); + + auto arg = ctx.NewArgument(TPositionHandle(), "row"); + TExprNode::TListType asStructItems; + for (const auto& x : returnItemStruct->GetItems()) { + TExprNode::TPtr value; + if (Filter_->contains(x->GetName())) { + // clang-format off + value = ctx.Builder({}) + .Callable("Member") + .Add(0, arg) + .Atom(1, x->GetName()) + .Seal() + .Build(); + // clang-format on + } else { + auto type = x->GetItemType(); + // clang-format off + value = ctx.Builder({}) + .Callable(type->GetKind() == ETypeAnnotationKind::Optional ? "Nothing" : "Default") + .Add(0, ExpandType({}, *type, ctx)) + .Seal() + .Build(); + // clang-format on } - auto body = ctx.NewCallable(TPositionHandle(), "AsStruct", std::move(asStructItems)); - auto lambda = ctx.NewLambda(TPositionHandle(), ctx.NewArguments(TPositionHandle(), {arg}), std::move(body)); - output = ctx.Builder(TPositionHandle()) - .Callable("Map") - .Add(0, output) - .Add(1, lambda) - .Seal() - .Build(); - - Fired_ = true; + // clang-format off + auto item = ctx.Builder({}) + .List() + .Atom(0, x->GetName()) + .Add(1, value) + .Seal() + .Build(); + // clang-format on - return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); + asStructItems.push_back(item); } - }; -} + + auto body = ctx.NewCallable(TPositionHandle(), "AsStruct", std::move(asStructItems)); + auto lambda = ctx.NewLambda(TPositionHandle(), ctx.NewArguments(TPositionHandle(), {arg}), std::move(body)); + // clang-format off + output = ctx.Builder(TPositionHandle()) + .Callable("Map") + .Add(0, output) + .Add(1, lambda) + .Seal() + .Build(); + // clang-format on + + Fired_ = true; + + return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true); + } +}; +} // namespace TAutoPtr<IGraphTransformer> NYql::NPureCalc::MakeOutputColumnsFilter(const TMaybe<THashSet<TString>>& columns) { return new TOutputColumnsFilter(columns); diff --git a/yql/essentials/public/purecalc/common/transformations/output_columns_filter.h b/yql/essentials/public/purecalc/common/transformations/output_columns_filter.h index 85302d82feb..235f02a8379 100644 --- a/yql/essentials/public/purecalc/common/transformations/output_columns_filter.h +++ b/yql/essentials/public/purecalc/common/transformations/output_columns_filter.h @@ -6,13 +6,13 @@ #include <yql/essentials/core/yql_type_annotation.h> namespace NYql { - namespace NPureCalc { - /** - * A transformer which removes unwanted columns from output. - * - * @param columns remove all columns that are not in this set. - * @return a graph transformer for filtering output. - */ - TAutoPtr<IGraphTransformer> MakeOutputColumnsFilter(const TMaybe<THashSet<TString>>& columns); - } -} +namespace NPureCalc { +/** + * A transformer which removes unwanted columns from output. + * + * @param columns remove all columns that are not in this set. + * @return a graph transformer for filtering output. + */ +TAutoPtr<IGraphTransformer> MakeOutputColumnsFilter(const TMaybe<THashSet<TString>>& columns); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/transformations/replace_table_reads.cpp b/yql/essentials/public/purecalc/common/transformations/replace_table_reads.cpp index 141e92baf28..ae160ee3e91 100644 --- a/yql/essentials/public/purecalc/common/transformations/replace_table_reads.cpp +++ b/yql/essentials/public/purecalc/common/transformations/replace_table_reads.cpp @@ -10,238 +10,242 @@ using namespace NYql; using namespace NYql::NPureCalc; namespace { - class TTableReadsReplacer: public TSyncTransformerBase { - private: - const TVector<const TStructExprType*>& InputStructs_; - bool UseSystemColumns_; - EProcessorMode ProcessorMode_; - TString CallableName_; - TString TablePrefix_; - bool Complete_ = false; - - public: - explicit TTableReadsReplacer( - const TVector<const TStructExprType*>& inputStructs, - bool useSystemColumns, - EProcessorMode processorMode, - TString inputNodeName, - TString tablePrefix - ) - : InputStructs_(inputStructs) - , UseSystemColumns_(useSystemColumns) - , ProcessorMode_(processorMode) - , CallableName_(std::move(inputNodeName)) - , TablePrefix_(std::move(tablePrefix)) - { +class TTableReadsReplacer: public TSyncTransformerBase { +private: + const TVector<const TStructExprType*>& InputStructs_; + bool UseSystemColumns_; + EProcessorMode ProcessorMode_; + TString CallableName_; + TString TablePrefix_; + bool Complete_ = false; + +public: + explicit TTableReadsReplacer( + const TVector<const TStructExprType*>& inputStructs, + bool useSystemColumns, + EProcessorMode processorMode, + TString inputNodeName, + TString tablePrefix) + : InputStructs_(inputStructs) + , UseSystemColumns_(useSystemColumns) + , ProcessorMode_(processorMode) + , CallableName_(std::move(inputNodeName)) + , TablePrefix_(std::move(tablePrefix)) + { + } + + TTableReadsReplacer(TVector<const TStructExprType*>&&, TString, TString) = delete; + +public: + TStatus DoTransform(const TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + output = input; + if (Complete_) { + return TStatus::Ok; } - TTableReadsReplacer(TVector<const TStructExprType*>&&, TString, TString) = delete; - - public: - TStatus DoTransform(const TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - output = input; - if (Complete_) { - return TStatus::Ok; - } - - TOptimizeExprSettings settings(nullptr); + TOptimizeExprSettings settings(nullptr); - auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { - if (node->IsCallable(NNodes::TCoRight::CallableName())) { - TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return new TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); - }); + auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + if (node->IsCallable(NNodes::TCoRight::CallableName())) { + TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { + return new TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); + }); - if (!EnsureMinArgsCount(*node, 1, ctx)) { - return nullptr; - } - - if (node->Child(0)->IsCallable(NNodes::TCoCons::CallableName())) { - return node; - } - - if (!node->Child(0)->IsCallable(NNodes::TCoRead::CallableName())) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(0)->Pos()), TStringBuilder() << "Expected Read!")); - return nullptr; - } + if (!EnsureMinArgsCount(*node, 1, ctx)) { + return nullptr; + } - return BuildInputFromRead(node->Pos(), node->ChildPtr(0), ctx); - } else if (node->IsCallable(NNodes::TCoLeft::CallableName())) { - TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return new TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); - }); + if (node->Child(0)->IsCallable(NNodes::TCoCons::CallableName())) { + return node; + } - if (!EnsureMinArgsCount(*node, 1, ctx)) { - return nullptr; - } + if (!node->Child(0)->IsCallable(NNodes::TCoRead::CallableName())) { + ctx.AddError(TIssue(ctx.GetPosition(node->Child(0)->Pos()), TStringBuilder() << "Expected Read!")); + return nullptr; + } - if (!node->Child(0)->IsCallable(NNodes::TCoRead::CallableName())) { - ctx.AddError(TIssue(ctx.GetPosition(node->Child(0)->Pos()), TStringBuilder() << "Expected Read!")); - return nullptr; - } + return BuildInputFromRead(node->Pos(), node->ChildPtr(0), ctx); + } else if (node->IsCallable(NNodes::TCoLeft::CallableName())) { + TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { + return new TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); + }); - return node->Child(0)->HeadPtr(); + if (!EnsureMinArgsCount(*node, 1, ctx)) { + return nullptr; } - return node; - }, ctx, settings); + if (!node->Child(0)->IsCallable(NNodes::TCoRead::CallableName())) { + ctx.AddError(TIssue(ctx.GetPosition(node->Child(0)->Pos()), TStringBuilder() << "Expected Read!")); + return nullptr; + } - if (status.Level == TStatus::Ok) { - Complete_ = true; + return node->Child(0)->HeadPtr(); } - return status; - } - - void Rewind() override { - Complete_ = false; - } - private: - TExprNode::TPtr BuildInputFromRead(TPositionHandle replacePos, const TExprNode::TPtr& node, TExprContext& ctx) { - TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); - }); + return node; + }, ctx, settings); - if (!EnsureMinArgsCount(*node, 3, ctx)) { - return nullptr; - } + if (status.Level == TStatus::Ok) { + Complete_ = true; + } + return status; + } - const auto source = node->ChildPtr(2); - if (source->IsCallable(NNodes::TCoKey::CallableName())) { - return BuildInputFromKey(replacePos, source, ctx); - } - if (source->IsCallable("DataTables")) { - return BuildInputFromDataTables(replacePos, source, ctx); - } + void Rewind() override { + Complete_ = false; + } - ctx.AddError(TIssue(ctx.GetPosition(source->Pos()), TStringBuilder() << "Unsupported read source: " << source->Content())); +private: + TExprNode::TPtr BuildInputFromRead(TPositionHandle replacePos, const TExprNode::TPtr& node, TExprContext& ctx) { + TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { + return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); + }); + if (!EnsureMinArgsCount(*node, 3, ctx)) { return nullptr; } - TExprNode::TPtr BuildInputFromKey(TPositionHandle replacePos, const TExprNode::TPtr& node, TExprContext& ctx) { - TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); - }); + const auto source = node->ChildPtr(2); + if (source->IsCallable(NNodes::TCoKey::CallableName())) { + return BuildInputFromKey(replacePos, source, ctx); + } + if (source->IsCallable("DataTables")) { + return BuildInputFromDataTables(replacePos, source, ctx); + } - ui32 inputIndex; - TExprNode::TPtr inputTableName; + ctx.AddError(TIssue(ctx.GetPosition(source->Pos()), TStringBuilder() << "Unsupported read source: " << source->Content())); - if (!TryFetchInputIndexFromKey(node, ctx, inputIndex, inputTableName)) { - return nullptr; - } + return nullptr; + } - YQL_ENSURE(inputTableName->IsCallable(NNodes::TCoString::CallableName())); + TExprNode::TPtr BuildInputFromKey(TPositionHandle replacePos, const TExprNode::TPtr& node, TExprContext& ctx) { + TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { + return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); + }); - auto inputNode = ctx.Builder(replacePos) - .Callable(CallableName_) - .Atom(0, ToString(inputIndex)) - .Seal() - .Build(); + ui32 inputIndex; + TExprNode::TPtr inputTableName; - if (inputNode->IsCallable(PurecalcBlockInputCallableName)) { - const auto inputStruct = InputStructs_[inputIndex]->Cast<TStructExprType>(); - const auto blocksLambda = NodeFromBlocks(replacePos, inputStruct, ctx); - bool wrapLMap = ProcessorMode_ == EProcessorMode::PullList; - inputNode = ApplyToIterable(replacePos, inputNode, blocksLambda, wrapLMap, ctx); - } + if (!TryFetchInputIndexFromKey(node, ctx, inputIndex, inputTableName)) { + return nullptr; + } - if (UseSystemColumns_) { - auto mapLambda = ctx.Builder(replacePos) - .Lambda() - .Param("row") - .Callable(0, NNodes::TCoAddMember::CallableName()) - .Arg(0, "row") - .Atom(1, PurecalcSysColumnTablePath) - .Add(2, inputTableName) - .Seal() - .Seal() - .Build(); + YQL_ENSURE(inputTableName->IsCallable(NNodes::TCoString::CallableName())); + + // clang-format off + auto inputNode = ctx.Builder(replacePos) + .Callable(CallableName_) + .Atom(0, ToString(inputIndex)) + .Seal() + .Build(); + // clang-format on + + if (inputNode->IsCallable(PurecalcBlockInputCallableName)) { + const auto inputStruct = InputStructs_[inputIndex]->Cast<TStructExprType>(); + const auto blocksLambda = NodeFromBlocks(replacePos, inputStruct, ctx); + bool wrapLMap = ProcessorMode_ == EProcessorMode::PullList; + inputNode = ApplyToIterable(replacePos, inputNode, blocksLambda, wrapLMap, ctx); + } - return ctx.Builder(replacePos) - .Callable(NNodes::TCoMap::CallableName()) - .Add(0, std::move(inputNode)) - .Add(1, std::move(mapLambda)) + if (UseSystemColumns_) { + // clang-format off + auto mapLambda = ctx.Builder(replacePos) + .Lambda() + .Param("row") + .Callable(0, NNodes::TCoAddMember::CallableName()) + .Arg(0, "row") + .Atom(1, PurecalcSysColumnTablePath) + .Add(2, inputTableName) .Seal() - .Build(); - } + .Seal() + .Build(); + // clang-format on - return inputNode; + // clang-format off + return ctx.Builder(replacePos) + .Callable(NNodes::TCoMap::CallableName()) + .Add(0, std::move(inputNode)) + .Add(1, std::move(mapLambda)) + .Seal() + .Build(); + // clang-format on } - TExprNode::TPtr BuildInputFromDataTables(TPositionHandle replacePos, const TExprNode::TPtr& node, TExprContext& ctx) { - TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); - }); + return inputNode; + } - if (InputStructs_.empty()) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "No inputs provided by input spec")); - return nullptr; - } + TExprNode::TPtr BuildInputFromDataTables(TPositionHandle replacePos, const TExprNode::TPtr& node, TExprContext& ctx) { + TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { + return MakeIntrusive<TIssue>(ctx.GetPosition(node->Pos()), TStringBuilder() << "At function: " << node->Content()); + }); - if (!EnsureArgsCount(*node, 0, ctx)) { - return nullptr; - } + if (InputStructs_.empty()) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "No inputs provided by input spec")); + return nullptr; + } - auto builder = ctx.Builder(replacePos); + if (!EnsureArgsCount(*node, 0, ctx)) { + return nullptr; + } - if (InputStructs_.size() > 1) { - auto listBuilder = builder.List(); + auto builder = ctx.Builder(replacePos); - for (ui32 i = 0; i < InputStructs_.size(); ++i) { - listBuilder.Callable(i, CallableName_).Atom(0, ToString(i)).Seal(); - } + if (InputStructs_.size() > 1) { + auto listBuilder = builder.List(); - return listBuilder.Seal().Build(); + for (ui32 i = 0; i < InputStructs_.size(); ++i) { + listBuilder.Callable(i, CallableName_).Atom(0, ToString(i)).Seal(); } - return builder.Callable(CallableName_).Atom(0, "0").Seal().Build(); + return listBuilder.Seal().Build(); } - bool TryFetchInputIndexFromKey(const TExprNode::TPtr& node, TExprContext& ctx, ui32& resultIndex, TExprNode::TPtr& resultTableName) { - if (!EnsureArgsCount(*node, 1, ctx)) { - return false; - } + return builder.Callable(CallableName_).Atom(0, "0").Seal().Build(); + } - const auto* keyArg = node->Child(0); - if (!keyArg->IsList() || keyArg->ChildrenSize() != 2 || !keyArg->Child(0)->IsAtom("table") || - !keyArg->Child(1)->IsCallable(NNodes::TCoString::CallableName())) - { - ctx.AddError(TIssue(ctx.GetPosition(keyArg->Pos()), "Expected single table name")); - return false; - } + bool TryFetchInputIndexFromKey(const TExprNode::TPtr& node, TExprContext& ctx, ui32& resultIndex, TExprNode::TPtr& resultTableName) { + if (!EnsureArgsCount(*node, 1, ctx)) { + return false; + } - resultTableName = keyArg->ChildPtr(1); + const auto* keyArg = node->Child(0); + if (!keyArg->IsList() || keyArg->ChildrenSize() != 2 || !keyArg->Child(0)->IsAtom("table") || + !keyArg->Child(1)->IsCallable(NNodes::TCoString::CallableName())) + { + ctx.AddError(TIssue(ctx.GetPosition(keyArg->Pos()), "Expected single table name")); + return false; + } - auto tableName = resultTableName->Child(0)->Content(); + resultTableName = keyArg->ChildPtr(1); - if (!tableName.StartsWith(TablePrefix_)) { - ctx.AddError(TIssue(ctx.GetPosition(resultTableName->Child(0)->Pos()), - TStringBuilder() << "Invalid table name " << TString{tableName}.Quote() << ": prefix must be " << TablePrefix_.Quote())); - return false; - } + auto tableName = resultTableName->Child(0)->Content(); - tableName.SkipPrefix(TablePrefix_); + if (!tableName.StartsWith(TablePrefix_)) { + ctx.AddError(TIssue(ctx.GetPosition(resultTableName->Child(0)->Pos()), + TStringBuilder() << "Invalid table name " << TString{tableName}.Quote() << ": prefix must be " << TablePrefix_.Quote())); + return false; + } - if (!tableName) { - resultIndex = 0; - } else if (!TryFromString(tableName, resultIndex)) { - ctx.AddError(TIssue(ctx.GetPosition(resultTableName->Child(0)->Pos()), - TStringBuilder() << "Invalid table name " << TString{tableName}.Quote() << ": suffix must be UI32 number")); - return false; - } + tableName.SkipPrefix(TablePrefix_); - return true; + if (!tableName) { + resultIndex = 0; + } else if (!TryFromString(tableName, resultIndex)) { + ctx.AddError(TIssue(ctx.GetPosition(resultTableName->Child(0)->Pos()), + TStringBuilder() << "Invalid table name " << TString{tableName}.Quote() << ": suffix must be UI32 number")); + return false; } - }; -} + + return true; + } +}; +} // namespace TAutoPtr<IGraphTransformer> NYql::NPureCalc::MakeTableReadsReplacer( const TVector<const TStructExprType*>& inputStructs, bool useSystemColumns, EProcessorMode processorMode, TString callableName, - TString tablePrefix -) { + TString tablePrefix) { return new TTableReadsReplacer(inputStructs, useSystemColumns, processorMode, std::move(callableName), std::move(tablePrefix)); } diff --git a/yql/essentials/public/purecalc/common/transformations/replace_table_reads.h b/yql/essentials/public/purecalc/common/transformations/replace_table_reads.h index 33bc7174ac4..750ed8b37fb 100644 --- a/yql/essentials/public/purecalc/common/transformations/replace_table_reads.h +++ b/yql/essentials/public/purecalc/common/transformations/replace_table_reads.h @@ -6,25 +6,24 @@ #include <yql/essentials/core/yql_graph_transformer.h> namespace NYql::NPureCalc { - /** - * SQL translation would generate a standard Read! call to read each input table. It will than generate - * a Right! call to get the table data from a tuple returned by Read!. This transformation replaces any Right! - * call with a call to special function used to get input data. - * - * Each table name must starts with the specified prefix and ends with an index of program input (e.g. `Input0`). - * Name without numeric suffix is an alias for the first input. - * - * @param inputStructs types of each input. - * @param useSystemColumns whether to allow special system columns in input structs. - * @param callableName name of the special callable used to get input data (e.g. `Self`). - * @param tablePrefix required prefix for all table names (e.g. `Input`). - * @param return a graph transformer for replacing table reads. - */ - TAutoPtr<IGraphTransformer> MakeTableReadsReplacer( - const TVector<const TStructExprType*>& inputStructs, - bool useSystemColumns, - EProcessorMode processorMode, - TString callableName = TString{PurecalcInputCallableName}, - TString tablePrefix = TString{PurecalcInputTablePrefix} - ); -} +/** + * SQL translation would generate a standard Read! call to read each input table. It will than generate + * a Right! call to get the table data from a tuple returned by Read!. This transformation replaces any Right! + * call with a call to special function used to get input data. + * + * Each table name must starts with the specified prefix and ends with an index of program input (e.g. `Input0`). + * Name without numeric suffix is an alias for the first input. + * + * @param inputStructs types of each input. + * @param useSystemColumns whether to allow special system columns in input structs. + * @param callableName name of the special callable used to get input data (e.g. `Self`). + * @param tablePrefix required prefix for all table names (e.g. `Input`). + * @param return a graph transformer for replacing table reads. + */ +TAutoPtr<IGraphTransformer> MakeTableReadsReplacer( + const TVector<const TStructExprType*>& inputStructs, + bool useSystemColumns, + EProcessorMode processorMode, + TString callableName = TString{PurecalcInputCallableName}, + TString tablePrefix = TString{PurecalcInputTablePrefix}); +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/transformations/root_to_blocks.cpp b/yql/essentials/public/purecalc/common/transformations/root_to_blocks.cpp index 07c959d1077..67f6f59a1ef 100644 --- a/yql/essentials/public/purecalc/common/transformations/root_to_blocks.cpp +++ b/yql/essentials/public/purecalc/common/transformations/root_to_blocks.cpp @@ -59,7 +59,6 @@ public: TAutoPtr<IGraphTransformer> NYql::NPureCalc::MakeRootToBlocks( bool acceptsBlocks, - EProcessorMode processorMode -) { + EProcessorMode processorMode) { return new TRootToBlocks(acceptsBlocks, processorMode); } diff --git a/yql/essentials/public/purecalc/common/transformations/root_to_blocks.h b/yql/essentials/public/purecalc/common/transformations/root_to_blocks.h index 13a7a9dfc11..46ff25d2051 100644 --- a/yql/essentials/public/purecalc/common/transformations/root_to_blocks.h +++ b/yql/essentials/public/purecalc/common/transformations/root_to_blocks.h @@ -5,18 +5,17 @@ #include <yql/essentials/core/yql_graph_transformer.h> namespace NYql { - namespace NPureCalc { - /** - * A transformer which rewrite the root to respect block types. - * - * @param acceptsBlock allows using this transformer in pipeline and - * skip this phase if no block output is required. - * @param processorMode specifies the top-most container of the result. - * @return a graph transformer for rewriting the root node. - */ - TAutoPtr<IGraphTransformer> MakeRootToBlocks( - bool acceptsBlocks, - EProcessorMode processorMode - ); - } -} +namespace NPureCalc { +/** + * A transformer which rewrite the root to respect block types. + * + * @param acceptsBlock allows using this transformer in pipeline and + * skip this phase if no block output is required. + * @param processorMode specifies the top-most container of the result. + * @return a graph transformer for rewriting the root node. + */ +TAutoPtr<IGraphTransformer> MakeRootToBlocks( + bool acceptsBlocks, + EProcessorMode processorMode); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/transformations/type_annotation.cpp b/yql/essentials/public/purecalc/common/transformations/type_annotation.cpp index 63105cb8878..61f14fcc65a 100644 --- a/yql/essentials/public/purecalc/common/transformations/type_annotation.cpp +++ b/yql/essentials/public/purecalc/common/transformations/type_annotation.cpp @@ -14,171 +14,144 @@ using namespace NYql; using namespace NYql::NPureCalc; namespace { - class TTypeAnnotatorBase: public TSyncTransformerBase { - public: - using THandler = std::function<TStatus(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)>; +class TTypeAnnotatorBase: public TSyncTransformerBase { +public: + using THandler = std::function<TStatus(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)>; - TTypeAnnotatorBase(TTypeAnnotationContextPtr typeAnnotationContext) - { - OriginalTransformer_.reset(CreateExtCallableTypeAnnotationTransformer(*typeAnnotationContext).Release()); - } + TTypeAnnotatorBase(TTypeAnnotationContextPtr typeAnnotationContext) + { + OriginalTransformer_.reset(CreateExtCallableTypeAnnotationTransformer(*typeAnnotationContext).Release()); + } - TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { - if (input->Type() == TExprNode::Callable) { - if (auto handler = Handlers_.FindPtr(input->Content())) { - return (*handler)(input, output, ctx); - } + TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final { + if (input->Type() == TExprNode::Callable) { + if (auto handler = Handlers_.FindPtr(input->Content())) { + return (*handler)(input, output, ctx); } - - auto status = OriginalTransformer_->Transform(input, output, ctx); - - YQL_ENSURE(status.Level != IGraphTransformer::TStatus::Async, "Async type check is not supported"); - - return status; } - void Rewind() final { - OriginalTransformer_->Rewind(); - } + auto status = OriginalTransformer_->Transform(input, output, ctx); - protected: - void AddHandler(std::initializer_list<TStringBuf> names, THandler handler) { - for (auto name: names) { - YQL_ENSURE(Handlers_.emplace(name, handler).second, "Duplicate handler for " << name); - } - } + YQL_ENSURE(status.Level != IGraphTransformer::TStatus::Async, "Async type check is not supported"); - template <class TDerived> - THandler Hndl(TStatus(TDerived::* handler)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)) { - return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) { - return (static_cast<TDerived*>(this)->*handler)(input, output, ctx); - }; - } - - template <class TDerived> - THandler Hndl(TStatus(TDerived::* handler)(const TExprNode::TPtr&, TExprContext&)) { - return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) { - return (static_cast<TDerived*>(this)->*handler)(input, ctx); - }; - } - - private: - std::shared_ptr<IGraphTransformer> OriginalTransformer_; - THashMap<TStringBuf, THandler> Handlers_; - }; - - class TTypeAnnotator : public TTypeAnnotatorBase { - private: - TTypeAnnotationContextPtr TypeAnnotationContext_; - const TVector<const TStructExprType*>& InputStructs_; - TVector<const TStructExprType*>& RawInputTypes_; - EProcessorMode ProcessorMode_; - TString InputNodeName_; - - public: - TTypeAnnotator( - TTypeAnnotationContextPtr typeAnnotationContext, - const TVector<const TStructExprType*>& inputStructs, - TVector<const TStructExprType*>& rawInputTypes, - EProcessorMode processorMode, - TString nodeName - ) - : TTypeAnnotatorBase(typeAnnotationContext) - , TypeAnnotationContext_(typeAnnotationContext) - , InputStructs_(inputStructs) - , RawInputTypes_(rawInputTypes) - , ProcessorMode_(processorMode) - , InputNodeName_(std::move(nodeName)) - { - AddHandler({InputNodeName_}, Hndl(&TTypeAnnotator::HandleInputNode)); - AddHandler({NNodes::TCoTableName::CallableName()}, Hndl(&TTypeAnnotator::HandleTableName)); - AddHandler({NNodes::TCoTablePath::CallableName()}, Hndl(&TTypeAnnotator::HandleTablePath)); - AddHandler({NNodes::TCoHoppingTraits::CallableName()}, Hndl(&TTypeAnnotator::HandleHoppingTraits)); - } - - TTypeAnnotator(TTypeAnnotationContextPtr, TVector<const TStructExprType*>&&, EProcessorMode, TString) = delete; - - private: - TStatus HandleInputNode(const TExprNode::TPtr& input, TExprContext& ctx) { - ui32 inputIndex; - if (!TryFetchInputIndexFromSelf(*input, ctx, InputStructs_.size(), inputIndex)) { - return IGraphTransformer::TStatus::Error; - } + return status; + } - YQL_ENSURE(inputIndex < InputStructs_.size()); + void Rewind() final { + OriginalTransformer_->Rewind(); + } - auto itemType = InputStructs_[inputIndex]; +protected: + void AddHandler(std::initializer_list<TStringBuf> names, THandler handler) { + for (auto name : names) { + YQL_ENSURE(Handlers_.emplace(name, handler).second, "Duplicate handler for " << name); + } + } + + template <class TDerived> + THandler Hndl(TStatus (TDerived::*handler)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)) { + return [this, handler](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) { + return (static_cast<TDerived*>(this)->*handler)(input, output, ctx); + }; + } + + template <class TDerived> + THandler Hndl(TStatus (TDerived::*handler)(const TExprNode::TPtr&, TExprContext&)) { + return [this, handler](TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) { + return (static_cast<TDerived*>(this)->*handler)(input, ctx); + }; + } + +private: + std::shared_ptr<IGraphTransformer> OriginalTransformer_; + THashMap<TStringBuf, THandler> Handlers_; +}; + +class TTypeAnnotator: public TTypeAnnotatorBase { +private: + TTypeAnnotationContextPtr TypeAnnotationContext_; + const TVector<const TStructExprType*>& InputStructs_; + TVector<const TStructExprType*>& RawInputTypes_; + EProcessorMode ProcessorMode_; + TString InputNodeName_; + +public: + TTypeAnnotator( + TTypeAnnotationContextPtr typeAnnotationContext, + const TVector<const TStructExprType*>& inputStructs, + TVector<const TStructExprType*>& rawInputTypes, + EProcessorMode processorMode, + TString nodeName) + : TTypeAnnotatorBase(typeAnnotationContext) + , TypeAnnotationContext_(typeAnnotationContext) + , InputStructs_(inputStructs) + , RawInputTypes_(rawInputTypes) + , ProcessorMode_(processorMode) + , InputNodeName_(std::move(nodeName)) + { + AddHandler({InputNodeName_}, Hndl(&TTypeAnnotator::HandleInputNode)); + AddHandler({NNodes::TCoTableName::CallableName()}, Hndl(&TTypeAnnotator::HandleTableName)); + AddHandler({NNodes::TCoTablePath::CallableName()}, Hndl(&TTypeAnnotator::HandleTablePath)); + AddHandler({NNodes::TCoHoppingTraits::CallableName()}, Hndl(&TTypeAnnotator::HandleHoppingTraits)); + } + + TTypeAnnotator(TTypeAnnotationContextPtr, TVector<const TStructExprType*>&&, EProcessorMode, TString) = delete; + +private: + TStatus HandleInputNode(const TExprNode::TPtr& input, TExprContext& ctx) { + ui32 inputIndex; + if (!TryFetchInputIndexFromSelf(*input, ctx, InputStructs_.size(), inputIndex)) { + return IGraphTransformer::TStatus::Error; + } - // XXX: Tweak the input expression type, if the spec supports blocks: - // 1. Add "_yql_block_length" attribute for internal usage. - // 2. Add block container to wrap the actual item type. - if (input->IsCallable(PurecalcBlockInputCallableName)) { - itemType = WrapBlockStruct(itemType, ctx); - } + YQL_ENSURE(inputIndex < InputStructs_.size()); - RawInputTypes_[inputIndex] = itemType; + auto itemType = InputStructs_[inputIndex]; - TColumnOrder columnOrder; - for (const auto& i : itemType->GetItems()) { - columnOrder.AddColumn(TString(i->GetName())); - } + // XXX: Tweak the input expression type, if the spec supports blocks: + // 1. Add "_yql_block_length" attribute for internal usage. + // 2. Add block container to wrap the actual item type. + if (input->IsCallable(PurecalcBlockInputCallableName)) { + itemType = WrapBlockStruct(itemType, ctx); + } - if (ProcessorMode_ != EProcessorMode::PullList) { - input->SetTypeAnn(ctx.MakeType<TStreamExprType>(itemType)); - } else { - input->SetTypeAnn(ctx.MakeType<TListExprType>(itemType)); - } + RawInputTypes_[inputIndex] = itemType; - TypeAnnotationContext_->SetColumnOrder(*input, columnOrder, ctx); - return TStatus::Ok; + TColumnOrder columnOrder; + for (const auto& i : itemType->GetItems()) { + columnOrder.AddColumn(TString(i->GetName())); } - TStatus HandleTableName(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - if (!EnsureMinMaxArgsCount(*input, 1, 2, ctx)) { - return TStatus::Error; - } - - if (input->ChildrenSize() > 1) { - if (!EnsureAtom(input->Tail(), ctx)) { - return TStatus::Error; - } - - if (input->Tail().Content() != PurecalcDefaultService) { - ctx.AddError( - TIssue( - ctx.GetPosition(input->Tail().Pos()), - TStringBuilder() << "Unsupported system: " << input->Tail().Content())); - return TStatus::Error; - } - } + if (ProcessorMode_ != EProcessorMode::PullList) { + input->SetTypeAnn(ctx.MakeType<TStreamExprType>(itemType)); + } else { + input->SetTypeAnn(ctx.MakeType<TListExprType>(itemType)); + } - if (NNodes::TCoDependsOnBase::Match(&input->Head())) { - if (!EnsureArgsCount(input->Head(), 1, ctx)) { - return TStatus::Error; - } - - if (!TryBuildTableNameNode(input->Pos(), input->Head().HeadPtr(), output, ctx)) { - return TStatus::Error; - } - } else { - if (!EnsureSpecificDataType(input->Head(), EDataSlot::String, ctx)) { - return TStatus::Error; - } - output = input->HeadPtr(); - } + TypeAnnotationContext_->SetColumnOrder(*input, columnOrder, ctx); + return TStatus::Ok; + } - return TStatus::Repeat; + TStatus HandleTableName(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + if (!EnsureMinMaxArgsCount(*input, 1, 2, ctx)) { + return TStatus::Error; } - TStatus HandleTablePath(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - if (!EnsureArgsCount(*input, 1, ctx)) { + if (input->ChildrenSize() > 1) { + if (!EnsureAtom(input->Tail(), ctx)) { return TStatus::Error; } - if (!EnsureDependsOn(input->Head(), ctx)) { + if (input->Tail().Content() != PurecalcDefaultService) { + ctx.AddError( + TIssue( + ctx.GetPosition(input->Tail().Pos()), + TStringBuilder() << "Unsupported system: " << input->Tail().Content())); return TStatus::Error; } + } + if (NNodes::TCoDependsOnBase::Match(&input->Head())) { if (!EnsureArgsCount(input->Head(), 1, ctx)) { return TStatus::Error; } @@ -186,66 +159,97 @@ namespace { if (!TryBuildTableNameNode(input->Pos(), input->Head().HeadPtr(), output, ctx)) { return TStatus::Error; } + } else { + if (!EnsureSpecificDataType(input->Head(), EDataSlot::String, ctx)) { + return TStatus::Error; + } + output = input->HeadPtr(); + } + + return TStatus::Repeat; + } + + TStatus HandleTablePath(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + if (!EnsureArgsCount(*input, 1, ctx)) { + return TStatus::Error; + } + + if (!EnsureDependsOn(input->Head(), ctx)) { + return TStatus::Error; + } + if (!EnsureArgsCount(input->Head(), 1, ctx)) { + return TStatus::Error; + } + + if (!TryBuildTableNameNode(input->Pos(), input->Head().HeadPtr(), output, ctx)) { + return TStatus::Error; + } + + return TStatus::Repeat; + } + + TStatus HandleHoppingTraits(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + Y_UNUSED(output); + if (input->ChildrenSize() == 1) { + auto children = input->ChildrenList(); + // clang-format off + auto falseArg = ctx.Builder(input->Pos()) + .Atom("false") + .Seal() + .Build(); + // clang-format on + children.emplace_back(falseArg); + input->ChangeChildrenInplace(std::move(children)); return TStatus::Repeat; } - TStatus HandleHoppingTraits(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - Y_UNUSED(output); - if (input->ChildrenSize() == 1) { - auto children = input->ChildrenList(); - auto falseArg = ctx.Builder(input->Pos()) - .Atom("false") - .Seal() - .Build(); - children.emplace_back(falseArg); - input->ChangeChildrenInplace(std::move(children)); - return TStatus::Repeat; - } + return TStatus::Ok; + } - return TStatus::Ok; +private: + bool TryBuildTableNameNode( + TPositionHandle position, const TExprNode::TPtr& row, TExprNode::TPtr& result, TExprContext& ctx) + { + if (!EnsureStructType(*row, ctx)) { + return false; } - private: - bool TryBuildTableNameNode( - TPositionHandle position, const TExprNode::TPtr& row, TExprNode::TPtr& result, TExprContext& ctx) - { - if (!EnsureStructType(*row, ctx)) { - return false; - } + const auto* structType = row->GetTypeAnn()->Cast<TStructExprType>(); - const auto* structType = row->GetTypeAnn()->Cast<TStructExprType>(); - - if (auto pos = structType->FindItem(PurecalcSysColumnTablePath)) { - if (!EnsureSpecificDataType(row->Pos(), *structType->GetItems()[*pos]->GetItemType(), EDataSlot::String, ctx)) { - return false; - } - - result = ctx.Builder(position) - .Callable(NNodes::TCoMember::CallableName()) - .Add(0, row) - .Atom(1, PurecalcSysColumnTablePath) - .Seal() - .Build(); - } else { - result = ctx.Builder(position) - .Callable(NNodes::TCoString::CallableName()) - .Atom(0, "") - .Seal() - .Build(); + if (auto pos = structType->FindItem(PurecalcSysColumnTablePath)) { + if (!EnsureSpecificDataType(row->Pos(), *structType->GetItems()[*pos]->GetItemType(), EDataSlot::String, ctx)) { + return false; } - return true; + // clang-format off + result = ctx.Builder(position) + .Callable(NNodes::TCoMember::CallableName()) + .Add(0, row) + .Atom(1, PurecalcSysColumnTablePath) + .Seal() + .Build(); + // clang-format on + } else { + // clang-format off + result = ctx.Builder(position) + .Callable(NNodes::TCoString::CallableName()) + .Atom(0, "") + .Seal() + .Build(); + // clang-format on } - }; -} + + return true; + } +}; +} // namespace TAutoPtr<IGraphTransformer> NYql::NPureCalc::MakeTypeAnnotationTransformer( TTypeAnnotationContextPtr typeAnnotationContext, const TVector<const TStructExprType*>& inputStructs, TVector<const TStructExprType*>& rawInputTypes, EProcessorMode processorMode, - const TString& nodeName -) { + const TString& nodeName) { return new TTypeAnnotator(typeAnnotationContext, inputStructs, rawInputTypes, processorMode, nodeName); } diff --git a/yql/essentials/public/purecalc/common/transformations/type_annotation.h b/yql/essentials/public/purecalc/common/transformations/type_annotation.h index 87649fd231a..5ec0df2da5c 100644 --- a/yql/essentials/public/purecalc/common/transformations/type_annotation.h +++ b/yql/essentials/public/purecalc/common/transformations/type_annotation.h @@ -7,24 +7,23 @@ #include <yql/essentials/core/yql_type_annotation.h> namespace NYql { - namespace NPureCalc { - /** - * Build type annotation transformer that is aware of type of the input rows. - * - * @param typeAnnotationContext current context. - * @param inputStructs types of each input. - * @param rawInputStructs container to store the resulting input item type. - * @param processorMode current processor mode. This will affect generated input type, - * e.g. list node or struct node. - * @param nodeName name of the callable used to get input data, e.g. `Self`. - * @return a graph transformer for type annotation. - */ - TAutoPtr<IGraphTransformer> MakeTypeAnnotationTransformer( - TTypeAnnotationContextPtr typeAnnotationContext, - const TVector<const TStructExprType*>& inputStructs, - TVector<const TStructExprType*>& rawInputStructs, - EProcessorMode processorMode, - const TString& nodeName = TString{PurecalcInputCallableName} - ); - } -} +namespace NPureCalc { +/** + * Build type annotation transformer that is aware of type of the input rows. + * + * @param typeAnnotationContext current context. + * @param inputStructs types of each input. + * @param rawInputStructs container to store the resulting input item type. + * @param processorMode current processor mode. This will affect generated input type, + * e.g. list node or struct node. + * @param nodeName name of the callable used to get input data, e.g. `Self`. + * @return a graph transformer for type annotation. + */ +TAutoPtr<IGraphTransformer> MakeTypeAnnotationTransformer( + TTypeAnnotationContextPtr typeAnnotationContext, + const TVector<const TStructExprType*>& inputStructs, + TVector<const TStructExprType*>& rawInputStructs, + EProcessorMode processorMode, + const TString& nodeName = TString{PurecalcInputCallableName}); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/transformations/utils.cpp b/yql/essentials/public/purecalc/common/transformations/utils.cpp index 54e0ef7caf8..82d84187a44 100644 --- a/yql/essentials/public/purecalc/common/transformations/utils.cpp +++ b/yql/essentials/public/purecalc/common/transformations/utils.cpp @@ -9,10 +9,10 @@ using namespace NYql::NPureCalc; TExprNode::TPtr NYql::NPureCalc::NodeFromBlocks( const TPositionHandle& pos, const TStructExprType* structType, - TExprContext& ctx -) { + TExprContext& ctx) { const auto items = structType->GetItems(); Y_ENSURE(items.size() > 0); + // clang-format off return ctx.Builder(pos) .Lambda() .Param("stream") @@ -50,30 +50,31 @@ TExprNode::TPtr NYql::NPureCalc::NodeFromBlocks( .Params("fields", items.size()) .Callable("AsStruct") .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& { - ui32 i = 0; - for (const auto& item : items) { - parent.List(i) - .Atom(0, item->GetName()) - .Arg(1, "fields", i++) - .Seal(); - } - return parent; - }) + ui32 i = 0; + for (const auto& item : items) { + parent.List(i) + .Atom(0, item->GetName()) + .Arg(1, "fields", i++) + .Seal(); + } + return parent; + }) .Seal() .Seal() .Seal() .Seal() .Seal() .Build(); + // clang-format on } TExprNode::TPtr NYql::NPureCalc::NodeToBlocks( const TPositionHandle& pos, const TStructExprType* structType, - TExprContext& ctx -) { + TExprContext& ctx) { const auto items = structType->GetItems(); Y_ENSURE(items.size() > 0); + // clang-format off return ctx.Builder(pos) .Lambda() .Param("stream") @@ -126,6 +127,7 @@ TExprNode::TPtr NYql::NPureCalc::NodeToBlocks( .Seal() .Seal() .Build(); + // clang-format on } TExprNode::TPtr NYql::NPureCalc::ApplyToIterable( @@ -133,9 +135,9 @@ TExprNode::TPtr NYql::NPureCalc::ApplyToIterable( const TExprNode::TPtr iterable, const TExprNode::TPtr lambda, bool wrapLMap, - TExprContext& ctx -) { + TExprContext& ctx) { if (wrapLMap) { + // clang-format off return ctx.Builder(pos) .Callable("LMap") .Add(0, iterable) @@ -147,19 +149,21 @@ TExprNode::TPtr NYql::NPureCalc::ApplyToIterable( .Seal() .Seal() .Build(); + // clang-format on } else { + // clang-format off return ctx.Builder(pos) .Apply(lambda) .With(0, iterable) .Seal() .Build(); + // clang-format on } } const TStructExprType* NYql::NPureCalc::WrapBlockStruct( const TStructExprType* structType, - TExprContext& ctx -) { + TExprContext& ctx) { TVector<const TItemExprType*> members; for (const auto& item : structType->GetItems()) { const auto blockItemType = ctx.MakeType<TBlockExprType>(item->GetItemType()); @@ -172,8 +176,7 @@ const TStructExprType* NYql::NPureCalc::WrapBlockStruct( const TStructExprType* NYql::NPureCalc::UnwrapBlockStruct( const TStructExprType* structType, - TExprContext& ctx -) { + TExprContext& ctx) { TVector<const TItemExprType*> members; for (const auto& item : structType->GetItems()) { if (item->GetName() == PurecalcBlockColumnLength) { diff --git a/yql/essentials/public/purecalc/common/transformations/utils.h b/yql/essentials/public/purecalc/common/transformations/utils.h index cc8849b7e3a..c2ddf299ce3 100644 --- a/yql/essentials/public/purecalc/common/transformations/utils.h +++ b/yql/essentials/public/purecalc/common/transformations/utils.h @@ -3,81 +3,76 @@ #include <yql/essentials/core/yql_graph_transformer.h> namespace NYql { - namespace NPureCalc { - /** - * A transformer which wraps the given input node with the pipeline - * converting the input type to the block one. - * - * @param pos the position of the given node to be rewritten. - * @param structType the item type of the container provided by the node. - * @param ctx the context to make ExprNode rewrites. - * @return the resulting ExprNode. - */ - TExprNode::TPtr NodeFromBlocks( - const TPositionHandle& pos, - const TStructExprType* structType, - TExprContext& ctx - ); +namespace NPureCalc { +/** + * A transformer which wraps the given input node with the pipeline + * converting the input type to the block one. + * + * @param pos the position of the given node to be rewritten. + * @param structType the item type of the container provided by the node. + * @param ctx the context to make ExprNode rewrites. + * @return the resulting ExprNode. + */ +TExprNode::TPtr NodeFromBlocks( + const TPositionHandle& pos, + const TStructExprType* structType, + TExprContext& ctx); - /** - * A transformer which wraps the given root node with the pipeline - * converting the output type to the block one. - * - * @param pos the position of the given node to be rewritten. - * @param structType the item type of the container provided by the node. - * @param ctx the context to make ExprNode rewrites. - * @return the resulting ExprNode. - */ - TExprNode::TPtr NodeToBlocks( - const TPositionHandle& pos, - const TStructExprType* structType, - TExprContext& ctx - ); +/** + * A transformer which wraps the given root node with the pipeline + * converting the output type to the block one. + * + * @param pos the position of the given node to be rewritten. + * @param structType the item type of the container provided by the node. + * @param ctx the context to make ExprNode rewrites. + * @return the resulting ExprNode. + */ +TExprNode::TPtr NodeToBlocks( + const TPositionHandle& pos, + const TStructExprType* structType, + TExprContext& ctx); - /** - * A transformer to apply the given lambda to the given iterable (either - * list or stream). If the iterable is list, the lambda should be passed - * to the <LMap> callable; if the iterable is stream, the lambda should - * be applied right to the iterable. - * - * @param pos the position of the given node to be rewritten. - * @param iterable the node, that provides the iterable to be processed. - * @param lambda the node, that provides lambda to be applied. - * @param wrapLMap indicator to wrap the result with LMap callable. - * @oaram ctx the context to make ExprNode rewrites. - */ - TExprNode::TPtr ApplyToIterable( - const TPositionHandle& pos, - const TExprNode::TPtr iterable, - const TExprNode::TPtr lambda, - bool wrapLMap, - TExprContext& ctx - ); +/** + * A transformer to apply the given lambda to the given iterable (either + * list or stream). If the iterable is list, the lambda should be passed + * to the <LMap> callable; if the iterable is stream, the lambda should + * be applied right to the iterable. + * + * @param pos the position of the given node to be rewritten. + * @param iterable the node, that provides the iterable to be processed. + * @param lambda the node, that provides lambda to be applied. + * @param wrapLMap indicator to wrap the result with LMap callable. + * @oaram ctx the context to make ExprNode rewrites. + */ +TExprNode::TPtr ApplyToIterable( + const TPositionHandle& pos, + const TExprNode::TPtr iterable, + const TExprNode::TPtr lambda, + bool wrapLMap, + TExprContext& ctx); - /** - * A helper which wraps the items of the given struct with the block - * type container and appends the new item for _yql_block_length column. - * - * @param structType original struct to be wrapped. - * @param ctx the context to make ExprType rewrite. - * @return the new struct with block items. - */ - const TStructExprType* WrapBlockStruct( - const TStructExprType* structType, - TExprContext& ctx - ); +/** + * A helper which wraps the items of the given struct with the block + * type container and appends the new item for _yql_block_length column. + * + * @param structType original struct to be wrapped. + * @param ctx the context to make ExprType rewrite. + * @return the new struct with block items. + */ +const TStructExprType* WrapBlockStruct( + const TStructExprType* structType, + TExprContext& ctx); - /** - * A helper which unwraps the block container from the items of the - * given struct and removes the item for _yql_block_length column. - * - * @param structType original struct to be unwrapped. - * @param ctx the context to make ExprType rewrite. - * @return the new struct without block items. - */ - const TStructExprType* UnwrapBlockStruct( - const TStructExprType* structType, - TExprContext& ctx - ); - } -} +/** + * A helper which unwraps the block container from the items of the + * given struct and removes the item for _yql_block_length column. + * + * @param structType original struct to be unwrapped. + * @param ctx the context to make ExprType rewrite. + * @return the new struct without block items. + */ +const TStructExprType* UnwrapBlockStruct( + const TStructExprType* structType, + TExprContext& ctx); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/type_from_schema.cpp b/yql/essentials/public/purecalc/common/type_from_schema.cpp index 09a3ff50a77..184419bbc97 100644 --- a/yql/essentials/public/purecalc/common/type_from_schema.cpp +++ b/yql/essentials/public/purecalc/common/type_from_schema.cpp @@ -6,255 +6,251 @@ #include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h> namespace { - using namespace NYql; +using namespace NYql; #define REPORT(...) ctx.AddError(TIssue(TString(TStringBuilder() << __VA_ARGS__))) - bool CheckStruct(const TStructExprType* got, const TStructExprType* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { - auto status = true; - - if (expected) { - for (const auto* gotNamedItem : got->GetItems()) { - auto expectedIndex = expected->FindItem(gotNamedItem->GetName()); - if (expectedIndex) { - const auto* gotItem = gotNamedItem->GetItemType(); - const auto* expectedItem = expected->GetItems()[*expectedIndex]->GetItemType(); - - auto arg = ctx.NewArgument(TPositionHandle(), "arg"); - auto fieldConversionStatus = TryConvertTo(arg, *gotItem, *expectedItem, ctx, typeCtx); - if (fieldConversionStatus.Level == IGraphTransformer::TStatus::Error) { - status = false; - } - } else { - REPORT("Got unexpected item " << TString{gotNamedItem->GetName()}.Quote()); - status = false; - } - } +bool CheckStruct(const TStructExprType* got, const TStructExprType* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { + auto status = true; - for (const auto* expectedNamedItem : expected->GetItems()) { - if (expectedNamedItem->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) { - continue; - } - if (!got->FindItem(expectedNamedItem->GetName())) { - REPORT("Expected item " << TString{expectedNamedItem->GetName()}.Quote()); + if (expected) { + for (const auto* gotNamedItem : got->GetItems()) { + auto expectedIndex = expected->FindItem(gotNamedItem->GetName()); + if (expectedIndex) { + const auto* gotItem = gotNamedItem->GetItemType(); + const auto* expectedItem = expected->GetItems()[*expectedIndex]->GetItemType(); + + auto arg = ctx.NewArgument(TPositionHandle(), "arg"); + auto fieldConversionStatus = TryConvertTo(arg, *gotItem, *expectedItem, ctx, typeCtx); + if (fieldConversionStatus.Level == IGraphTransformer::TStatus::Error) { status = false; } + } else { + REPORT("Got unexpected item " << TString{gotNamedItem->GetName()}.Quote()); + status = false; } } - return status; - } - - bool CheckVariantContent(const TStructExprType* got, const TStructExprType* expected, TExprContext& ctx, - const TTypeAnnotationContext& typeCtx) { - auto status = true; - - if (expected) { - for (const auto* gotNamedItem : got->GetItems()) { - if (!expected->FindItem(gotNamedItem->GetName())) { - REPORT("Got unexpected alternative " << TString{gotNamedItem->GetName()}.Quote()); - status = false; - } + for (const auto* expectedNamedItem : expected->GetItems()) { + if (expectedNamedItem->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) { + continue; } - - for (const auto* expectedNamedItem : expected->GetItems()) { - if (!got->FindItem(expectedNamedItem->GetName())) { - REPORT("Expected alternative " << TString{expectedNamedItem->GetName()}.Quote()); - status = false; - } + if (!got->FindItem(expectedNamedItem->GetName())) { + REPORT("Expected item " << TString{expectedNamedItem->GetName()}.Quote()); + status = false; } } + } - for (const auto* gotNamedItem : got->GetItems()) { - const auto* gotItem = gotNamedItem->GetItemType(); - auto expectedIndex = expected ? expected->FindItem(gotNamedItem->GetName()) : Nothing(); - const auto* expectedItem = expected && expectedIndex ? expected->GetItems()[*expectedIndex]->GetItemType() : nullptr; + return status; +} - TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { - return new TIssue(TPosition(), TStringBuilder() << "Alternative " << TString{gotNamedItem->GetName()}.Quote()); - }); +bool CheckVariantContent(const TStructExprType* got, const TStructExprType* expected, TExprContext& ctx, + const TTypeAnnotationContext& typeCtx) { + auto status = true; - if (expectedItem && expectedItem->GetKind() != gotItem->GetKind()) { - REPORT("Expected to be " << expectedItem->GetKind() << ", but got " << gotItem->GetKind()); + if (expected) { + for (const auto* gotNamedItem : got->GetItems()) { + if (!expected->FindItem(gotNamedItem->GetName())) { + REPORT("Got unexpected alternative " << TString{gotNamedItem->GetName()}.Quote()); status = false; } + } - if (gotItem->GetKind() != ETypeAnnotationKind::Struct) { - REPORT("Expected to be Struct, but got " << gotItem->GetKind()); + for (const auto* expectedNamedItem : expected->GetItems()) { + if (!got->FindItem(expectedNamedItem->GetName())) { + REPORT("Expected alternative " << TString{expectedNamedItem->GetName()}.Quote()); status = false; } + } + } - const auto* gotStruct = gotItem->Cast<TStructExprType>(); - const auto* expectedStruct = expectedItem ? expectedItem->Cast<TStructExprType>() : nullptr; + for (const auto* gotNamedItem : got->GetItems()) { + const auto* gotItem = gotNamedItem->GetItemType(); + auto expectedIndex = expected ? expected->FindItem(gotNamedItem->GetName()) : Nothing(); + const auto* expectedItem = expected && expectedIndex ? expected->GetItems()[*expectedIndex]->GetItemType() : nullptr; - if (!CheckStruct(gotStruct, expectedStruct, ctx, typeCtx)) { - status = false; - } + TIssueScopeGuard issueScope(ctx.IssueManager, [&]() { + return new TIssue(TPosition(), TStringBuilder() << "Alternative " << TString{gotNamedItem->GetName()}.Quote()); + }); + + if (expectedItem && expectedItem->GetKind() != gotItem->GetKind()) { + REPORT("Expected to be " << expectedItem->GetKind() << ", but got " << gotItem->GetKind()); + status = false; } - return status; - } + if (gotItem->GetKind() != ETypeAnnotationKind::Struct) { + REPORT("Expected to be Struct, but got " << gotItem->GetKind()); + status = false; + } - bool CheckVariantContent(const TTupleExprType* got, const TTupleExprType* expected, TExprContext& ctx, - const TTypeAnnotationContext& typeCtx) { - if (expected && expected->GetSize() != got->GetSize()) { - REPORT("Expected to have " << expected->GetSize() << " alternatives, but got " << got->GetSize()); - return false; + const auto* gotStruct = gotItem->Cast<TStructExprType>(); + const auto* expectedStruct = expectedItem ? expectedItem->Cast<TStructExprType>() : nullptr; + + if (!CheckStruct(gotStruct, expectedStruct, ctx, typeCtx)) { + status = false; } + } - auto status = true; + return status; +} - for (size_t i = 0; i < got->GetSize(); i++) { - const auto* gotItem = got->GetItems()[i]; - const auto* expectedItem = expected ? expected->GetItems()[i] : nullptr; +bool CheckVariantContent(const TTupleExprType* got, const TTupleExprType* expected, TExprContext& ctx, + const TTypeAnnotationContext& typeCtx) { + if (expected && expected->GetSize() != got->GetSize()) { + REPORT("Expected to have " << expected->GetSize() << " alternatives, but got " << got->GetSize()); + return false; + } - TIssueScopeGuard issueScope(ctx.IssueManager, [i]() { - return new TIssue(TPosition(), TStringBuilder() << "Alternative #" << i); - }); + auto status = true; - if (expectedItem && expectedItem->GetKind() != gotItem->GetKind()) { - REPORT("Expected " << expectedItem->GetKind() << ", but got " << gotItem->GetKind()); - status = false; - } + for (size_t i = 0; i < got->GetSize(); i++) { + const auto* gotItem = got->GetItems()[i]; + const auto* expectedItem = expected ? expected->GetItems()[i] : nullptr; - if (gotItem->GetKind() != ETypeAnnotationKind::Struct) { - REPORT("Expected Struct, but got " << gotItem->GetKind()); - status = false; - } + TIssueScopeGuard issueScope(ctx.IssueManager, [i]() { + return new TIssue(TPosition(), TStringBuilder() << "Alternative #" << i); + }); - const auto* gotStruct = gotItem->Cast<TStructExprType>(); - const auto* expectedStruct = expectedItem ? expectedItem->Cast<TStructExprType>() : nullptr; + if (expectedItem && expectedItem->GetKind() != gotItem->GetKind()) { + REPORT("Expected " << expectedItem->GetKind() << ", but got " << gotItem->GetKind()); + status = false; + } - if (!CheckStruct(gotStruct, expectedStruct, ctx, typeCtx)) { - status = false; - } + if (gotItem->GetKind() != ETypeAnnotationKind::Struct) { + REPORT("Expected Struct, but got " << gotItem->GetKind()); + status = false; } - return status; - } + const auto* gotStruct = gotItem->Cast<TStructExprType>(); + const auto* expectedStruct = expectedItem ? expectedItem->Cast<TStructExprType>() : nullptr; - bool CheckVariant(const TVariantExprType* got, const TVariantExprType* expected, TExprContext& ctx, - const TTypeAnnotationContext& typeCtx) { - if (expected && expected->GetUnderlyingType()->GetKind() != got->GetUnderlyingType()->GetKind()) { - REPORT("Expected Variant over " << expected->GetUnderlyingType()->GetKind() << - ", but got Variant over " << got->GetUnderlyingType()->GetKind()); - return false; + if (!CheckStruct(gotStruct, expectedStruct, ctx, typeCtx)) { + status = false; } + } - switch (got->GetUnderlyingType()->GetKind()) { - case ETypeAnnotationKind::Struct: - { - const auto* gotStruct = got->GetUnderlyingType()->Cast<TStructExprType>(); - const auto* expectedStruct = expected ? expected->GetUnderlyingType()->Cast<TStructExprType>() : nullptr; - return CheckVariantContent(gotStruct, expectedStruct, ctx, typeCtx); - } - case ETypeAnnotationKind::Tuple: - { - const auto* gotTuple = got->GetUnderlyingType()->Cast<TTupleExprType>(); - const auto* expectedTuple = expected ? expected->GetUnderlyingType()->Cast<TTupleExprType>() : nullptr; - return CheckVariantContent(gotTuple, expectedTuple, ctx, typeCtx); - } - default: - Y_UNREACHABLE(); - } + return status; +} +bool CheckVariant(const TVariantExprType* got, const TVariantExprType* expected, TExprContext& ctx, + const TTypeAnnotationContext& typeCtx) { + if (expected && expected->GetUnderlyingType()->GetKind() != got->GetUnderlyingType()->GetKind()) { + REPORT("Expected Variant over " << expected->GetUnderlyingType()->GetKind() << ", but got Variant over " << got->GetUnderlyingType()->GetKind()); return false; } - bool CheckSchema(const TTypeAnnotationNode* got, const TTypeAnnotationNode* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, bool allowVariant) { - if (expected && expected->GetKind() != got->GetKind()) { - REPORT("Expected " << expected->GetKind() << ", but got " << got->GetKind()); - return false; + switch (got->GetUnderlyingType()->GetKind()) { + case ETypeAnnotationKind::Struct: { + const auto* gotStruct = got->GetUnderlyingType()->Cast<TStructExprType>(); + const auto* expectedStruct = expected ? expected->GetUnderlyingType()->Cast<TStructExprType>() : nullptr; + return CheckVariantContent(gotStruct, expectedStruct, ctx, typeCtx); } + case ETypeAnnotationKind::Tuple: { + const auto* gotTuple = got->GetUnderlyingType()->Cast<TTupleExprType>(); + const auto* expectedTuple = expected ? expected->GetUnderlyingType()->Cast<TTupleExprType>() : nullptr; + return CheckVariantContent(gotTuple, expectedTuple, ctx, typeCtx); + } + default: + Y_UNREACHABLE(); + } - switch (got->GetKind()) { - case ETypeAnnotationKind::Struct: - { - TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Toplevel struct"); }); + return false; +} - const auto* gotStruct = got->Cast<TStructExprType>(); - const auto* expectedStruct = expected ? expected->Cast<TStructExprType>() : nullptr; +bool CheckSchema(const TTypeAnnotationNode* got, const TTypeAnnotationNode* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, bool allowVariant) { + if (expected && expected->GetKind() != got->GetKind()) { + REPORT("Expected " << expected->GetKind() << ", but got " << got->GetKind()); + return false; + } - if (!gotStruct->Validate(TPositionHandle(), ctx)) { - return false; - } + switch (got->GetKind()) { + case ETypeAnnotationKind::Struct: { + TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Toplevel struct"); }); + + const auto* gotStruct = got->Cast<TStructExprType>(); + const auto* expectedStruct = expected ? expected->Cast<TStructExprType>() : nullptr; - return CheckStruct(gotStruct, expectedStruct, ctx, typeCtx); + if (!gotStruct->Validate(TPositionHandle(), ctx)) { + return false; } - case ETypeAnnotationKind::Variant: - if (allowVariant) { - TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Toplevel variant"); }); - const auto* gotVariant = got->Cast<TVariantExprType>(); - const auto* expectedVariant = expected ? expected->Cast<TVariantExprType>() : nullptr; + return CheckStruct(gotStruct, expectedStruct, ctx, typeCtx); + } + case ETypeAnnotationKind::Variant: + if (allowVariant) { + TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Toplevel variant"); }); - if (!gotVariant->Validate(TPositionHandle(), ctx)) { - return false; - } + const auto* gotVariant = got->Cast<TVariantExprType>(); + const auto* expectedVariant = expected ? expected->Cast<TVariantExprType>() : nullptr; - return CheckVariant(gotVariant, expectedVariant, ctx, typeCtx); - } - [[fallthrough]]; - default: - if (allowVariant) { - REPORT("Expected Struct or Variant, but got " << got->GetKind()); - } else { - REPORT("Expected Struct, but got " << got->GetKind()); + if (!gotVariant->Validate(TPositionHandle(), ctx)) { + return false; } - return false; - } + + return CheckVariant(gotVariant, expectedVariant, ctx, typeCtx); + } + [[fallthrough]]; + default: + if (allowVariant) { + REPORT("Expected Struct or Variant, but got " << got->GetKind()); + } else { + REPORT("Expected Struct, but got " << got->GetKind()); + } + return false; } } +} // namespace namespace NYql::NPureCalc { - const TTypeAnnotationNode* MakeTypeFromSchema(const NYT::TNode& yson, TExprContext& ctx) { - const auto* type = NCommon::ParseTypeFromYson(yson, ctx); - - if (!type) { - auto issues = ctx.IssueManager.GetIssues(); - CheckFatalIssues(issues); - ythrow TCompileError("", issues.ToString()) - << "Incorrect schema: " << NYT::NodeToYsonString(yson, NYson::EYsonFormat::Text); - } - - return type; +const TTypeAnnotationNode* MakeTypeFromSchema(const NYT::TNode& yson, TExprContext& ctx) { + const auto* type = NCommon::ParseTypeFromYson(yson, ctx); + + if (!type) { + auto issues = ctx.IssueManager.GetIssues(); + CheckFatalIssues(issues); + ythrow TCompileError("", issues.ToString()) + << "Incorrect schema: " << NYT::NodeToYsonString(yson, NYson::EYsonFormat::Text); } - const TStructExprType* ExtendStructType( - const TStructExprType* type, const THashMap<TString, NYT::TNode>& extraColumns, TExprContext& ctx) - { - if (extraColumns.empty()) { - return type; - } + return type; +} - auto items = type->GetItems(); - for (const auto& pair : extraColumns) { - items.push_back(ctx.MakeType<TItemExprType>(pair.first, MakeTypeFromSchema(pair.second, ctx))); - } +const TStructExprType* ExtendStructType( + const TStructExprType* type, const THashMap<TString, NYT::TNode>& extraColumns, TExprContext& ctx) +{ + if (extraColumns.empty()) { + return type; + } - auto result = ctx.MakeType<TStructExprType>(items); + auto items = type->GetItems(); + for (const auto& pair : extraColumns) { + items.push_back(ctx.MakeType<TItemExprType>(pair.first, MakeTypeFromSchema(pair.second, ctx))); + } - if (!result->Validate(TPosition(), ctx)) { - auto issues = ctx.IssueManager.GetIssues(); - CheckFatalIssues(issues); - ythrow TCompileError("", issues.ToString()) << "Incorrect extended struct type"; - } + auto result = ctx.MakeType<TStructExprType>(items); - return result; + if (!result->Validate(TPosition(), ctx)) { + auto issues = ctx.IssueManager.GetIssues(); + CheckFatalIssues(issues); + ythrow TCompileError("", issues.ToString()) << "Incorrect extended struct type"; } - bool ValidateInputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { - TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Input schema"); }); - return CheckSchema(type, nullptr, ctx, typeCtx, false); - } + return result; +} - bool ValidateOutputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { - TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Output schema"); }); - return CheckSchema(type, nullptr, ctx, typeCtx, true); - } +bool ValidateInputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { + TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Input schema"); }); + return CheckSchema(type, nullptr, ctx, typeCtx, false); +} - bool ValidateOutputType(const TTypeAnnotationNode* type, const TTypeAnnotationNode* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { - TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Program return type"); }); - return CheckSchema(type, expected, ctx, typeCtx, true); - } +bool ValidateOutputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { + TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Output schema"); }); + return CheckSchema(type, nullptr, ctx, typeCtx, true); +} + +bool ValidateOutputType(const TTypeAnnotationNode* type, const TTypeAnnotationNode* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { + TIssueScopeGuard issueScope(ctx.IssueManager, []() { return new TIssue(TPosition(), "Program return type"); }); + return CheckSchema(type, expected, ctx, typeCtx, true); } +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/common/type_from_schema.h b/yql/essentials/public/purecalc/common/type_from_schema.h index 18e6e8373f4..a34210b1bed 100644 --- a/yql/essentials/public/purecalc/common/type_from_schema.h +++ b/yql/essentials/public/purecalc/common/type_from_schema.h @@ -7,32 +7,32 @@ #include <library/cpp/yson/node/node.h> namespace NYql { - struct TTypeAnnotationContext; - - namespace NPureCalc { - /** - * Load struct type from yson. Use methods below to check returned type for correctness. - */ - const TTypeAnnotationNode* MakeTypeFromSchema(const NYT::TNode&, TExprContext&); - - /** - * Extend struct type with additional columns. Type of each extra column is loaded from yson. - */ - const TStructExprType* ExtendStructType(const TStructExprType*, const THashMap<TString, NYT::TNode>&, TExprContext&); - - /** - * Check if the given type can be used as an input schema, i.e. it is a struct. - */ - bool ValidateInputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); - - /** - * Check if the given type can be used as an output schema, i.e. it is a struct or a variant of structs. - */ - bool ValidateOutputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); - - /** - * Check if output type can be silently converted to the expected type. - */ - bool ValidateOutputType(const TTypeAnnotationNode* type, const TTypeAnnotationNode* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); - } -} +struct TTypeAnnotationContext; + +namespace NPureCalc { +/** + * Load struct type from yson. Use methods below to check returned type for correctness. + */ +const TTypeAnnotationNode* MakeTypeFromSchema(const NYT::TNode&, TExprContext&); + +/** + * Extend struct type with additional columns. Type of each extra column is loaded from yson. + */ +const TStructExprType* ExtendStructType(const TStructExprType*, const THashMap<TString, NYT::TNode>&, TExprContext&); + +/** + * Check if the given type can be used as an input schema, i.e. it is a struct. + */ +bool ValidateInputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); + +/** + * Check if the given type can be used as an output schema, i.e. it is a struct or a variant of structs. + */ +bool ValidateOutputSchema(const TTypeAnnotationNode* type, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); + +/** + * Check if output type can be silently converted to the expected type. + */ +bool ValidateOutputType(const TTypeAnnotationNode* type, const TTypeAnnotationNode* expected, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/worker.cpp b/yql/essentials/public/purecalc/common/worker.cpp index d5561fc4f73..e6a32ff4118 100644 --- a/yql/essentials/public/purecalc/common/worker.cpp +++ b/yql/essentials/public/purecalc/common/worker.cpp @@ -46,15 +46,12 @@ TWorkerGraph::TWorkerGraph( ui64 nativeYtTypeFlags, TMaybe<ui64> deterministicTimeProviderSeed, TLangVersion langver, - bool insideEvaluation -) + bool insideEvaluation) : ScopedAlloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), funcRegistry.SupportsSizedAllocators()) , Env(ScopedAlloc) , FuncRegistry(funcRegistry) , RandomProvider(CreateDefaultRandomProvider()) - , TimeProvider(deterministicTimeProviderSeed ? - CreateDeterministicTimeProvider(*deterministicTimeProviderSeed) : - CreateDefaultTimeProvider()) + , TimeProvider(deterministicTimeProviderSeed ? CreateDeterministicTimeProvider(*deterministicTimeProviderSeed) : CreateDefaultTimeProvider()) , LLVMSettings(LLVMSettings) , NativeYtTypeFlags(nativeYtTypeFlags) { @@ -124,19 +121,16 @@ TWorkerGraph::TWorkerGraph( const THashSet<NKikimr::NMiniKQL::TInternName> selfCallableNames = { Env.InternName(PurecalcInputCallableName), - Env.InternName(PurecalcBlockInputCallableName) - }; + Env.InternName(PurecalcBlockInputCallableName)}; NKikimr::NMiniKQL::TExploringNodeVisitor explorer; explorer.Walk(rootNode.GetNode(), Env.GetNodeStack()); auto compositeNodeFactory = NKikimr::NMiniKQL::GetCompositeWithBuiltinFactory( - {NKikimr::NMiniKQL::GetYqlFactory(), NYql::GetPgFactory()} - ); + {NKikimr::NMiniKQL::GetYqlFactory(), NYql::GetPgFactory()}); auto nodeFactory = [&]( - NKikimr::NMiniKQL::TCallable& callable, const NKikimr::NMiniKQL::TComputationNodeFactoryContext& ctx - ) -> NKikimr::NMiniKQL::IComputationNode* { + NKikimr::NMiniKQL::TCallable& callable, const NKikimr::NMiniKQL::TComputationNodeFactoryContext& ctx) -> NKikimr::NMiniKQL::IComputationNode* { if (selfCallableNames.contains(callable.GetType()->GetNameStr())) { if (insideEvaluation) { throw TErrorException(0) << "Inputs aren't available during evaluation"; @@ -147,8 +141,7 @@ TWorkerGraph::TWorkerGraph( YQL_ENSURE(inputIndex < inputsCount, "Self index is out of range"); YQL_ENSURE(!SelfNodes[inputIndex], "Self can be called at most once with each index"); return SelfNodes[inputIndex] = new NKikimr::NMiniKQL::TExternalComputationNode(ctx.Mutables); - } - else { + } else { return compositeNodeFactory(callable, ctx); } }; @@ -171,7 +164,7 @@ TWorkerGraph::TWorkerGraph( ComputationPattern = NKikimr::NMiniKQL::MakeComputationPattern( explorer, rootNode, - { rootNode.GetNode() }, + {rootNode.GetNode()}, computationPatternOpts); ComputationGraph = ComputationPattern->Clone( @@ -206,12 +199,11 @@ TWorker<TBase>::TWorker( NKikimr::NUdf::ICountersProvider* countersProvider, ui64 nativeYtTypeFlags, TMaybe<ui64> deterministicTimeProviderSeed, - TLangVersion langver -) + TLangVersion langver) : WorkerFactory_(std::move(factory)) , Graph_(exprRoot, exprCtx, serializedProgram, funcRegistry, userData, - inputTypes, originalInputTypes, rawInputTypes, outputType, rawOutputType, - LLVMSettings, countersProvider, nativeYtTypeFlags, deterministicTimeProviderSeed, langver, false) + inputTypes, originalInputTypes, rawInputTypes, outputType, rawOutputType, + LLVMSettings, countersProvider, nativeYtTypeFlags, deterministicTimeProviderSeed, langver, false) { } @@ -404,9 +396,9 @@ NKikimr::NUdf::TUnboxedValue& TPullStreamWorker::GetOutput() { } void TPullStreamWorker::Release() { - with_lock(GetScopedAlloc()) { + with_lock (GetScopedAlloc()) { Output_ = NKikimr::NUdf::TUnboxedValue::Invalid(); - for (auto selfNode: Graph_.SelfNodes) { + for (auto selfNode : Graph_.SelfNodes) { if (selfNode) { selfNode->SetValue(Graph_.ComputationGraph->GetContext(), NKikimr::NUdf::TUnboxedValue::Invalid()); } @@ -477,11 +469,11 @@ void TPullListWorker::ResetOutputIterator() { } void TPullListWorker::Release() { - with_lock(GetScopedAlloc()) { + with_lock (GetScopedAlloc()) { Output_ = NKikimr::NUdf::TUnboxedValue::Invalid(); OutputIterator_ = NKikimr::NUdf::TUnboxedValue::Invalid(); - for (auto selfNode: Graph_.SelfNodes) { + for (auto selfNode : Graph_.SelfNodes) { if (selfNode) { selfNode->SetValue(Graph_.ComputationGraph->GetContext(), NKikimr::NUdf::TUnboxedValue::Invalid()); } @@ -492,45 +484,45 @@ void TPullListWorker::Release() { } namespace { - class TPushStream final: public NKikimr::NMiniKQL::TCustomListValue { - private: - mutable bool HasIterator_ = false; - bool HasValue_ = false; - bool IsFinished_ = false; - NKikimr::NUdf::TUnboxedValue Value_ = NKikimr::NUdf::TUnboxedValue::Invalid(); - - public: - using TCustomListValue::TCustomListValue; - - public: - void SetValue(NKikimr::NUdf::TUnboxedValue&& value) { - Value_ = std::move(value); - HasValue_ = true; - } +class TPushStream final: public NKikimr::NMiniKQL::TCustomListValue { +private: + mutable bool HasIterator_ = false; + bool HasValue_ = false; + bool IsFinished_ = false; + NKikimr::NUdf::TUnboxedValue Value_ = NKikimr::NUdf::TUnboxedValue::Invalid(); + +public: + using TCustomListValue::TCustomListValue; + +public: + void SetValue(NKikimr::NUdf::TUnboxedValue&& value) { + Value_ = std::move(value); + HasValue_ = true; + } - void SetFinished() { - IsFinished_ = true; - } + void SetFinished() { + IsFinished_ = true; + } - NKikimr::NUdf::TUnboxedValue GetListIterator() const override { - YQL_ENSURE(!HasIterator_, "only one pass over input is supported"); - HasIterator_ = true; - return NKikimr::NUdf::TUnboxedValuePod(const_cast<TPushStream*>(this)); - } + NKikimr::NUdf::TUnboxedValue GetListIterator() const override { + YQL_ENSURE(!HasIterator_, "only one pass over input is supported"); + HasIterator_ = true; + return NKikimr::NUdf::TUnboxedValuePod(const_cast<TPushStream*>(this)); + } - NKikimr::NUdf::EFetchStatus Fetch(NKikimr::NUdf::TUnboxedValue& result) override { - if (IsFinished_) { - return NKikimr::NUdf::EFetchStatus::Finish; - } else if (!HasValue_) { - return NKikimr::NUdf::EFetchStatus::Yield; - } else { - result = std::move(Value_); - HasValue_ = false; - return NKikimr::NUdf::EFetchStatus::Ok; - } + NKikimr::NUdf::EFetchStatus Fetch(NKikimr::NUdf::TUnboxedValue& result) override { + if (IsFinished_) { + return NKikimr::NUdf::EFetchStatus::Finish; + } else if (!HasValue_) { + return NKikimr::NUdf::EFetchStatus::Yield; + } else { + result = std::move(Value_); + HasValue_ = false; + return NKikimr::NUdf::EFetchStatus::Ok; } - }; -} + } +}; +} // namespace void TPushStreamWorker::FeedToConsumer() { auto value = Graph_.ComputationGraph->GetValue(); @@ -608,7 +600,7 @@ void TPushStreamWorker::OnFinish() { } void TPushStreamWorker::Release() { - with_lock(GetScopedAlloc()) { + with_lock (GetScopedAlloc()) { Consumer_.Destroy(); if (SelfNode_) { SelfNode_->SetValue(Graph_.ComputationGraph->GetContext(), NKikimr::NUdf::TUnboxedValue::Invalid()); @@ -619,16 +611,12 @@ void TPushStreamWorker::Release() { TWorker<IPushStreamWorker>::Release(); } - namespace NYql { - namespace NPureCalc { - template - class TWorker<IPullStreamWorker>; +namespace NPureCalc { +template class TWorker<IPullStreamWorker>; - template - class TWorker<IPullListWorker>; +template class TWorker<IPullListWorker>; - template - class TWorker<IPushStreamWorker>; - } -} +template class TWorker<IPushStreamWorker>; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/worker.h b/yql/essentials/public/purecalc/common/worker.h index b032b15af15..415e57fcf5e 100644 --- a/yql/essentials/public/purecalc/common/worker.h +++ b/yql/essentials/public/purecalc/common/worker.h @@ -14,169 +14,169 @@ #include <memory> namespace NYql { - namespace NPureCalc { - struct TWorkerGraph { - TWorkerGraph( - const TExprNode::TPtr& exprRoot, - TExprContext& exprCtx, - const TString& serializedProgram, - const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, - const TUserDataTable& userData, - const TVector<const TStructExprType*>& inputTypes, - const TVector<const TStructExprType*>& originalInputTypes, - const TVector<const TStructExprType*>& rawInputTypes, - const TTypeAnnotationNode* outputType, - const TTypeAnnotationNode* rawOutputType, - const TString& LLVMSettings, - NKikimr::NUdf::ICountersProvider* countersProvider, - ui64 nativeYtTypeFlags, - TMaybe<ui64> deterministicTimeProviderSeed, - TLangVersion langver, - bool insideEvaluation - ); - - ~TWorkerGraph(); - - NKikimr::NMiniKQL::TScopedAlloc ScopedAlloc; - NKikimr::NMiniKQL::TTypeEnvironment Env; - const NKikimr::NMiniKQL::IFunctionRegistry& FuncRegistry; - TIntrusivePtr<IRandomProvider> RandomProvider; - TIntrusivePtr<ITimeProvider> TimeProvider; - NKikimr::NMiniKQL::IComputationPattern::TPtr ComputationPattern; - THolder<NKikimr::NMiniKQL::IComputationGraph> ComputationGraph; - TString LLVMSettings; - ui64 NativeYtTypeFlags; - TMaybe<TString> TimestampColumn; - const NKikimr::NMiniKQL::TType* OutputType; - const NKikimr::NMiniKQL::TType* RawOutputType; - TVector<NKikimr::NMiniKQL::IComputationExternalNode*> SelfNodes; - TVector<const NKikimr::NMiniKQL::TStructType*> InputTypes; - TVector<const NKikimr::NMiniKQL::TStructType*> OriginalInputTypes; - TVector<const NKikimr::NMiniKQL::TStructType*> RawInputTypes; - }; - - template <typename TBase> - class TWorker: public TBase { - public: - using TWorkerFactoryPtr = std::weak_ptr<IWorkerFactory>; - private: - // Worker factory implementation should stay alive for this worker to operate correctly. - TWorkerFactoryPtr WorkerFactory_; - - protected: - TWorkerGraph Graph_; - - public: - TWorker( - TWorkerFactoryPtr factory, - const TExprNode::TPtr& exprRoot, - TExprContext& exprCtx, - const TString& serializedProgram, - const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, - const TUserDataTable& userData, - const TVector<const TStructExprType*>& inputTypes, - const TVector<const TStructExprType*>& originalInputTypes, - const TVector<const TStructExprType*>& rawInputTypes, - const TTypeAnnotationNode* outputType, - const TTypeAnnotationNode* rawOutputType, - const TString& LLVMSettings, - NKikimr::NUdf::ICountersProvider* countersProvider, - ui64 nativeYtTypeFlags, - TMaybe<ui64> deterministicTimeProviderSeed, - TLangVersion langver - ); - - public: - ui32 GetInputsCount() const override; - const NKikimr::NMiniKQL::TStructType* GetInputType(ui32, bool) const override; - const NKikimr::NMiniKQL::TStructType* GetInputType(bool) const override; - const NKikimr::NMiniKQL::TStructType* GetRawInputType(ui32) const override; - const NKikimr::NMiniKQL::TStructType* GetRawInputType() const override; - const NKikimr::NMiniKQL::TType* GetOutputType() const override; - const NKikimr::NMiniKQL::TType* GetRawOutputType() const override; - NYT::TNode MakeInputSchema() const override; - NYT::TNode MakeInputSchema(ui32) const override; - NYT::TNode MakeOutputSchema() const override; - NYT::TNode MakeOutputSchema(ui32) const override; - NYT::TNode MakeOutputSchema(TStringBuf) const override; - NYT::TNode MakeFullOutputSchema() const override; - NKikimr::NMiniKQL::TScopedAlloc& GetScopedAlloc() override; - NKikimr::NMiniKQL::IComputationGraph& GetGraph() override; - const NKikimr::NMiniKQL::IFunctionRegistry& GetFunctionRegistry() const override; - NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnvironment() override; - const TString& GetLLVMSettings() const override; - ui64 GetNativeYtTypeFlags() const override; - ITimeProvider* GetTimeProvider() const override; - void Invalidate() override; - protected: - void Release() override; - }; - - class TPullStreamWorker final: public TWorker<IPullStreamWorker> { - private: - NKikimr::NUdf::TUnboxedValue Output_ = NKikimr::NUdf::TUnboxedValue::Invalid(); - TVector<bool> HasInput_; - - inline bool CheckAllInputsSet() { - return AllOf(HasInput_, [](bool x) { return x; }); - } - - public: - using TWorker::TWorker; - ~TPullStreamWorker(); - - public: - void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) override; - NKikimr::NUdf::TUnboxedValue& GetOutput() override; - - protected: - void Release() override; - }; - - class TPullListWorker final: public TWorker<IPullListWorker> { - private: - NKikimr::NUdf::TUnboxedValue Output_ = NKikimr::NUdf::TUnboxedValue::Invalid(); - NKikimr::NUdf::TUnboxedValue OutputIterator_ = NKikimr::NUdf::TUnboxedValue::Invalid(); - TVector<bool> HasInput_; - - inline bool CheckAllInputsSet() { - return AllOf(HasInput_, [](bool x) { return x; }); - } - - public: - using TWorker::TWorker; - ~TPullListWorker(); - - public: - void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) override; - NKikimr::NUdf::TUnboxedValue& GetOutput() override; - NKikimr::NUdf::TUnboxedValue& GetOutputIterator() override; - void ResetOutputIterator() override; - - protected: - void Release() override; - }; - - class TPushStreamWorker final: public TWorker<IPushStreamWorker> { - private: - THolder<IConsumer<const NKikimr::NUdf::TUnboxedValue*>> Consumer_{}; - bool Finished_ = false; - NKikimr::NMiniKQL::IComputationExternalNode* SelfNode_ = nullptr; - - public: - using TWorker::TWorker; - - private: - void FeedToConsumer(); - NYql::NUdf::IBoxedValue* GetPushStream() const; - - public: - void SetConsumer(THolder<IConsumer<const NKikimr::NUdf::TUnboxedValue*>>) override; - void Push(NKikimr::NUdf::TUnboxedValue&&) override; - void OnFinish() override; - - protected: - void Release() override; - }; +namespace NPureCalc { +struct TWorkerGraph { + TWorkerGraph( + const TExprNode::TPtr& exprRoot, + TExprContext& exprCtx, + const TString& serializedProgram, + const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, + const TUserDataTable& userData, + const TVector<const TStructExprType*>& inputTypes, + const TVector<const TStructExprType*>& originalInputTypes, + const TVector<const TStructExprType*>& rawInputTypes, + const TTypeAnnotationNode* outputType, + const TTypeAnnotationNode* rawOutputType, + const TString& LLVMSettings, + NKikimr::NUdf::ICountersProvider* countersProvider, + ui64 nativeYtTypeFlags, + TMaybe<ui64> deterministicTimeProviderSeed, + TLangVersion langver, + bool insideEvaluation); + + ~TWorkerGraph(); + + NKikimr::NMiniKQL::TScopedAlloc ScopedAlloc; + NKikimr::NMiniKQL::TTypeEnvironment Env; + const NKikimr::NMiniKQL::IFunctionRegistry& FuncRegistry; + TIntrusivePtr<IRandomProvider> RandomProvider; + TIntrusivePtr<ITimeProvider> TimeProvider; + NKikimr::NMiniKQL::IComputationPattern::TPtr ComputationPattern; + THolder<NKikimr::NMiniKQL::IComputationGraph> ComputationGraph; + TString LLVMSettings; + ui64 NativeYtTypeFlags; + TMaybe<TString> TimestampColumn; + const NKikimr::NMiniKQL::TType* OutputType; + const NKikimr::NMiniKQL::TType* RawOutputType; + TVector<NKikimr::NMiniKQL::IComputationExternalNode*> SelfNodes; + TVector<const NKikimr::NMiniKQL::TStructType*> InputTypes; + TVector<const NKikimr::NMiniKQL::TStructType*> OriginalInputTypes; + TVector<const NKikimr::NMiniKQL::TStructType*> RawInputTypes; +}; + +template <typename TBase> +class TWorker: public TBase { +public: + using TWorkerFactoryPtr = std::weak_ptr<IWorkerFactory>; + +private: + // Worker factory implementation should stay alive for this worker to operate correctly. + TWorkerFactoryPtr WorkerFactory_; + +protected: + TWorkerGraph Graph_; + +public: + TWorker( + TWorkerFactoryPtr factory, + const TExprNode::TPtr& exprRoot, + TExprContext& exprCtx, + const TString& serializedProgram, + const NKikimr::NMiniKQL::IFunctionRegistry& funcRegistry, + const TUserDataTable& userData, + const TVector<const TStructExprType*>& inputTypes, + const TVector<const TStructExprType*>& originalInputTypes, + const TVector<const TStructExprType*>& rawInputTypes, + const TTypeAnnotationNode* outputType, + const TTypeAnnotationNode* rawOutputType, + const TString& LLVMSettings, + NKikimr::NUdf::ICountersProvider* countersProvider, + ui64 nativeYtTypeFlags, + TMaybe<ui64> deterministicTimeProviderSeed, + TLangVersion langver); + +public: + ui32 GetInputsCount() const override; + const NKikimr::NMiniKQL::TStructType* GetInputType(ui32, bool) const override; + const NKikimr::NMiniKQL::TStructType* GetInputType(bool) const override; + const NKikimr::NMiniKQL::TStructType* GetRawInputType(ui32) const override; + const NKikimr::NMiniKQL::TStructType* GetRawInputType() const override; + const NKikimr::NMiniKQL::TType* GetOutputType() const override; + const NKikimr::NMiniKQL::TType* GetRawOutputType() const override; + NYT::TNode MakeInputSchema() const override; + NYT::TNode MakeInputSchema(ui32) const override; + NYT::TNode MakeOutputSchema() const override; + NYT::TNode MakeOutputSchema(ui32) const override; + NYT::TNode MakeOutputSchema(TStringBuf) const override; + NYT::TNode MakeFullOutputSchema() const override; + NKikimr::NMiniKQL::TScopedAlloc& GetScopedAlloc() override; + NKikimr::NMiniKQL::IComputationGraph& GetGraph() override; + const NKikimr::NMiniKQL::IFunctionRegistry& GetFunctionRegistry() const override; + NKikimr::NMiniKQL::TTypeEnvironment& GetTypeEnvironment() override; + const TString& GetLLVMSettings() const override; + ui64 GetNativeYtTypeFlags() const override; + ITimeProvider* GetTimeProvider() const override; + void Invalidate() override; + +protected: + void Release() override; +}; + +class TPullStreamWorker final: public TWorker<IPullStreamWorker> { +private: + NKikimr::NUdf::TUnboxedValue Output_ = NKikimr::NUdf::TUnboxedValue::Invalid(); + TVector<bool> HasInput_; + + inline bool CheckAllInputsSet() { + return AllOf(HasInput_, [](bool x) { return x; }); } -} + +public: + using TWorker::TWorker; + ~TPullStreamWorker(); + +public: + void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) override; + NKikimr::NUdf::TUnboxedValue& GetOutput() override; + +protected: + void Release() override; +}; + +class TPullListWorker final: public TWorker<IPullListWorker> { +private: + NKikimr::NUdf::TUnboxedValue Output_ = NKikimr::NUdf::TUnboxedValue::Invalid(); + NKikimr::NUdf::TUnboxedValue OutputIterator_ = NKikimr::NUdf::TUnboxedValue::Invalid(); + TVector<bool> HasInput_; + + inline bool CheckAllInputsSet() { + return AllOf(HasInput_, [](bool x) { return x; }); + } + +public: + using TWorker::TWorker; + ~TPullListWorker(); + +public: + void SetInput(NKikimr::NUdf::TUnboxedValue&&, ui32) override; + NKikimr::NUdf::TUnboxedValue& GetOutput() override; + NKikimr::NUdf::TUnboxedValue& GetOutputIterator() override; + void ResetOutputIterator() override; + +protected: + void Release() override; +}; + +class TPushStreamWorker final: public TWorker<IPushStreamWorker> { +private: + THolder<IConsumer<const NKikimr::NUdf::TUnboxedValue*>> Consumer_{}; + bool Finished_ = false; + NKikimr::NMiniKQL::IComputationExternalNode* SelfNode_ = nullptr; + +public: + using TWorker::TWorker; + +private: + void FeedToConsumer(); + NYql::NUdf::IBoxedValue* GetPushStream() const; + +public: + void SetConsumer(THolder<IConsumer<const NKikimr::NUdf::TUnboxedValue*>>) override; + void Push(NKikimr::NUdf::TUnboxedValue&&) override; + void OnFinish() override; + +protected: + void Release() override; +}; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp index 8261528b237..347fbbe585a 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.cpp +++ b/yql/essentials/public/purecalc/common/worker_factory.cpp @@ -113,8 +113,8 @@ TWorkerFactory<TBase>::TWorkerFactory(TWorkerFactoryOptions options, EProcessorM SerializedProgram_ = TString{options.Query}; } else { ExprRoot_ = Compile(options.Query, options.TranslationMode, - options.SyntaxVersion, options.Modules, - options.InputSpec, options.OutputSpec, options.UseAntlr4, processorMode, typeCtx.Get()); + options.SyntaxVersion, options.Modules, + options.InputSpec, options.OutputSpec, options.UseAntlr4, processorMode, typeCtx.Get()); RawOutputType_ = GetSequenceItemType(ExprRoot_->Pos(), ExprRoot_->GetTypeAnn(), true, ExprContext_); @@ -138,8 +138,7 @@ TWorkerFactory<TBase>::TWorkerFactory(TWorkerFactoryOptions options, EProcessorM template <typename TBase> TIntrusivePtr<TTypeAnnotationContext> TWorkerFactory<TBase>::PrepareTypeContext( - IModuleResolver::TPtr factoryModuleResolver -) { + IModuleResolver::TPtr factoryModuleResolver) { // Prepare type annotation context IModuleResolver::TPtr moduleResolver = factoryModuleResolver ? factoryModuleResolver->CreateMutableChild() : nullptr; @@ -147,9 +146,7 @@ TIntrusivePtr<TTypeAnnotationContext> TWorkerFactory<TBase>::PrepareTypeContext( typeContext->LangVer = LangVer_; typeContext->UseTypeDiffForConvertToError = true; typeContext->RandomProvider = CreateDefaultRandomProvider(); - typeContext->TimeProvider = DeterministicTimeProviderSeed_ ? - CreateDeterministicTimeProvider(*DeterministicTimeProviderSeed_) : - CreateDefaultTimeProvider(); + typeContext->TimeProvider = DeterministicTimeProviderSeed_ ? CreateDeterministicTimeProvider(*DeterministicTimeProviderSeed_) : CreateDefaultTimeProvider(); typeContext->UdfResolver = NCommon::CreateSimpleUdfResolver(FuncRegistry_.Get()); typeContext->ArrowResolver = MakeSimpleArrowResolver(*FuncRegistry_.Get()); typeContext->UserDataStorage = MakeIntrusive<TUserDataStorage>(nullptr, UserData_, nullptr, nullptr); @@ -176,8 +173,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( const TOutputSpecBase& outputSpec, bool useAntlr4, EProcessorMode processorMode, - TTypeAnnotationContext* typeContext -) { + TTypeAnnotationContext* typeContext) { Y_ENSURE(useAntlr4, "Antlr3 support is dropped"); if (mode == ETranslationMode::PG && processorMode != EProcessorMode::PullList) { ythrow TCompileError("", "") << "only PullList mode is compatible to PostgreSQL syntax"; @@ -219,8 +215,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( "DisableAnsiOptionalAs", "DisableCoalesceJoinKeysOnQualifiedAll", "DisableUnorderedSubqueries", - "FlexibleTypes" - }; + "FlexibleTypes"}; if (BlockEngineMode_ != EBlockEngineMode::Disable) { settings.Flags.insert("EmitAggApply"); } @@ -242,8 +237,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( NSQLTranslation::TTranslators translators( nullptr, NSQLTranslationV1::MakeTranslator(lexers, parsers), - NSQLTranslationPG::MakeTranslator() - ); + NSQLTranslationPG::MakeTranslator()); astRes = SqlToYql(translators, TString(query), settings); } else { @@ -273,11 +267,9 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( ythrow TCompileError(astStr.Str(), GetIssues().ToString()) << "failed to compile"; } - // Prepare transformation pipeline THolder<IGraphTransformer> calcTransformer = CreateFunctorTransformer([&](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) - -> IGraphTransformer::TStatus - { + -> IGraphTransformer::TStatus { output = input; auto valueNode = input->HeadPtr(); @@ -309,8 +301,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( NativeYtTypeFlags_, DeterministicTimeProviderSeed_, LangVer_, - true - ); + true); with_lock (graph.ScopedAlloc) { const auto value = graph.ComputationGraph->GetValue(); @@ -324,8 +315,8 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( }); const TString& selfName = TString(inputSpec.ProvidesBlocks() - ? PurecalcBlockInputCallableName - : PurecalcInputCallableName); + ? PurecalcBlockInputCallableName + : PurecalcInputCallableName); TTypeAnnCallableFactory typeAnnCallableFactory = [&]() { return MakeTypeAnnotationTransformer(typeContext, InputTypes_, RawInputTypes_, processorMode, selfName); @@ -343,28 +334,26 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( pipeline.AddTypeAnnotationTransformer(); pipeline.AddPostTypeAnnotation(); pipeline.Add(CreateFunctorTransformer( - [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - return OptimizeExpr(input, output, [](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr { - if (node->IsCallable("Unordered") && node->Child(0)->IsCallable({ - PurecalcInputCallableName, PurecalcBlockInputCallableName - })) { - return node->ChildPtr(0); - } - return node; - }, ctx, TOptimizeExprSettings(nullptr)); - }), "Unordered", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, - "Unordered optimizations"); + [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + return OptimizeExpr(input, output, [](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr { + if (node->IsCallable("Unordered") && node->Child(0)->IsCallable({PurecalcInputCallableName, PurecalcBlockInputCallableName})) { + return node->ChildPtr(0); + } + return node; + }, ctx, TOptimizeExprSettings(nullptr)); + }), "Unordered", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, + "Unordered optimizations"); pipeline.Add(CreateFunctorTransformer( - [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - return OptimizeExpr(input, output, [](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr { - if (node->IsCallable("Right!") && node->Head().IsCallable("Cons!")) { - return node->Head().ChildPtr(1); - } - - return node; - }, ctx, TOptimizeExprSettings(nullptr)); - }), "Cons", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, - "Cons optimizations"); + [&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + return OptimizeExpr(input, output, [](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr { + if (node->IsCallable("Right!") && node->Head().IsCallable("Cons!")) { + return node->Head().ChildPtr(1); + } + + return node; + }, ctx, TOptimizeExprSettings(nullptr)); + }), "Cons", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, + "Cons optimizations"); pipeline.Add(MakeOutputColumnsFilter(outputSpec.GetOutputColumnsFilter()), "Filter", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, "Filter output columns"); @@ -406,10 +395,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( if (exprOut) { *exprOut << "After optimization:" << Endl; - ConvertToAst(*exprRoot, ExprContext_, 0, true).Root - ->PrettyPrintTo(*exprOut, TAstPrintFlags::PerLine - | TAstPrintFlags::ShortQuote - | TAstPrintFlags::AdaptArbitraryContent); + ConvertToAst(*exprRoot, ExprContext_, 0, true).Root->PrettyPrintTo(*exprOut, TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote | TAstPrintFlags::AdaptArbitraryContent); } return exprRoot; } @@ -523,7 +509,7 @@ template <typename TBase> TString TWorkerFactory<TBase>::GetCompiledProgram() { if (ExprRoot_) { NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), - FuncRegistry_->SupportsSizedAllocators()); + FuncRegistry_->SupportsSizedAllocators()); NKikimr::NMiniKQL::TTypeEnvironment env(alloc); auto rootNode = CompileMkql(ExprRoot_, ExprContext_, *FuncRegistry_, env, UserData_); @@ -541,32 +527,30 @@ void TWorkerFactory<TBase>::ReturnWorker(IWorker* worker) { } } - -#define DEFINE_WORKER_MAKER(MODE) \ - TWorkerHolder<I##MODE##Worker> T##MODE##WorkerFactory::MakeWorker() { \ - if (!WorkerPool_.empty()) { \ - auto res = std::move(WorkerPool_.back()); \ - WorkerPool_.pop_back(); \ - return TWorkerHolder<I##MODE##Worker>((I##MODE##Worker *)res.Release()); \ - } \ - return TWorkerHolder<I##MODE##Worker>(new T##MODE##Worker( \ - weak_from_this(), \ - ExprRoot_, \ - ExprContext_, \ - SerializedProgram_, \ - *FuncRegistry_, \ - UserData_, \ - InputTypes_, \ - OriginalInputTypes_, \ - RawInputTypes_, \ - OutputType_, \ - RawOutputType_, \ - LLVMSettings_, \ - CountersProvider_, \ - NativeYtTypeFlags_, \ - DeterministicTimeProviderSeed_, \ - LangVer_ \ - )); \ +#define DEFINE_WORKER_MAKER(MODE) \ + TWorkerHolder<I##MODE##Worker> T##MODE##WorkerFactory::MakeWorker() { \ + if (!WorkerPool_.empty()) { \ + auto res = std::move(WorkerPool_.back()); \ + WorkerPool_.pop_back(); \ + return TWorkerHolder<I##MODE##Worker>((I##MODE##Worker*)res.Release()); \ + } \ + return TWorkerHolder<I##MODE##Worker>(new T##MODE##Worker( \ + weak_from_this(), \ + ExprRoot_, \ + ExprContext_, \ + SerializedProgram_, \ + *FuncRegistry_, \ + UserData_, \ + InputTypes_, \ + OriginalInputTypes_, \ + RawInputTypes_, \ + OutputType_, \ + RawOutputType_, \ + LLVMSettings_, \ + CountersProvider_, \ + NativeYtTypeFlags_, \ + DeterministicTimeProviderSeed_, \ + LangVer_)); \ } DEFINE_WORKER_MAKER(PullStream) @@ -574,14 +558,11 @@ DEFINE_WORKER_MAKER(PullList) DEFINE_WORKER_MAKER(PushStream) namespace NYql { - namespace NPureCalc { - template - class TWorkerFactory<IPullStreamWorkerFactory>; +namespace NPureCalc { +template class TWorkerFactory<IPullStreamWorkerFactory>; - template - class TWorkerFactory<IPullListWorkerFactory>; +template class TWorkerFactory<IPullListWorkerFactory>; - template - class TWorkerFactory<IPushStreamWorkerFactory>; - } -} +template class TWorkerFactory<IPushStreamWorkerFactory>; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/worker_factory.h b/yql/essentials/public/purecalc/common/worker_factory.h index 719a625f0b6..e50a1236c9f 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.h +++ b/yql/essentials/public/purecalc/common/worker_factory.h @@ -12,169 +12,167 @@ #include <utility> namespace NYql { - namespace NPureCalc { - struct TWorkerFactoryOptions { - IProgramFactoryPtr Factory; - const TInputSpecBase& InputSpec; - const TOutputSpecBase& OutputSpec; - TStringBuf Query; - TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry; - IModuleResolver::TPtr ModuleResolver; - const TUserDataTable& UserData; - const THashMap<TString, TString>& Modules; - TString LLVMSettings; - EBlockEngineMode BlockEngineMode; - IOutputStream* ExprOutputStream; - NKikimr::NUdf::ICountersProvider* CountersProvider; - ETranslationMode TranslationMode; - ui16 SyntaxVersion; - TLangVersion LangVer; - ui64 NativeYtTypeFlags; - TMaybe<ui64> DeterministicTimeProviderSeed; - bool UseSystemColumns; - bool UseWorkerPool; - bool UseAntlr4; - - TWorkerFactoryOptions( - IProgramFactoryPtr Factory, - const TInputSpecBase& InputSpec, - const TOutputSpecBase& OutputSpec, - TStringBuf Query, - TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry, - IModuleResolver::TPtr ModuleResolver, - const TUserDataTable& UserData, - const THashMap<TString, TString>& Modules, - TString LLVMSettings, - EBlockEngineMode BlockEngineMode, - IOutputStream* ExprOutputStream, - NKikimr::NUdf::ICountersProvider* CountersProvider, - ETranslationMode translationMode, - ui16 syntaxVersion, - TLangVersion langver, - ui64 nativeYtTypeFlags, - TMaybe<ui64> deterministicTimeProviderSeed, - bool useSystemColumns, - bool useWorkerPool, - bool useAntlr4 - ) - : Factory(std::move(Factory)) - , InputSpec(InputSpec) - , OutputSpec(OutputSpec) - , Query(Query) - , FuncRegistry(std::move(FuncRegistry)) - , ModuleResolver(std::move(ModuleResolver)) - , UserData(UserData) - , Modules(Modules) - , LLVMSettings(std::move(LLVMSettings)) - , BlockEngineMode(BlockEngineMode) - , ExprOutputStream(ExprOutputStream) - , CountersProvider(CountersProvider) - , TranslationMode(translationMode) - , SyntaxVersion(syntaxVersion) - , LangVer(langver) - , NativeYtTypeFlags(nativeYtTypeFlags) - , DeterministicTimeProviderSeed(deterministicTimeProviderSeed) - , UseSystemColumns(useSystemColumns) - , UseWorkerPool(useWorkerPool) - , UseAntlr4(useAntlr4) - { - } - }; - - template <typename TBase> - class TWorkerFactory: public TBase { - private: - IProgramFactoryPtr Factory_; - - protected: - TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry_; - const TUserDataTable& UserData_; - TExprContext ExprContext_; - TExprNode::TPtr ExprRoot_; - TString SerializedProgram_; - TVector<const TStructExprType*> InputTypes_; - TVector<const TStructExprType*> OriginalInputTypes_; - TVector<const TStructExprType*> RawInputTypes_; - const TTypeAnnotationNode* OutputType_; - const TTypeAnnotationNode* RawOutputType_; - TVector<THashSet<TString>> AllColumns_; - TVector<THashSet<TString>> UsedColumns_; - TString LLVMSettings_; - EBlockEngineMode BlockEngineMode_; - IOutputStream* ExprOutputStream_; - NKikimr::NUdf::ICountersProvider* CountersProvider_; - ui64 NativeYtTypeFlags_; - TMaybe<ui64> DeterministicTimeProviderSeed_; - bool UseSystemColumns_; - bool UseWorkerPool_; - TLangVersion LangVer_; - TVector<THolder<IWorker>> WorkerPool_; - - public: - TWorkerFactory(TWorkerFactoryOptions, EProcessorMode); - - public: - NYT::TNode MakeInputSchema(ui32) const override; - NYT::TNode MakeInputSchema() const override; - NYT::TNode MakeOutputSchema() const override; - NYT::TNode MakeOutputSchema(ui32) const override; - NYT::TNode MakeOutputSchema(TStringBuf) const override; - NYT::TNode MakeFullOutputSchema() const override; - const THashSet<TString>& GetUsedColumns(ui32 inputIndex) const override; - const THashSet<TString>& GetUsedColumns() const override; - TIssues GetIssues() const override; - TString GetCompiledProgram() override; - - protected: - void ReturnWorker(IWorker* worker) override; - - private: - TIntrusivePtr<TTypeAnnotationContext> PrepareTypeContext( - IModuleResolver::TPtr factoryModuleResolver - ); - - TExprNode::TPtr Compile(TStringBuf query, - ETranslationMode mode, - ui16 syntaxVersion, - const THashMap<TString, TString>& modules, - const TInputSpecBase& inputSpec, - const TOutputSpecBase& outputSpec, - bool useAntlr4, - EProcessorMode processorMode, - TTypeAnnotationContext* typeContext); - }; - - class TPullStreamWorkerFactory final: public TWorkerFactory<IPullStreamWorkerFactory> { - public: - explicit TPullStreamWorkerFactory(TWorkerFactoryOptions options) - : TWorkerFactory(std::move(options), EProcessorMode::PullStream) - { - } - - public: - TWorkerHolder<IPullStreamWorker> MakeWorker() override; - }; - - class TPullListWorkerFactory final: public TWorkerFactory<IPullListWorkerFactory> { - public: - explicit TPullListWorkerFactory(TWorkerFactoryOptions options) - : TWorkerFactory(std::move(options), EProcessorMode::PullList) - { - } - - public: - TWorkerHolder<IPullListWorker> MakeWorker() override; - }; - - class TPushStreamWorkerFactory final: public TWorkerFactory<IPushStreamWorkerFactory> { - public: - explicit TPushStreamWorkerFactory(TWorkerFactoryOptions options) - : TWorkerFactory(std::move(options), EProcessorMode::PushStream) - { - } - - public: - TWorkerHolder<IPushStreamWorker> MakeWorker() override; - }; +namespace NPureCalc { +struct TWorkerFactoryOptions { + IProgramFactoryPtr Factory; + const TInputSpecBase& InputSpec; + const TOutputSpecBase& OutputSpec; + TStringBuf Query; + TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry; + IModuleResolver::TPtr ModuleResolver; + const TUserDataTable& UserData; + const THashMap<TString, TString>& Modules; + TString LLVMSettings; + EBlockEngineMode BlockEngineMode; + IOutputStream* ExprOutputStream; + NKikimr::NUdf::ICountersProvider* CountersProvider; + ETranslationMode TranslationMode; + ui16 SyntaxVersion; + TLangVersion LangVer; + ui64 NativeYtTypeFlags; + TMaybe<ui64> DeterministicTimeProviderSeed; + bool UseSystemColumns; + bool UseWorkerPool; + bool UseAntlr4; + + TWorkerFactoryOptions( + IProgramFactoryPtr Factory, + const TInputSpecBase& InputSpec, + const TOutputSpecBase& OutputSpec, + TStringBuf Query, + TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry, + IModuleResolver::TPtr ModuleResolver, + const TUserDataTable& UserData, + const THashMap<TString, TString>& Modules, + TString LLVMSettings, + EBlockEngineMode BlockEngineMode, + IOutputStream* ExprOutputStream, + NKikimr::NUdf::ICountersProvider* CountersProvider, + ETranslationMode translationMode, + ui16 syntaxVersion, + TLangVersion langver, + ui64 nativeYtTypeFlags, + TMaybe<ui64> deterministicTimeProviderSeed, + bool useSystemColumns, + bool useWorkerPool, + bool useAntlr4) + : Factory(std::move(Factory)) + , InputSpec(InputSpec) + , OutputSpec(OutputSpec) + , Query(Query) + , FuncRegistry(std::move(FuncRegistry)) + , ModuleResolver(std::move(ModuleResolver)) + , UserData(UserData) + , Modules(Modules) + , LLVMSettings(std::move(LLVMSettings)) + , BlockEngineMode(BlockEngineMode) + , ExprOutputStream(ExprOutputStream) + , CountersProvider(CountersProvider) + , TranslationMode(translationMode) + , SyntaxVersion(syntaxVersion) + , LangVer(langver) + , NativeYtTypeFlags(nativeYtTypeFlags) + , DeterministicTimeProviderSeed(deterministicTimeProviderSeed) + , UseSystemColumns(useSystemColumns) + , UseWorkerPool(useWorkerPool) + , UseAntlr4(useAntlr4) + { } -} +}; + +template <typename TBase> +class TWorkerFactory: public TBase { +private: + IProgramFactoryPtr Factory_; + +protected: + TIntrusivePtr<NKikimr::NMiniKQL::IMutableFunctionRegistry> FuncRegistry_; + const TUserDataTable& UserData_; + TExprContext ExprContext_; + TExprNode::TPtr ExprRoot_; + TString SerializedProgram_; + TVector<const TStructExprType*> InputTypes_; + TVector<const TStructExprType*> OriginalInputTypes_; + TVector<const TStructExprType*> RawInputTypes_; + const TTypeAnnotationNode* OutputType_; + const TTypeAnnotationNode* RawOutputType_; + TVector<THashSet<TString>> AllColumns_; + TVector<THashSet<TString>> UsedColumns_; + TString LLVMSettings_; + EBlockEngineMode BlockEngineMode_; + IOutputStream* ExprOutputStream_; + NKikimr::NUdf::ICountersProvider* CountersProvider_; + ui64 NativeYtTypeFlags_; + TMaybe<ui64> DeterministicTimeProviderSeed_; + bool UseSystemColumns_; + bool UseWorkerPool_; + TLangVersion LangVer_; + TVector<THolder<IWorker>> WorkerPool_; + +public: + TWorkerFactory(TWorkerFactoryOptions, EProcessorMode); + +public: + NYT::TNode MakeInputSchema(ui32) const override; + NYT::TNode MakeInputSchema() const override; + NYT::TNode MakeOutputSchema() const override; + NYT::TNode MakeOutputSchema(ui32) const override; + NYT::TNode MakeOutputSchema(TStringBuf) const override; + NYT::TNode MakeFullOutputSchema() const override; + const THashSet<TString>& GetUsedColumns(ui32 inputIndex) const override; + const THashSet<TString>& GetUsedColumns() const override; + TIssues GetIssues() const override; + TString GetCompiledProgram() override; + +protected: + void ReturnWorker(IWorker* worker) override; + +private: + TIntrusivePtr<TTypeAnnotationContext> PrepareTypeContext( + IModuleResolver::TPtr factoryModuleResolver); + + TExprNode::TPtr Compile(TStringBuf query, + ETranslationMode mode, + ui16 syntaxVersion, + const THashMap<TString, TString>& modules, + const TInputSpecBase& inputSpec, + const TOutputSpecBase& outputSpec, + bool useAntlr4, + EProcessorMode processorMode, + TTypeAnnotationContext* typeContext); +}; + +class TPullStreamWorkerFactory final: public TWorkerFactory<IPullStreamWorkerFactory> { +public: + explicit TPullStreamWorkerFactory(TWorkerFactoryOptions options) + : TWorkerFactory(std::move(options), EProcessorMode::PullStream) + { + } + +public: + TWorkerHolder<IPullStreamWorker> MakeWorker() override; +}; + +class TPullListWorkerFactory final: public TWorkerFactory<IPullListWorkerFactory> { +public: + explicit TPullListWorkerFactory(TWorkerFactoryOptions options) + : TWorkerFactory(std::move(options), EProcessorMode::PullList) + { + } + +public: + TWorkerHolder<IPullListWorker> MakeWorker() override; +}; + +class TPushStreamWorkerFactory final: public TWorkerFactory<IPushStreamWorkerFactory> { +public: + explicit TPushStreamWorkerFactory(TWorkerFactoryOptions options) + : TWorkerFactory(std::move(options), EProcessorMode::PushStream) + { + } + +public: + TWorkerHolder<IPushStreamWorker> MakeWorker() override; +}; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/common/wrappers.h b/yql/essentials/public/purecalc/common/wrappers.h index 35edeb7398a..57be6315a43 100644 --- a/yql/essentials/public/purecalc/common/wrappers.h +++ b/yql/essentials/public/purecalc/common/wrappers.h @@ -5,66 +5,66 @@ #include <util/generic/ptr.h> namespace NYql::NPureCalc::NPrivate { - template <typename TNew, typename TOld, typename TFunctor> - class TMappingStream final: public IStream<TNew> { - private: - THolder<IStream<TOld>> Old_; - TFunctor Functor_; +template <typename TNew, typename TOld, typename TFunctor> +class TMappingStream final: public IStream<TNew> { +private: + THolder<IStream<TOld>> Old_; + TFunctor Functor_; - public: - TMappingStream(THolder<IStream<TOld>> old, TFunctor functor) - : Old_(std::move(old)) - , Functor_(std::move(functor)) - { - } +public: + TMappingStream(THolder<IStream<TOld>> old, TFunctor functor) + : Old_(std::move(old)) + , Functor_(std::move(functor)) + { + } - public: - TNew Fetch() override { - return Functor_(Old_->Fetch()); - } - }; +public: + TNew Fetch() override { + return Functor_(Old_->Fetch()); + } +}; - template <typename TNew, typename TOld, typename TFunctor> - class TMappingConsumer final: public IConsumer<TNew> { - private: - THolder<IConsumer<TOld>> Old_; - TFunctor Functor_; +template <typename TNew, typename TOld, typename TFunctor> +class TMappingConsumer final: public IConsumer<TNew> { +private: + THolder<IConsumer<TOld>> Old_; + TFunctor Functor_; - public: - TMappingConsumer(THolder<IConsumer<TOld>> old, TFunctor functor) - : Old_(std::move(old)) - , Functor_(std::move(functor)) +public: + TMappingConsumer(THolder<IConsumer<TOld>> old, TFunctor functor) + : Old_(std::move(old)) + , Functor_(std::move(functor)) { } - public: - void OnObject(TNew object) override { - Old_->OnObject(Functor_(object)); - } +public: + void OnObject(TNew object) override { + Old_->OnObject(Functor_(object)); + } - void OnFinish() override { - Old_->OnFinish(); - } - }; + void OnFinish() override { + Old_->OnFinish(); + } +}; - template <typename T, typename C> - class TNonOwningConsumer final: public IConsumer<T> { - private: - C Consumer_; +template <typename T, typename C> +class TNonOwningConsumer final: public IConsumer<T> { +private: + C Consumer_; - public: - explicit TNonOwningConsumer(const C& consumer) - : Consumer_(consumer) - { - } +public: + explicit TNonOwningConsumer(const C& consumer) + : Consumer_(consumer) + { + } - public: - void OnObject(T t) override { - Consumer_->OnObject(t); - } +public: + void OnObject(T t) override { + Consumer_->OnObject(t); + } - void OnFinish() override { - Consumer_->OnFinish(); - } - }; -} + void OnFinish() override { + Consumer_->OnFinish(); + } +}; +} // namespace NYql::NPureCalc::NPrivate diff --git a/yql/essentials/public/purecalc/common/ya.make.inc b/yql/essentials/public/purecalc/common/ya.make.inc index 8aaf8e7c3f7..d5c4292d9b7 100644 --- a/yql/essentials/public/purecalc/common/ya.make.inc +++ b/yql/essentials/public/purecalc/common/ya.make.inc @@ -1,3 +1,5 @@ +ENABLE(YQL_STYLE_CPP) + SRCDIR( yql/essentials/public/purecalc/common ) diff --git a/yql/essentials/public/purecalc/examples/protobuf/main.cpp b/yql/essentials/public/purecalc/examples/protobuf/main.cpp index 2cf9ff47360..4bcf19be0ae 100644 --- a/yql/essentials/public/purecalc/examples/protobuf/main.cpp +++ b/yql/essentials/public/purecalc/examples/protobuf/main.cpp @@ -97,10 +97,10 @@ void PrecompileExample(IProgramFactoryPtr factory) { } auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<TInput>(), - TProtobufOutputSpec<TOutput>(), - prg, - ETranslationMode::Mkql); + TProtobufInputSpec<TInput>(), + TProtobufOutputSpec<TOutput>(), + prg, + ETranslationMode::Mkql); auto result = program->Apply(MakeInput()); diff --git a/yql/essentials/public/purecalc/examples/protobuf/ya.make b/yql/essentials/public/purecalc/examples/protobuf/ya.make index c50a3c4af25..9589ce28ccf 100644 --- a/yql/essentials/public/purecalc/examples/protobuf/ya.make +++ b/yql/essentials/public/purecalc/examples/protobuf/ya.make @@ -1,5 +1,7 @@ PROGRAM() +ENABLE(YQL_STYLE_CPP) + SRCS( main.proto main.cpp diff --git a/yql/essentials/public/purecalc/examples/protobuf_pull_list/main.cpp b/yql/essentials/public/purecalc/examples/protobuf_pull_list/main.cpp index b3e27cec10f..793738a4336 100644 --- a/yql/essentials/public/purecalc/examples/protobuf_pull_list/main.cpp +++ b/yql/essentials/public/purecalc/examples/protobuf_pull_list/main.cpp @@ -29,8 +29,7 @@ int main() { TProtobufInputSpec<TInput>(), TProtobufOutputSpec<TOutput>(), Query, - ETranslationMode::SQL - ); + ETranslationMode::SQL); auto result = program->Apply(MakeInput()); diff --git a/yql/essentials/public/purecalc/examples/protobuf_pull_list/ya.make b/yql/essentials/public/purecalc/examples/protobuf_pull_list/ya.make index a102f5fb2ca..9eceb2893a9 100644 --- a/yql/essentials/public/purecalc/examples/protobuf_pull_list/ya.make +++ b/yql/essentials/public/purecalc/examples/protobuf_pull_list/ya.make @@ -1,5 +1,7 @@ PROGRAM() +ENABLE(YQL_STYLE_CPP) + SRCS( main.proto main.cpp diff --git a/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.cpp b/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.cpp index e194e4f22c9..0d6c309f6cc 100644 --- a/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.cpp +++ b/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.cpp @@ -10,199 +10,193 @@ namespace pb = google::protobuf; namespace NYql { - namespace NPureCalc { +namespace NPureCalc { - TProtoSchemaOptions::TProtoSchemaOptions() - : EnumPolicy(EEnumPolicy::Int32) - , ListIsOptional(false) - , EnableRecursiveRenaming(false) - { - } +TProtoSchemaOptions::TProtoSchemaOptions() + : EnumPolicy(EEnumPolicy::Int32) + , ListIsOptional(false) + , EnableRecursiveRenaming(false) +{ +} - TProtoSchemaOptions& TProtoSchemaOptions::SetEnumPolicy(EEnumPolicy policy) { - EnumPolicy = policy; - return *this; - } +TProtoSchemaOptions& TProtoSchemaOptions::SetEnumPolicy(EEnumPolicy policy) { + EnumPolicy = policy; + return *this; +} - TProtoSchemaOptions& TProtoSchemaOptions::SetListIsOptional(bool value) { - ListIsOptional = value; - return *this; - } +TProtoSchemaOptions& TProtoSchemaOptions::SetListIsOptional(bool value) { + ListIsOptional = value; + return *this; +} - TProtoSchemaOptions& TProtoSchemaOptions::SetEnableRecursiveRenaming(bool value) { - EnableRecursiveRenaming = value; - return *this; - } +TProtoSchemaOptions& TProtoSchemaOptions::SetEnableRecursiveRenaming(bool value) { + EnableRecursiveRenaming = value; + return *this; +} - TProtoSchemaOptions& TProtoSchemaOptions::SetFieldRenames( - THashMap<TString, TString> fieldRenames - ) { - FieldRenames = std::move(fieldRenames); - return *this; - } +TProtoSchemaOptions& TProtoSchemaOptions::SetFieldRenames( + THashMap<TString, TString> fieldRenames) { + FieldRenames = std::move(fieldRenames); + return *this; +} - namespace { - EEnumFormatType EnumFormatTypeWithYTFlag(const pb::FieldDescriptor& enumField, EEnumFormatType defaultEnumFormatType) { - auto flags = enumField.options().GetRepeatedExtension(NYT::flags); - for (auto flag : flags) { - if (flag == NYT::EWrapperFieldFlag::ENUM_INT) { - return EEnumFormatType::Int32; - } else if (flag == NYT::EWrapperFieldFlag::ENUM_STRING) { - return EEnumFormatType::String; - } - } - return defaultEnumFormatType; - } +namespace { +EEnumFormatType EnumFormatTypeWithYTFlag(const pb::FieldDescriptor& enumField, EEnumFormatType defaultEnumFormatType) { + auto flags = enumField.options().GetRepeatedExtension(NYT::flags); + for (auto flag : flags) { + if (flag == NYT::EWrapperFieldFlag::ENUM_INT) { + return EEnumFormatType::Int32; + } else if (flag == NYT::EWrapperFieldFlag::ENUM_STRING) { + return EEnumFormatType::String; } + } + return defaultEnumFormatType; +} +} // namespace + +EEnumFormatType EnumFormatType(const pb::FieldDescriptor& enumField, EEnumPolicy enumPolicy) { + switch (enumPolicy) { + case EEnumPolicy::Int32: + return EEnumFormatType::Int32; + case EEnumPolicy::String: + return EEnumFormatType::String; + case EEnumPolicy::YTFlagDefaultInt32: + return EnumFormatTypeWithYTFlag(enumField, EEnumFormatType::Int32); + case EEnumPolicy::YTFlagDefaultString: + return EnumFormatTypeWithYTFlag(enumField, EEnumFormatType::String); + } +} - EEnumFormatType EnumFormatType(const pb::FieldDescriptor& enumField, EEnumPolicy enumPolicy) { - switch (enumPolicy) { - case EEnumPolicy::Int32: - return EEnumFormatType::Int32; - case EEnumPolicy::String: - return EEnumFormatType::String; - case EEnumPolicy::YTFlagDefaultInt32: - return EnumFormatTypeWithYTFlag(enumField, EEnumFormatType::Int32); - case EEnumPolicy::YTFlagDefaultString: - return EnumFormatTypeWithYTFlag(enumField, EEnumFormatType::String); +namespace { +const char* FormatTypeName(const pb::FieldDescriptor* field, EEnumPolicy enumPolicy) { + switch (field->type()) { + case pb::FieldDescriptor::TYPE_DOUBLE: + return "Double"; + case pb::FieldDescriptor::TYPE_FLOAT: + return "Float"; + case pb::FieldDescriptor::TYPE_INT64: + case pb::FieldDescriptor::TYPE_SFIXED64: + case pb::FieldDescriptor::TYPE_SINT64: + return "Int64"; + case pb::FieldDescriptor::TYPE_UINT64: + case pb::FieldDescriptor::TYPE_FIXED64: + return "Uint64"; + case pb::FieldDescriptor::TYPE_INT32: + case pb::FieldDescriptor::TYPE_SFIXED32: + case pb::FieldDescriptor::TYPE_SINT32: + return "Int32"; + case pb::FieldDescriptor::TYPE_UINT32: + case pb::FieldDescriptor::TYPE_FIXED32: + return "Uint32"; + case pb::FieldDescriptor::TYPE_BOOL: + return "Bool"; + case pb::FieldDescriptor::TYPE_STRING: + return "Utf8"; + case pb::FieldDescriptor::TYPE_BYTES: + return "String"; + case pb::FieldDescriptor::TYPE_ENUM: + switch (EnumFormatType(*field, enumPolicy)) { + case EEnumFormatType::Int32: + return "Int32"; + case EEnumFormatType::String: + return "String"; } - } + default: + ythrow yexception() << "Unsupported protobuf type: " << field->type_name() + << ", field: " << field->name() << ", " << int(field->type()); + } +} +} // namespace - namespace { - const char* FormatTypeName(const pb::FieldDescriptor* field, EEnumPolicy enumPolicy) { - switch (field->type()) { - case pb::FieldDescriptor::TYPE_DOUBLE: - return "Double"; - case pb::FieldDescriptor::TYPE_FLOAT: - return "Float"; - case pb::FieldDescriptor::TYPE_INT64: - case pb::FieldDescriptor::TYPE_SFIXED64: - case pb::FieldDescriptor::TYPE_SINT64: - return "Int64"; - case pb::FieldDescriptor::TYPE_UINT64: - case pb::FieldDescriptor::TYPE_FIXED64: - return "Uint64"; - case pb::FieldDescriptor::TYPE_INT32: - case pb::FieldDescriptor::TYPE_SFIXED32: - case pb::FieldDescriptor::TYPE_SINT32: - return "Int32"; - case pb::FieldDescriptor::TYPE_UINT32: - case pb::FieldDescriptor::TYPE_FIXED32: - return "Uint32"; - case pb::FieldDescriptor::TYPE_BOOL: - return "Bool"; - case pb::FieldDescriptor::TYPE_STRING: - return "Utf8"; - case pb::FieldDescriptor::TYPE_BYTES: - return "String"; - case pb::FieldDescriptor::TYPE_ENUM: - switch (EnumFormatType(*field, enumPolicy)) { - case EEnumFormatType::Int32: - return "Int32"; - case EEnumFormatType::String: - return "String"; - } - default: - ythrow yexception() << "Unsupported protobuf type: " << field->type_name() - << ", field: " << field->name() << ", " << int(field->type()); - } - } +NYT::TNode MakeSchemaFromProto(const pb::Descriptor& descriptor, TVector<const pb::Descriptor*>& nested, const TProtoSchemaOptions& options) { + if (Find(nested, &descriptor) != nested.end()) { + TVector<TString> nestedNames; + for (const auto* d : nested) { + nestedNames.push_back(d->full_name()); } + nestedNames.push_back(descriptor.full_name()); + ythrow yexception() << Sprintf("recursive messages are not supported (%s)", + JoinStrings(nestedNames, "->").c_str()); + } + nested.push_back(&descriptor); - NYT::TNode MakeSchemaFromProto(const pb::Descriptor& descriptor, TVector<const pb::Descriptor*>& nested, const TProtoSchemaOptions& options) { - if (Find(nested, &descriptor) != nested.end()) { - TVector<TString> nestedNames; - for (const auto* d : nested) { - nestedNames.push_back(d->full_name()); - } - nestedNames.push_back(descriptor.full_name()); - ythrow yexception() << Sprintf("recursive messages are not supported (%s)", - JoinStrings(nestedNames, "->").c_str()); - } - nested.push_back(&descriptor); - - auto items = NYT::TNode::CreateList(); - for (int fieldNo = 0; fieldNo < descriptor.field_count(); ++fieldNo) { - const auto& fieldDescriptor = *descriptor.field(fieldNo); - - auto name = fieldDescriptor.name(); - if ( - auto renamePtr = options.FieldRenames.FindPtr(name); - (options.EnableRecursiveRenaming || nested.size() == 1) && renamePtr - ) { - name = *renamePtr; - } + auto items = NYT::TNode::CreateList(); + for (int fieldNo = 0; fieldNo < descriptor.field_count(); ++fieldNo) { + const auto& fieldDescriptor = *descriptor.field(fieldNo); - NYT::TNode itemType; - if (fieldDescriptor.type() == pb::FieldDescriptor::TYPE_MESSAGE) { - itemType = MakeSchemaFromProto(*fieldDescriptor.message_type(), nested, options); - } else { - itemType = NYT::TNode::CreateList(); - itemType.Add("DataType"); - itemType.Add(FormatTypeName(&fieldDescriptor, options.EnumPolicy)); - } - switch (fieldDescriptor.label()) { - case pb::FieldDescriptor::LABEL_OPTIONAL: - { - auto optionalType = NYT::TNode::CreateList(); - optionalType.Add("OptionalType"); - optionalType.Add(std::move(itemType)); - itemType = std::move(optionalType); - } - break; - case pb::FieldDescriptor::LABEL_REQUIRED: - break; - case pb::FieldDescriptor::LABEL_REPEATED: - { - auto listType = NYT::TNode::CreateList(); - listType.Add("ListType"); - listType.Add(std::move(itemType)); - itemType = std::move(listType); - if (options.ListIsOptional) { - itemType = NYT::TNode::CreateList().Add("OptionalType").Add(std::move(itemType)); - } - } - break; - default: - ythrow yexception() << "Unknown protobuf label: " << (ui32)fieldDescriptor.label() << ", field: " << name; + auto name = fieldDescriptor.name(); + if ( + auto renamePtr = options.FieldRenames.FindPtr(name); + (options.EnableRecursiveRenaming || nested.size() == 1) && renamePtr) { + name = *renamePtr; + } + + NYT::TNode itemType; + if (fieldDescriptor.type() == pb::FieldDescriptor::TYPE_MESSAGE) { + itemType = MakeSchemaFromProto(*fieldDescriptor.message_type(), nested, options); + } else { + itemType = NYT::TNode::CreateList(); + itemType.Add("DataType"); + itemType.Add(FormatTypeName(&fieldDescriptor, options.EnumPolicy)); + } + switch (fieldDescriptor.label()) { + case pb::FieldDescriptor::LABEL_OPTIONAL: { + auto optionalType = NYT::TNode::CreateList(); + optionalType.Add("OptionalType"); + optionalType.Add(std::move(itemType)); + itemType = std::move(optionalType); + } break; + case pb::FieldDescriptor::LABEL_REQUIRED: + break; + case pb::FieldDescriptor::LABEL_REPEATED: { + auto listType = NYT::TNode::CreateList(); + listType.Add("ListType"); + listType.Add(std::move(itemType)); + itemType = std::move(listType); + if (options.ListIsOptional) { + itemType = NYT::TNode::CreateList().Add("OptionalType").Add(std::move(itemType)); } + } break; + default: + ythrow yexception() << "Unknown protobuf label: " << (ui32)fieldDescriptor.label() << ", field: " << name; + } - auto itemNode = NYT::TNode::CreateList(); - itemNode.Add(name); - itemNode.Add(std::move(itemType)); + auto itemNode = NYT::TNode::CreateList(); + itemNode.Add(name); + itemNode.Add(std::move(itemType)); - items.Add(std::move(itemNode)); - } - auto root = NYT::TNode::CreateList(); - root.Add("StructType"); - root.Add(std::move(items)); + items.Add(std::move(itemNode)); + } + auto root = NYT::TNode::CreateList(); + root.Add("StructType"); + root.Add(std::move(items)); - nested.pop_back(); - return root; - } + nested.pop_back(); + return root; +} - NYT::TNode MakeSchemaFromProto(const pb::Descriptor& descriptor, const TProtoSchemaOptions& options) { - TVector<const pb::Descriptor*> nested; - return MakeSchemaFromProto(descriptor, nested, options); - } +NYT::TNode MakeSchemaFromProto(const pb::Descriptor& descriptor, const TProtoSchemaOptions& options) { + TVector<const pb::Descriptor*> nested; + return MakeSchemaFromProto(descriptor, nested, options); +} - NYT::TNode MakeVariantSchemaFromProtos(const TVector<const pb::Descriptor*>& descriptors, const TProtoSchemaOptions& options) { - Y_ENSURE(options.FieldRenames.empty(), "Renames are not supported in variant mode"); +NYT::TNode MakeVariantSchemaFromProtos(const TVector<const pb::Descriptor*>& descriptors, const TProtoSchemaOptions& options) { + Y_ENSURE(options.FieldRenames.empty(), "Renames are not supported in variant mode"); - auto tupleItems = NYT::TNode::CreateList(); - for (auto descriptor : descriptors) { - tupleItems.Add(MakeSchemaFromProto(*descriptor, options)); - } + auto tupleItems = NYT::TNode::CreateList(); + for (auto descriptor : descriptors) { + tupleItems.Add(MakeSchemaFromProto(*descriptor, options)); + } - auto tupleType = NYT::TNode::CreateList(); - tupleType.Add("TupleType"); - tupleType.Add(std::move(tupleItems)); + auto tupleType = NYT::TNode::CreateList(); + tupleType.Add("TupleType"); + tupleType.Add(std::move(tupleItems)); - auto variantType = NYT::TNode::CreateList(); - variantType.Add("VariantType"); - variantType.Add(std::move(tupleType)); + auto variantType = NYT::TNode::CreateList(); + variantType.Add("VariantType"); + variantType.Add(std::move(tupleType)); - return variantType; - } - } + return variantType; } +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.h b/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.h index bf85fc5e33e..f1511d78b7a 100644 --- a/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.h +++ b/yql/essentials/public/purecalc/helpers/protobuf/schema_from_proto.h @@ -7,55 +7,54 @@ #include <google/protobuf/descriptor.h> - namespace NYql { - namespace NPureCalc { - enum class EEnumPolicy { - Int32, - String, - YTFlagDefaultInt32, - YTFlagDefaultString - }; - - enum class EEnumFormatType { - Int32, - String - }; - - /** - * Options that customize building of struct type from protobuf descriptor. - */ - struct TProtoSchemaOptions { - public: - EEnumPolicy EnumPolicy; - bool ListIsOptional; - bool EnableRecursiveRenaming; - THashMap<TString, TString> FieldRenames; - - public: - TProtoSchemaOptions(); - - public: - TProtoSchemaOptions& SetEnumPolicy(EEnumPolicy); - - TProtoSchemaOptions& SetListIsOptional(bool); - - TProtoSchemaOptions& SetEnableRecursiveRenaming(bool); - - TProtoSchemaOptions& SetFieldRenames(THashMap<TString, TString>); - }; - - EEnumFormatType EnumFormatType(const google::protobuf::FieldDescriptor& enumField, EEnumPolicy enumPolicy); - - /** - * Build struct type from a protobuf descriptor. The returned yson can be loaded into a struct annotation node - * using the ParseTypeFromYson function. - */ - NYT::TNode MakeSchemaFromProto(const google::protobuf::Descriptor&, const TProtoSchemaOptions& = {}); - - /** - * Build variant over tuple type from protobuf descriptors. - */ - NYT::TNode MakeVariantSchemaFromProtos(const TVector<const google::protobuf::Descriptor*>&, const TProtoSchemaOptions& = {}); - } -} +namespace NPureCalc { +enum class EEnumPolicy { + Int32, + String, + YTFlagDefaultInt32, + YTFlagDefaultString +}; + +enum class EEnumFormatType { + Int32, + String +}; + +/** + * Options that customize building of struct type from protobuf descriptor. + */ +struct TProtoSchemaOptions { +public: + EEnumPolicy EnumPolicy; + bool ListIsOptional; + bool EnableRecursiveRenaming; + THashMap<TString, TString> FieldRenames; + +public: + TProtoSchemaOptions(); + +public: + TProtoSchemaOptions& SetEnumPolicy(EEnumPolicy); + + TProtoSchemaOptions& SetListIsOptional(bool); + + TProtoSchemaOptions& SetEnableRecursiveRenaming(bool); + + TProtoSchemaOptions& SetFieldRenames(THashMap<TString, TString>); +}; + +EEnumFormatType EnumFormatType(const google::protobuf::FieldDescriptor& enumField, EEnumPolicy enumPolicy); + +/** + * Build struct type from a protobuf descriptor. The returned yson can be loaded into a struct annotation node + * using the ParseTypeFromYson function. + */ +NYT::TNode MakeSchemaFromProto(const google::protobuf::Descriptor&, const TProtoSchemaOptions& = {}); + +/** + * Build variant over tuple type from protobuf descriptors. + */ +NYT::TNode MakeVariantSchemaFromProtos(const TVector<const google::protobuf::Descriptor*>&, const TProtoSchemaOptions& = {}); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/helpers/protobuf/ya.make b/yql/essentials/public/purecalc/helpers/protobuf/ya.make index 11300baba84..5e1b006bf92 100644 --- a/yql/essentials/public/purecalc/helpers/protobuf/ya.make +++ b/yql/essentials/public/purecalc/helpers/protobuf/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( schema_from_proto.cpp ) diff --git a/yql/essentials/public/purecalc/helpers/stream/stream_from_vector.h b/yql/essentials/public/purecalc/helpers/stream/stream_from_vector.h index a2a50558003..69bcbb1f566 100644 --- a/yql/essentials/public/purecalc/helpers/stream/stream_from_vector.h +++ b/yql/essentials/public/purecalc/helpers/stream/stream_from_vector.h @@ -3,38 +3,38 @@ #include <yql/essentials/public/purecalc/common/interface.h> namespace NYql { - namespace NPureCalc { - namespace NPrivate { - template <typename T> - class TVectorStream final: public IStream<T*> { - private: - size_t I_; - TVector<T> Data_; +namespace NPureCalc { +namespace NPrivate { +template <typename T> +class TVectorStream final: public IStream<T*> { +private: + size_t I_; + TVector<T> Data_; - public: - explicit TVectorStream(TVector<T> data) - : I_(0) - , Data_(std::move(data)) - { - } - - public: - T* Fetch() override { - if (I_ >= Data_.size()) { - return nullptr; - } else { - return &Data_[I_++]; - } - } - }; - } +public: + explicit TVectorStream(TVector<T> data) + : I_(0) + , Data_(std::move(data)) + { + } - /** - * Convert vector into a purecalc stream. - */ - template <typename T> - THolder<IStream<T*>> StreamFromVector(TVector<T> data) { - return MakeHolder<NPrivate::TVectorStream<T>>(std::move(data)); +public: + T* Fetch() override { + if (I_ >= Data_.size()) { + return nullptr; + } else { + return &Data_[I_++]; } } +}; +} // namespace NPrivate + +/** + * Convert vector into a purecalc stream. + */ +template <typename T> +THolder<IStream<T*>> StreamFromVector(TVector<T> data) { + return MakeHolder<NPrivate::TVectorStream<T>>(std::move(data)); } +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/helpers/stream/ya.make b/yql/essentials/public/purecalc/helpers/stream/ya.make index f40bb9af559..c6aa839b803 100644 --- a/yql/essentials/public/purecalc/helpers/stream/ya.make +++ b/yql/essentials/public/purecalc/helpers/stream/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( stream_from_vector.cpp ) diff --git a/yql/essentials/public/purecalc/io_specs/arrow/spec.cpp b/yql/essentials/public/purecalc/io_specs/arrow/spec.cpp index 84562ee82ce..3b717813794 100644 --- a/yql/essentials/public/purecalc/io_specs/arrow/spec.cpp +++ b/yql/essentials/public/purecalc/io_specs/arrow/spec.cpp @@ -28,8 +28,7 @@ inline TVector<THolder<T>> VectorFromHolder(THolder<T> holder) { return result; } - -class TArrowIStreamImpl : public IArrowIStream { +class TArrowIStreamImpl: public IArrowIStream { private: IArrowIStream* Underlying_; // If we own Underlying_, than Owned_ == Underlying_; @@ -59,7 +58,6 @@ public: } }; - /** * Converts input Datums to unboxed values. */ @@ -73,16 +71,15 @@ public: explicit TArrowInputConverter( const TArrowInputSpec& inputSpec, ui32 index, - IWorker* worker - ) + IWorker* worker) : Factory_(worker->GetGraph().GetHolderFactory()) { const NYT::TNode& inputSchema = inputSpec.GetSchema(index); // Deduce the schema from the input MKQL type, if no is // provided by <inputSpec>. const NYT::TNode& schema = inputSchema.IsEntity() - ? worker->MakeInputSchema(index) - : inputSchema; + ? worker->MakeInputSchema(index) + : inputSchema; const auto* type = worker->GetRawInputType(index); @@ -118,7 +115,6 @@ public: } }; - /** * Converts unboxed values to output Datums (single-output program case). */ @@ -132,8 +128,7 @@ protected: public: explicit TArrowOutputConverter( const TArrowOutputSpec& outputSpec, - IWorker* worker - ) + IWorker* worker) : Factory_(worker->GetGraph().GetHolderFactory()) { Batch_.Reset(new arrow::compute::ExecBatch); @@ -142,8 +137,8 @@ public: // Deduce the schema from the output MKQL type, if no is // provided by <outputSpec>. const NYT::TNode& schema = outputSchema.IsEntity() - ? worker->MakeOutputSchema() - : outputSchema; + ? worker->MakeOutputSchema() + : outputSchema; const auto* type = worker->GetRawOutputType(); @@ -194,7 +189,6 @@ public: } }; - /** * List (or, better, stream) of unboxed values. * Used as an input value in pull workers. @@ -213,13 +207,12 @@ public: const TArrowInputSpec& inputSpec, ui32 index, THolder<IArrowIStream> underlying, - IWorker* worker - ) - : TCustomListValue(memInfo) - , Underlying_(std::move(underlying)) - , Worker_(worker) - , Converter_(inputSpec, index, Worker_) - , ScopedAlloc_(Worker_->GetScopedAlloc()) + IWorker* worker) + : TCustomListValue(memInfo) + , Underlying_(std::move(underlying)) + , Worker_(worker) + , Converter_(inputSpec, index, Worker_) + , ScopedAlloc_(Worker_->GetScopedAlloc()) { } @@ -273,7 +266,6 @@ public: } }; - /** * Arrow input stream for unboxed value lists. */ @@ -285,8 +277,7 @@ protected: public: explicit TArrowListImpl( const TArrowOutputSpec& outputSpec, - TWorkerHolder<IPullListWorker> worker - ) + TWorkerHolder<IPullListWorker> worker) : WorkerHolder_(std::move(worker)) , Converter_(outputSpec, WorkerHolder_.Get()) { @@ -295,7 +286,7 @@ public: OutputItemType Fetch() override { TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - with_lock(WorkerHolder_->GetScopedAlloc()) { + with_lock (WorkerHolder_->GetScopedAlloc()) { TUnboxedValue value; if (!WorkerHolder_->GetOutputIterator().Next(value)) { @@ -307,7 +298,6 @@ public: } }; - /** * Arrow input stream for unboxed value streams. */ @@ -326,7 +316,7 @@ public: OutputItemType Fetch() override { TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - with_lock(WorkerHolder_->GetScopedAlloc()) { + with_lock (WorkerHolder_->GetScopedAlloc()) { TUnboxedValue value; auto status = WorkerHolder_->GetOutput().Fetch(value); @@ -341,7 +331,6 @@ public: } }; - /** * Consumer which converts Datums to unboxed values and relays them to the * worker. Used as a return value of the push processor's Process function. @@ -354,8 +343,7 @@ private: public: explicit TArrowConsumerImpl( const TArrowInputSpec& inputSpec, - TWorkerHolder<IPushStreamWorker> worker - ) + TWorkerHolder<IPushStreamWorker> worker) : TArrowConsumerImpl(inputSpec, 0, std::move(worker)) { } @@ -363,8 +351,7 @@ public: explicit TArrowConsumerImpl( const TArrowInputSpec& inputSpec, ui32 index, - TWorkerHolder<IPushStreamWorker> worker - ) + TWorkerHolder<IPushStreamWorker> worker) : WorkerHolder_(std::move(worker)) , Converter_(inputSpec, index, WorkerHolder_.Get()) { @@ -373,7 +360,7 @@ public: void OnObject(arrow::compute::ExecBatch* batch) override { TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - with_lock(WorkerHolder_->GetScopedAlloc()) { + with_lock (WorkerHolder_->GetScopedAlloc()) { TUnboxedValue result; Converter_.DoConvert(batch, result); WorkerHolder_->Push(std::move(result)); @@ -383,13 +370,12 @@ public: void OnFinish() override { TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - with_lock(WorkerHolder_->GetScopedAlloc()) { + with_lock (WorkerHolder_->GetScopedAlloc()) { WorkerHolder_->OnFinish(); } } }; - /** * Push relay used to convert generated unboxed value to a Datum and push it to * the user's consumer. @@ -404,8 +390,7 @@ public: TArrowPushRelayImpl( const TArrowOutputSpec& outputSpec, IPushStreamWorker* worker, - THolder<IConsumer<OutputItemType>> underlying - ) + THolder<IConsumer<OutputItemType>> underlying) : Underlying_(std::move(underlying)) , Worker_(worker) , Converter_(outputSpec, Worker_) @@ -429,15 +414,13 @@ public: } }; - template <typename TWorker> void PrepareWorkerImpl(const TArrowInputSpec& inputSpec, TWorker* worker, - TVector<THolder<TArrowIStreamImpl>>&& streams -) { + TVector<THolder<TArrowIStreamImpl>>&& streams) { YQL_ENSURE(worker->GetInputsCount() == streams.size(), - "number of input streams should match number of inputs provided by spec"); + "number of input streams should match number of inputs provided by spec"); - with_lock(worker->GetScopedAlloc()) { + with_lock (worker->GetScopedAlloc()) { auto& holderFactory = worker->GetGraph().GetHolderFactory(); for (ui32 i = 0; i < streams.size(); i++) { auto input = holderFactory.template Create<TArrowListValue>( @@ -449,7 +432,6 @@ void PrepareWorkerImpl(const TArrowInputSpec& inputSpec, TWorker* worker, } // namespace - TArrowInputSpec::TArrowInputSpec(const TVector<NYT::TNode>& schemas) : Schemas_(schemas) { @@ -465,16 +447,14 @@ const NYT::TNode& TArrowInputSpec::GetSchema(ui32 index) const { void TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( const TArrowInputSpec& inputSpec, IPullListWorker* worker, - IArrowIStream* stream -) { + IArrowIStream* stream) { TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( inputSpec, worker, TVector<IArrowIStream*>({stream})); } void TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( const TArrowInputSpec& inputSpec, IPullListWorker* worker, - const TVector<IArrowIStream*>& streams -) { + const TVector<IArrowIStream*>& streams) { TVector<THolder<TArrowIStreamImpl>> wrappers; for (ui32 i = 0; i < streams.size(); i++) { wrappers.push_back(MakeHolder<TArrowIStreamImpl>(streams[i])); @@ -484,16 +464,14 @@ void TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( void TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( const TArrowInputSpec& inputSpec, IPullListWorker* worker, - THolder<IArrowIStream> stream -) { + THolder<IArrowIStream> stream) { TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker(inputSpec, worker, - VectorFromHolder<IArrowIStream>(std::move(stream))); + VectorFromHolder<IArrowIStream>(std::move(stream))); } void TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( const TArrowInputSpec& inputSpec, IPullListWorker* worker, - TVector<THolder<IArrowIStream>>&& streams -) { + TVector<THolder<IArrowIStream>>&& streams) { TVector<THolder<TArrowIStreamImpl>> wrappers; for (ui32 i = 0; i < streams.size(); i++) { wrappers.push_back(MakeHolder<TArrowIStreamImpl>(std::move(streams[i]))); @@ -501,19 +479,16 @@ void TInputSpecTraits<TArrowInputSpec>::PreparePullListWorker( PrepareWorkerImpl(inputSpec, worker, std::move(wrappers)); } - void TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( const TArrowInputSpec& inputSpec, IPullStreamWorker* worker, - IArrowIStream* stream -) { + IArrowIStream* stream) { TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( inputSpec, worker, TVector<IArrowIStream*>({stream})); } void TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( const TArrowInputSpec& inputSpec, IPullStreamWorker* worker, - const TVector<IArrowIStream*>& streams -) { + const TVector<IArrowIStream*>& streams) { TVector<THolder<TArrowIStreamImpl>> wrappers; for (ui32 i = 0; i < streams.size(); i++) { wrappers.push_back(MakeHolder<TArrowIStreamImpl>(streams[i])); @@ -523,16 +498,14 @@ void TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( void TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( const TArrowInputSpec& inputSpec, IPullStreamWorker* worker, - THolder<IArrowIStream> stream -) { + THolder<IArrowIStream> stream) { TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( inputSpec, worker, VectorFromHolder<IArrowIStream>(std::move(stream))); } void TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( const TArrowInputSpec& inputSpec, IPullStreamWorker* worker, - TVector<THolder<IArrowIStream>>&& streams -) { + TVector<THolder<IArrowIStream>>&& streams) { TVector<THolder<TArrowIStreamImpl>> wrappers; for (ui32 i = 0; i < streams.size(); i++) { wrappers.push_back(MakeHolder<TArrowIStreamImpl>(std::move(streams[i]))); @@ -540,14 +513,11 @@ void TInputSpecTraits<TArrowInputSpec>::PreparePullStreamWorker( PrepareWorkerImpl(inputSpec, worker, std::move(wrappers)); } - ConsumerType TInputSpecTraits<TArrowInputSpec>::MakeConsumer( - const TArrowInputSpec& inputSpec, TWorkerHolder<IPushStreamWorker> worker -) { + const TArrowInputSpec& inputSpec, TWorkerHolder<IPushStreamWorker> worker) { return MakeHolder<TArrowConsumerImpl>(inputSpec, std::move(worker)); } - TArrowOutputSpec::TArrowOutputSpec(const NYT::TNode& schema) : Schema_(schema) { @@ -557,22 +527,18 @@ const NYT::TNode& TArrowOutputSpec::GetSchema() const { return Schema_; } - PullListReturnType TOutputSpecTraits<TArrowOutputSpec>::ConvertPullListWorkerToOutputType( - const TArrowOutputSpec& outputSpec, TWorkerHolder<IPullListWorker> worker -) { + const TArrowOutputSpec& outputSpec, TWorkerHolder<IPullListWorker> worker) { return MakeHolder<TArrowListImpl>(outputSpec, std::move(worker)); } PullStreamReturnType TOutputSpecTraits<TArrowOutputSpec>::ConvertPullStreamWorkerToOutputType( - const TArrowOutputSpec& outputSpec, TWorkerHolder<IPullStreamWorker> worker -) { + const TArrowOutputSpec& outputSpec, TWorkerHolder<IPullStreamWorker> worker) { return MakeHolder<TArrowStreamImpl>(outputSpec, std::move(worker)); } void TOutputSpecTraits<TArrowOutputSpec>::SetConsumerToWorker( const TArrowOutputSpec& outputSpec, IPushStreamWorker* worker, - THolder<IConsumer<TOutputItemType>> consumer -) { + THolder<IConsumer<TOutputItemType>> consumer) { worker->SetConsumer(MakeHolder<TArrowPushRelayImpl>(outputSpec, worker, std::move(consumer))); } diff --git a/yql/essentials/public/purecalc/io_specs/arrow/spec.h b/yql/essentials/public/purecalc/io_specs/arrow/spec.h index 42780b1a376..0b48321539d 100644 --- a/yql/essentials/public/purecalc/io_specs/arrow/spec.h +++ b/yql/essentials/public/purecalc/io_specs/arrow/spec.h @@ -37,7 +37,9 @@ public: explicit TArrowInputSpec(const TVector<NYT::TNode>& schemas); const TVector<NYT::TNode>& GetSchemas() const override; const NYT::TNode& GetSchema(ui32 index) const; - bool ProvidesBlocks() const override { return true; } + bool ProvidesBlocks() const override { + return true; + } }; /** @@ -70,7 +72,9 @@ private: public: explicit TArrowOutputSpec(const NYT::TNode& schema); const NYT::TNode& GetSchema() const override; - bool AcceptsBlocks() const override { return true; } + bool AcceptsBlocks() const override { + return true; + } }; template <> @@ -86,22 +90,22 @@ struct TInputSpecTraits<TArrowInputSpec> { using TConsumerType = THolder<IConsumer<TInputItemType>>; static void PreparePullListWorker(const TArrowInputSpec&, IPullListWorker*, - IInputStream*); + IInputStream*); static void PreparePullListWorker(const TArrowInputSpec&, IPullListWorker*, - THolder<IInputStream>); + THolder<IInputStream>); static void PreparePullListWorker(const TArrowInputSpec&, IPullListWorker*, - const TVector<IInputStream*>&); + const TVector<IInputStream*>&); static void PreparePullListWorker(const TArrowInputSpec&, IPullListWorker*, - TVector<THolder<IInputStream>>&&); + TVector<THolder<IInputStream>>&&); static void PreparePullStreamWorker(const TArrowInputSpec&, IPullStreamWorker*, - IInputStream*); + IInputStream*); static void PreparePullStreamWorker(const TArrowInputSpec&, IPullStreamWorker*, - THolder<IInputStream>); + THolder<IInputStream>); static void PreparePullStreamWorker(const TArrowInputSpec&, IPullStreamWorker*, - const TVector<IInputStream*>&); + const TVector<IInputStream*>&); static void PreparePullStreamWorker(const TArrowInputSpec&, IPullStreamWorker*, - TVector<THolder<IInputStream>>&&); + TVector<THolder<IInputStream>>&&); static TConsumerType MakeConsumer(const TArrowInputSpec&, TWorkerHolder<IPushStreamWorker>); }; diff --git a/yql/essentials/public/purecalc/io_specs/arrow/ut/test_spec.cpp b/yql/essentials/public/purecalc/io_specs/arrow/ut/test_spec.cpp index e9e0cb7072b..2d0d42ca21a 100644 --- a/yql/essentials/public/purecalc/io_specs/arrow/ut/test_spec.cpp +++ b/yql/essentials/public/purecalc/io_specs/arrow/ut/test_spec.cpp @@ -12,20 +12,20 @@ namespace { #define Y_UNIT_TEST_ADD_BLOCK_TEST(N, MODE) \ TCurrentTest::AddTest(#N ":BlockEngineMode=" #MODE, \ - static_cast<void (*)(NUnitTest::TTestContext&)>(&N<NYql::EBlockEngineMode::MODE>), false); - -#define Y_UNIT_TEST_BLOCKS(N) \ - template<NYql::EBlockEngineMode BlockEngineMode> \ - void N(NUnitTest::TTestContext&); \ - struct TTestRegistration##N { \ - TTestRegistration##N() { \ - Y_UNIT_TEST_ADD_BLOCK_TEST(N, Disable) \ - Y_UNIT_TEST_ADD_BLOCK_TEST(N, Auto) \ - Y_UNIT_TEST_ADD_BLOCK_TEST(N, Force) \ - } \ - }; \ - static TTestRegistration##N testRegistration##N; \ - template<NYql::EBlockEngineMode BlockEngineMode> \ + static_cast<void (*)(NUnitTest::TTestContext&)>(&N<NYql::EBlockEngineMode::MODE>), false); + +#define Y_UNIT_TEST_BLOCKS(N) \ + template <NYql::EBlockEngineMode BlockEngineMode> \ + void N(NUnitTest::TTestContext&); \ + struct TTestRegistration##N { \ + TTestRegistration##N() { \ + Y_UNIT_TEST_ADD_BLOCK_TEST(N, Disable) \ + Y_UNIT_TEST_ADD_BLOCK_TEST(N, Auto) \ + Y_UNIT_TEST_ADD_BLOCK_TEST(N, Force) \ + } \ + }; \ + static TTestRegistration##N testRegistration##N; \ + template <NYql::EBlockEngineMode BlockEngineMode> \ void N(NUnitTest::TTestContext&) NYql::NPureCalc::TProgramFactoryOptions TestOptions(NYql::EBlockEngineMode mode) { @@ -39,7 +39,6 @@ NYql::NPureCalc::TProgramFactoryOptions TestOptions(NYql::EBlockEngineMode mode) return options; } - template <typename T> struct TVectorStream: public NYql::NPureCalc::IStream<T*> { TVector<T> Data; @@ -56,8 +55,7 @@ public: } }; - -template<typename T> +template <typename T> struct TVectorConsumer: public NYql::NPureCalc::IConsumer<T*> { TVector<T>& Data; size_t Index = 0; @@ -78,15 +76,13 @@ public: } }; - using ExecBatchStreamImpl = TVectorStream<arrow::compute::ExecBatch>; using ExecBatchConsumerImpl = TVectorConsumer<arrow::compute::ExecBatch>; template <typename TBuilder> arrow::Datum MakeArrayDatumFromVector( const TVector<typename TBuilder::value_type>& data, - const TVector<bool>& valid -) { + const TVector<bool>& valid) { TBuilder builder; ARROW_OK(builder.Reserve(data.size())); ARROW_OK(builder.AppendValues(data, valid)); @@ -96,17 +92,16 @@ arrow::Datum MakeArrayDatumFromVector( template <typename TValue> TVector<TValue> MakeVectorFromArrayDatum( const arrow::Datum& datum, - const int64_t dsize -) { + const int64_t dsize) { Y_ENSURE(datum.is_array(), "ExecBatch layout doesn't respect the schema"); const auto& array = *datum.array(); Y_ENSURE(array.length == dsize, - "Array Datum size differs from the given ExecBatch size"); + "Array Datum size differs from the given ExecBatch size"); Y_ENSURE(array.GetNullCount() == 0, - "Null values conversion is not supported"); + "Null values conversion is not supported"); Y_ENSURE(array.buffers.size() == 2, - "Array Datum layout doesn't respect the schema"); + "Array Datum layout doesn't respect the schema"); const TValue* adata1 = array.GetValuesSafe<TValue>(1); return TVector<TValue>(adata1, adata1 + dsize); @@ -122,8 +117,7 @@ arrow::compute::ExecBatch MakeBatch(ui64 bsize, i64 value, ui64 init = 1) { TVector<arrow::Datum> batchArgs = { MakeArrayDatumFromVector<arrow::UInt64Builder>(data1, valid), - MakeArrayDatumFromVector<arrow::Int64Builder>(data2, valid) - }; + MakeArrayDatumFromVector<arrow::Int64Builder>(data2, valid)}; return arrow::compute::ExecBatch(std::move(batchArgs), bsize); } @@ -146,274 +140,261 @@ TVector<std::tuple<ui64, i64>> CanonBatches(const TVector<arrow::compute::ExecBa } // namespace - Y_UNIT_TEST_SUITE(TestSimplePullListArrowIO) { - Y_UNIT_TEST_BLOCKS(TestSingleInput) { - using namespace NYql::NPureCalc; - - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - - try { - auto program = factory->MakePullListProgram( - TArrowInputSpec({schema}), - TArrowOutputSpec(schema), - "SELECT * FROM Input", - ETranslationMode::SQL - ); - - const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); - const auto canonInput = CanonBatches(input); - ExecBatchStreamImpl items(input); - - auto stream = program->Apply(&items); - - TVector<arrow::compute::ExecBatch> output; - while (arrow::compute::ExecBatch* batch = stream->Fetch()) { - output.push_back(*batch); - } - const auto canonOutput = CanonBatches(output); - UNIT_ASSERT_EQUAL(canonInput, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); +Y_UNIT_TEST_BLOCKS(TestSingleInput) { + using namespace NYql::NPureCalc; + + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + + try { + auto program = factory->MakePullListProgram( + TArrowInputSpec({schema}), + TArrowOutputSpec(schema), + "SELECT * FROM Input", + ETranslationMode::SQL); + + const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); + const auto canonInput = CanonBatches(input); + ExecBatchStreamImpl items(input); + + auto stream = program->Apply(&items); + + TVector<arrow::compute::ExecBatch> output; + while (arrow::compute::ExecBatch* batch = stream->Fetch()) { + output.push_back(*batch); } + const auto canonOutput = CanonBatches(output); + UNIT_ASSERT_EQUAL(canonInput, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } +} - Y_UNIT_TEST_BLOCKS(TestMultiInput) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST_BLOCKS(TestMultiInput) { + using namespace NYql::NPureCalc; - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - try { - auto program = factory->MakePullListProgram( - TArrowInputSpec({schema, schema}), - TArrowOutputSpec(schema), - R"( + try { + auto program = factory->MakePullListProgram( + TArrowInputSpec({schema, schema}), + TArrowOutputSpec(schema), + R"( SELECT * FROM Input0 UNION ALL SELECT * FROM Input1 )", - ETranslationMode::SQL - ); - - TVector<arrow::compute::ExecBatch> inputs = { - MakeBatch(9, 19), - MakeBatch(7, 17) - }; - const auto canonInputs = CanonBatches(inputs); - - ExecBatchStreamImpl items0({inputs[0]}); - ExecBatchStreamImpl items1({inputs[1]}); - - const TVector<IStream<arrow::compute::ExecBatch*>*> items({&items0, &items1}); - - auto stream = program->Apply(items); - - TVector<arrow::compute::ExecBatch> output; - while (arrow::compute::ExecBatch* batch = stream->Fetch()) { - output.push_back(*batch); - } - const auto canonOutput = CanonBatches(output); - UNIT_ASSERT_EQUAL(canonInputs, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); + ETranslationMode::SQL); + + TVector<arrow::compute::ExecBatch> inputs = { + MakeBatch(9, 19), + MakeBatch(7, 17)}; + const auto canonInputs = CanonBatches(inputs); + + ExecBatchStreamImpl items0({inputs[0]}); + ExecBatchStreamImpl items1({inputs[1]}); + + const TVector<IStream<arrow::compute::ExecBatch*>*> items({&items0, &items1}); + + auto stream = program->Apply(items); + + TVector<arrow::compute::ExecBatch> output; + while (arrow::compute::ExecBatch* batch = stream->Fetch()) { + output.push_back(*batch); } + const auto canonOutput = CanonBatches(output); + UNIT_ASSERT_EQUAL(canonInputs, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } } - +} // Y_UNIT_TEST_SUITE(TestSimplePullListArrowIO) Y_UNIT_TEST_SUITE(TestMorePullListArrowIO) { - Y_UNIT_TEST_BLOCKS(TestInc) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST_BLOCKS(TestInc) { + using namespace NYql::NPureCalc; - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - try { - auto program = factory->MakePullListProgram( - TArrowInputSpec({schema}), - TArrowOutputSpec(schema), - R"(SELECT + try { + auto program = factory->MakePullListProgram( + TArrowInputSpec({schema}), + TArrowOutputSpec(schema), + R"(SELECT uint64 + 1 as uint64, int64 - 2 as int64, FROM Input)", - ETranslationMode::SQL - ); - - const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); - const auto canonInput = CanonBatches(input); - ExecBatchStreamImpl items(input); - - auto stream = program->Apply(&items); - - TVector<arrow::compute::ExecBatch> output; - while (arrow::compute::ExecBatch* batch = stream->Fetch()) { - output.push_back(*batch); - } - const auto canonOutput = CanonBatches(output); - const TVector<arrow::compute::ExecBatch> check({MakeBatch(9, 17, 2)}); - const auto canonCheck = CanonBatches(check); - UNIT_ASSERT_EQUAL(canonCheck, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); + ETranslationMode::SQL); + + const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); + const auto canonInput = CanonBatches(input); + ExecBatchStreamImpl items(input); + + auto stream = program->Apply(&items); + + TVector<arrow::compute::ExecBatch> output; + while (arrow::compute::ExecBatch* batch = stream->Fetch()) { + output.push_back(*batch); } + const auto canonOutput = CanonBatches(output); + const TVector<arrow::compute::ExecBatch> check({MakeBatch(9, 17, 2)}); + const auto canonCheck = CanonBatches(check); + UNIT_ASSERT_EQUAL(canonCheck, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } } - +} // Y_UNIT_TEST_SUITE(TestMorePullListArrowIO) Y_UNIT_TEST_SUITE(TestSimplePullStreamArrowIO) { - Y_UNIT_TEST_BLOCKS(TestSingleInput) { - using namespace NYql::NPureCalc; - - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - - try { - auto program = factory->MakePullStreamProgram( - TArrowInputSpec({schema}), - TArrowOutputSpec(schema), - "SELECT * FROM Input", - ETranslationMode::SQL - ); - - const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); - const auto canonInput = CanonBatches(input); - ExecBatchStreamImpl items(input); - - auto stream = program->Apply(&items); - - TVector<arrow::compute::ExecBatch> output; - while (arrow::compute::ExecBatch* batch = stream->Fetch()) { - output.push_back(*batch); - } - const auto canonOutput = CanonBatches(output); - UNIT_ASSERT_EQUAL(canonInput, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); +Y_UNIT_TEST_BLOCKS(TestSingleInput) { + using namespace NYql::NPureCalc; + + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + + try { + auto program = factory->MakePullStreamProgram( + TArrowInputSpec({schema}), + TArrowOutputSpec(schema), + "SELECT * FROM Input", + ETranslationMode::SQL); + + const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); + const auto canonInput = CanonBatches(input); + ExecBatchStreamImpl items(input); + + auto stream = program->Apply(&items); + + TVector<arrow::compute::ExecBatch> output; + while (arrow::compute::ExecBatch* batch = stream->Fetch()) { + output.push_back(*batch); } + const auto canonOutput = CanonBatches(output); + UNIT_ASSERT_EQUAL(canonInput, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } } - +} // Y_UNIT_TEST_SUITE(TestSimplePullStreamArrowIO) Y_UNIT_TEST_SUITE(TestMorePullStreamArrowIO) { - Y_UNIT_TEST_BLOCKS(TestInc) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST_BLOCKS(TestInc) { + using namespace NYql::NPureCalc; - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - try { - auto program = factory->MakePullStreamProgram( - TArrowInputSpec({schema}), - TArrowOutputSpec(schema), - R"(SELECT + try { + auto program = factory->MakePullStreamProgram( + TArrowInputSpec({schema}), + TArrowOutputSpec(schema), + R"(SELECT uint64 + 1 as uint64, int64 - 2 as int64, FROM Input)", - ETranslationMode::SQL - ); - - const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); - const auto canonInput = CanonBatches(input); - ExecBatchStreamImpl items(input); - - auto stream = program->Apply(&items); - - TVector<arrow::compute::ExecBatch> output; - while (arrow::compute::ExecBatch* batch = stream->Fetch()) { - output.push_back(*batch); - } - const auto canonOutput = CanonBatches(output); - const TVector<arrow::compute::ExecBatch> check({MakeBatch(9, 17, 2)}); - const auto canonCheck = CanonBatches(check); - UNIT_ASSERT_EQUAL(canonCheck, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); + ETranslationMode::SQL); + + const TVector<arrow::compute::ExecBatch> input({MakeBatch(9, 19)}); + const auto canonInput = CanonBatches(input); + ExecBatchStreamImpl items(input); + + auto stream = program->Apply(&items); + + TVector<arrow::compute::ExecBatch> output; + while (arrow::compute::ExecBatch* batch = stream->Fetch()) { + output.push_back(*batch); } + const auto canonOutput = CanonBatches(output); + const TVector<arrow::compute::ExecBatch> check({MakeBatch(9, 17, 2)}); + const auto canonCheck = CanonBatches(check); + UNIT_ASSERT_EQUAL(canonCheck, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } } - +} // Y_UNIT_TEST_SUITE(TestMorePullStreamArrowIO) Y_UNIT_TEST_SUITE(TestPushStreamArrowIO) { - Y_UNIT_TEST_BLOCKS(TestAllColumns) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST_BLOCKS(TestAllColumns) { + using namespace NYql::NPureCalc; - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - try { - auto program = factory->MakePushStreamProgram( - TArrowInputSpec({schema}), - TArrowOutputSpec(schema), - "SELECT * FROM Input", - ETranslationMode::SQL - ); + try { + auto program = factory->MakePushStreamProgram( + TArrowInputSpec({schema}), + TArrowOutputSpec(schema), + "SELECT * FROM Input", + ETranslationMode::SQL); - arrow::compute::ExecBatch input = MakeBatch(9, 19); - const auto canonInput = CanonBatches({input}); - TVector<arrow::compute::ExecBatch> output; + arrow::compute::ExecBatch input = MakeBatch(9, 19); + const auto canonInput = CanonBatches({input}); + TVector<arrow::compute::ExecBatch> output; - auto consumer = program->Apply(MakeHolder<ExecBatchConsumerImpl>(output)); + auto consumer = program->Apply(MakeHolder<ExecBatchConsumerImpl>(output)); - UNIT_ASSERT_NO_EXCEPTION([&](){ consumer->OnObject(&input); }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ consumer->OnFinish(); }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { consumer->OnObject(&input); }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { consumer->OnFinish(); }()); - const auto canonOutput = CanonBatches(output); - UNIT_ASSERT_EQUAL(canonInput, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); - } + const auto canonOutput = CanonBatches(output); + UNIT_ASSERT_EQUAL(canonInput, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } } +} // Y_UNIT_TEST_SUITE(TestPushStreamArrowIO) Y_UNIT_TEST_SUITE(TestMorePushStreamArrowIO) { - Y_UNIT_TEST_BLOCKS(TestInc) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST_BLOCKS(TestInc) { + using namespace NYql::NPureCalc; - TVector<TString> fields = {"uint64", "int64"}; - auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); + TVector<TString> fields = {"uint64", "int64"}; + auto schema = NYql::NPureCalc::NPrivate::GetSchema(fields); - auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); + auto factory = MakeProgramFactory(TestOptions(BlockEngineMode)); - try { - auto program = factory->MakePushStreamProgram( - TArrowInputSpec({schema}), - TArrowOutputSpec(schema), - R"(SELECT + try { + auto program = factory->MakePushStreamProgram( + TArrowInputSpec({schema}), + TArrowOutputSpec(schema), + R"(SELECT uint64 + 1 as uint64, int64 - 2 as int64, FROM Input)", - ETranslationMode::SQL - ); + ETranslationMode::SQL); - arrow::compute::ExecBatch input = MakeBatch(9, 19); - const auto canonInput = CanonBatches({input}); - TVector<arrow::compute::ExecBatch> output; + arrow::compute::ExecBatch input = MakeBatch(9, 19); + const auto canonInput = CanonBatches({input}); + TVector<arrow::compute::ExecBatch> output; - auto consumer = program->Apply(MakeHolder<ExecBatchConsumerImpl>(output)); + auto consumer = program->Apply(MakeHolder<ExecBatchConsumerImpl>(output)); - UNIT_ASSERT_NO_EXCEPTION([&](){ consumer->OnObject(&input); }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ consumer->OnFinish(); }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { consumer->OnObject(&input); }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { consumer->OnFinish(); }()); - const auto canonOutput = CanonBatches(output); - const TVector<arrow::compute::ExecBatch> check({MakeBatch(9, 17, 2)}); - const auto canonCheck = CanonBatches(check); - UNIT_ASSERT_EQUAL(canonCheck, canonOutput); - } catch (const TCompileError& error) { - UNIT_FAIL(error.GetIssues()); - } + const auto canonOutput = CanonBatches(output); + const TVector<arrow::compute::ExecBatch> check({MakeBatch(9, 17, 2)}); + const auto canonCheck = CanonBatches(check); + UNIT_ASSERT_EQUAL(canonCheck, canonOutput); + } catch (const TCompileError& error) { + UNIT_FAIL(error.GetIssues()); } } +} // Y_UNIT_TEST_SUITE(TestMorePushStreamArrowIO) diff --git a/yql/essentials/public/purecalc/io_specs/arrow/ut/ya.make b/yql/essentials/public/purecalc/io_specs/arrow/ut/ya.make index ad7eb5881f5..6e50cdd16d1 100644 --- a/yql/essentials/public/purecalc/io_specs/arrow/ut/ya.make +++ b/yql/essentials/public/purecalc/io_specs/arrow/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST() +ENABLE(YQL_STYLE_CPP) + SIZE(MEDIUM) TIMEOUT(300) diff --git a/yql/essentials/public/purecalc/io_specs/arrow/ya.make.inc b/yql/essentials/public/purecalc/io_specs/arrow/ya.make.inc index 37ff3be849e..ef958d3a24a 100644 --- a/yql/essentials/public/purecalc/io_specs/arrow/ya.make.inc +++ b/yql/essentials/public/purecalc/io_specs/arrow/ya.make.inc @@ -1,3 +1,5 @@ +ENABLE(YQL_STYLE_CPP) + SRCDIR( yql/essentials/public/purecalc/io_specs/arrow ) diff --git a/yql/essentials/public/purecalc/io_specs/protobuf/proto_variant.h b/yql/essentials/public/purecalc/io_specs/protobuf/proto_variant.h index 0692440ca1c..6f1703f01b6 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf/proto_variant.h +++ b/yql/essentials/public/purecalc/io_specs/protobuf/proto_variant.h @@ -5,76 +5,76 @@ #include <array> namespace NYql::NPureCalc::NPrivate { - using TProtoRawMultiOutput = std::pair<ui32, google::protobuf::Message*>; +using TProtoRawMultiOutput = std::pair<ui32, google::protobuf::Message*>; - template <typename... T> - using TProtoMultiOutput = std::variant<T*...>; +template <typename... T> +using TProtoMultiOutput = std::variant<T*...>; - template <size_t I, typename... T> - using TProtoOutput = std::add_pointer_t<typename TTypeList<T...>::template TGet<I>>; +template <size_t I, typename... T> +using TProtoOutput = std::add_pointer_t<typename TTypeList<T...>::template TGet<I>>; - template <size_t I, typename... T> - TProtoMultiOutput<T...> InitProtobufsVariant(google::protobuf::Message* ptr) { - static_assert(std::conjunction_v<std::is_base_of<google::protobuf::Message, T>...>); - return TProtoMultiOutput<T...>(std::in_place_index<I>, static_cast<TProtoOutput<I, T...>>(ptr)); +template <size_t I, typename... T> +TProtoMultiOutput<T...> InitProtobufsVariant(google::protobuf::Message* ptr) { + static_assert(std::conjunction_v<std::is_base_of<google::protobuf::Message, T>...>); + return TProtoMultiOutput<T...>(std::in_place_index<I>, static_cast<TProtoOutput<I, T...>>(ptr)); +} + +template <typename... T> +class TProtobufsMappingBase { +public: + TProtobufsMappingBase() + : InitFuncs_(BuildInitFuncs(std::make_index_sequence<sizeof...(T)>())) + { } - template <typename... T> - class TProtobufsMappingBase { - public: - TProtobufsMappingBase() - : InitFuncs_(BuildInitFuncs(std::make_index_sequence<sizeof...(T)>())) - { - } - - private: - typedef TProtoMultiOutput<T...> (*initfunc)(google::protobuf::Message*); - - template <size_t... I> - inline std::array<initfunc, sizeof...(T)> BuildInitFuncs(std::index_sequence<I...>) { - return {&InitProtobufsVariant<I, T...>...}; - } - - protected: - const std::array<initfunc, sizeof...(T)> InitFuncs_; - }; - - template <typename... T> - class TProtobufsMappingStream: public IStream<TProtoMultiOutput<T...>>, public TProtobufsMappingBase<T...> { - public: - TProtobufsMappingStream(THolder<IStream<TProtoRawMultiOutput>> oldStream) - : OldStream_(std::move(oldStream)) - { - } - - public: - TProtoMultiOutput<T...> Fetch() override { - auto&& oldItem = OldStream_->Fetch(); - return this->InitFuncs_[oldItem.first](oldItem.second); - } - - private: - THolder<IStream<TProtoRawMultiOutput>> OldStream_; - }; - - template <typename... T> - class TProtobufsMappingConsumer: public IConsumer<TProtoRawMultiOutput>, public TProtobufsMappingBase<T...> { - public: - TProtobufsMappingConsumer(THolder<IConsumer<TProtoMultiOutput<T...>>> oldConsumer) - : OldConsumer_(std::move(oldConsumer)) - { - } - - public: - void OnObject(TProtoRawMultiOutput oldItem) override { - OldConsumer_->OnObject(this->InitFuncs_[oldItem.first](oldItem.second)); - } - - void OnFinish() override { - OldConsumer_->OnFinish(); - } - - private: - THolder<IConsumer<TProtoMultiOutput<T...>>> OldConsumer_; - }; -} +private: + typedef TProtoMultiOutput<T...> (*initfunc)(google::protobuf::Message*); + + template <size_t... I> + inline std::array<initfunc, sizeof...(T)> BuildInitFuncs(std::index_sequence<I...>) { + return {&InitProtobufsVariant<I, T...>...}; + } + +protected: + const std::array<initfunc, sizeof...(T)> InitFuncs_; +}; + +template <typename... T> +class TProtobufsMappingStream: public IStream<TProtoMultiOutput<T...>>, public TProtobufsMappingBase<T...> { +public: + TProtobufsMappingStream(THolder<IStream<TProtoRawMultiOutput>> oldStream) + : OldStream_(std::move(oldStream)) + { + } + +public: + TProtoMultiOutput<T...> Fetch() override { + auto&& oldItem = OldStream_->Fetch(); + return this->InitFuncs_[oldItem.first](oldItem.second); + } + +private: + THolder<IStream<TProtoRawMultiOutput>> OldStream_; +}; + +template <typename... T> +class TProtobufsMappingConsumer: public IConsumer<TProtoRawMultiOutput>, public TProtobufsMappingBase<T...> { +public: + TProtobufsMappingConsumer(THolder<IConsumer<TProtoMultiOutput<T...>>> oldConsumer) + : OldConsumer_(std::move(oldConsumer)) + { + } + +public: + void OnObject(TProtoRawMultiOutput oldItem) override { + OldConsumer_->OnObject(this->InitFuncs_[oldItem.first](oldItem.second)); + } + + void OnFinish() override { + OldConsumer_->OnFinish(); + } + +private: + THolder<IConsumer<TProtoMultiOutput<T...>>> OldConsumer_; +}; +} // namespace NYql::NPureCalc::NPrivate diff --git a/yql/essentials/public/purecalc/io_specs/protobuf/spec.h b/yql/essentials/public/purecalc/io_specs/protobuf/spec.h index 0e1a97f632a..f2e45fc8c74 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf/spec.h +++ b/yql/essentials/public/purecalc/io_specs/protobuf/spec.h @@ -5,143 +5,143 @@ #include <yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.h> namespace NYql { - namespace NPureCalc { - /** - * Processing mode for working with non-raw protobuf messages. - * - * @tparam T message type. - */ - template <typename T> - class TProtobufInputSpec: public TProtobufRawInputSpec { - static_assert(std::is_base_of<google::protobuf::Message, T>::value, - "should be derived from google::protobuf::Message"); - public: - TProtobufInputSpec( - const TMaybe<TString>& timestampColumn = Nothing(), - const TProtoSchemaOptions& options = {} - ) - : TProtobufRawInputSpec(*T::descriptor(), timestampColumn, options) - { - } - }; - - /** - * Processing mode for working with non-raw protobuf messages. - * - * @tparam T message type. - */ - template <typename T> - class TProtobufOutputSpec: public TProtobufRawOutputSpec { - static_assert(std::is_base_of<google::protobuf::Message, T>::value, - "should be derived from google::protobuf::Message"); - public: - TProtobufOutputSpec( - const TProtoSchemaOptions& options = {}, - google::protobuf::Arena* arena = nullptr - ) - : TProtobufRawOutputSpec(*T::descriptor(), nullptr, options, arena) - { - } - }; - - /** - * Processing mode for working with non-raw protobuf messages and several outputs. - */ - template <typename... T> - class TProtobufMultiOutputSpec: public TProtobufRawMultiOutputSpec { - static_assert( - std::conjunction_v<std::is_base_of<google::protobuf::Message, T>...>, - "all types should be derived from google::protobuf::Message"); - public: - TProtobufMultiOutputSpec( - const TProtoSchemaOptions& options = {}, - TMaybe<TVector<google::protobuf::Arena*>> arenas = {} - ) - : TProtobufRawMultiOutputSpec({T::descriptor()...}, Nothing(), options, std::move(arenas)) - { - } - }; - - template <typename T> - struct TInputSpecTraits<TProtobufInputSpec<T>> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = true; - static const constexpr bool SupportPullListMode = true; - static const constexpr bool SupportPushStreamMode = true; - - using TConsumerType = THolder<IConsumer<T*>>; - - static void PreparePullStreamWorker(const TProtobufInputSpec<T>& inputSpec, IPullStreamWorker* worker, THolder<IStream<T*>> stream) { - auto raw = ConvertStream<google::protobuf::Message*>(std::move(stream)); - TInputSpecTraits<TProtobufRawInputSpec>::PreparePullStreamWorker(inputSpec, worker, std::move(raw)); - } - - static void PreparePullListWorker(const TProtobufInputSpec<T>& inputSpec, IPullListWorker* worker, THolder<IStream<T*>> stream) { - auto raw = ConvertStream<google::protobuf::Message*>(std::move(stream)); - TInputSpecTraits<TProtobufRawInputSpec>::PreparePullListWorker(inputSpec, worker, std::move(raw)); - } - - static TConsumerType MakeConsumer(const TProtobufInputSpec<T>& inputSpec, TWorkerHolder<IPushStreamWorker> worker) { - auto raw = TInputSpecTraits<TProtobufRawInputSpec>::MakeConsumer(inputSpec, std::move(worker)); - return ConvertConsumer<T*>(std::move(raw)); - } - }; - - template <typename T> - struct TOutputSpecTraits<TProtobufOutputSpec<T>> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = true; - static const constexpr bool SupportPullListMode = true; - static const constexpr bool SupportPushStreamMode = true; - - using TOutputItemType = T*; - using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; - using TPullListReturnType = THolder<IStream<TOutputItemType>>; - - static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufOutputSpec<T>& outputSpec, TWorkerHolder<IPullStreamWorker> worker) { - auto raw = TOutputSpecTraits<TProtobufRawOutputSpec>::ConvertPullStreamWorkerToOutputType(outputSpec, std::move(worker)); - return ConvertStreamUnsafe<TOutputItemType>(std::move(raw)); - } - - static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufOutputSpec<T>& outputSpec, TWorkerHolder<IPullListWorker> worker) { - auto raw = TOutputSpecTraits<TProtobufRawOutputSpec>::ConvertPullListWorkerToOutputType(outputSpec, std::move(worker)); - return ConvertStreamUnsafe<TOutputItemType>(std::move(raw)); - } - - static void SetConsumerToWorker(const TProtobufOutputSpec<T>& outputSpec, IPushStreamWorker* worker, THolder<IConsumer<T*>> consumer) { - auto raw = ConvertConsumerUnsafe<google::protobuf::Message*>(std::move(consumer)); - TOutputSpecTraits<TProtobufRawOutputSpec>::SetConsumerToWorker(outputSpec, worker, std::move(raw)); - } - }; - - template <typename... T> - struct TOutputSpecTraits<TProtobufMultiOutputSpec<T...>> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = true; - static const constexpr bool SupportPullListMode = true; - static const constexpr bool SupportPushStreamMode = true; - - using TOutputItemType = std::variant<T*...>; - using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; - using TPullListReturnType = THolder<IStream<TOutputItemType>>; - - static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufMultiOutputSpec<T...>& outputSpec, TWorkerHolder<IPullStreamWorker> worker) { - auto raw = TOutputSpecTraits<TProtobufRawMultiOutputSpec>::ConvertPullStreamWorkerToOutputType(outputSpec, std::move(worker)); - return THolder(new NPrivate::TProtobufsMappingStream<T...>(std::move(raw))); - } - - static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufMultiOutputSpec<T...>& outputSpec, TWorkerHolder<IPullListWorker> worker) { - auto raw = TOutputSpecTraits<TProtobufRawMultiOutputSpec>::ConvertPullListWorkerToOutputType(outputSpec, std::move(worker)); - return THolder(new NPrivate::TProtobufsMappingStream<T...>(std::move(raw))); - } - - static void SetConsumerToWorker(const TProtobufMultiOutputSpec<T...>& outputSpec, IPushStreamWorker* worker, THolder<IConsumer<TOutputItemType>> consumer) { - auto wrapper = MakeHolder<NPrivate::TProtobufsMappingConsumer<T...>>(std::move(consumer)); - TOutputSpecTraits<TProtobufRawMultiOutputSpec>::SetConsumerToWorker(outputSpec, worker, std::move(wrapper)); - } - }; +namespace NPureCalc { +/** + * Processing mode for working with non-raw protobuf messages. + * + * @tparam T message type. + */ +template <typename T> +class TProtobufInputSpec: public TProtobufRawInputSpec { + static_assert(std::is_base_of<google::protobuf::Message, T>::value, + "should be derived from google::protobuf::Message"); + +public: + TProtobufInputSpec( + const TMaybe<TString>& timestampColumn = Nothing(), + const TProtoSchemaOptions& options = {}) + : TProtobufRawInputSpec(*T::descriptor(), timestampColumn, options) + { } -} +}; + +/** + * Processing mode for working with non-raw protobuf messages. + * + * @tparam T message type. + */ +template <typename T> +class TProtobufOutputSpec: public TProtobufRawOutputSpec { + static_assert(std::is_base_of<google::protobuf::Message, T>::value, + "should be derived from google::protobuf::Message"); + +public: + TProtobufOutputSpec( + const TProtoSchemaOptions& options = {}, + google::protobuf::Arena* arena = nullptr) + : TProtobufRawOutputSpec(*T::descriptor(), nullptr, options, arena) + { + } +}; + +/** + * Processing mode for working with non-raw protobuf messages and several outputs. + */ +template <typename... T> +class TProtobufMultiOutputSpec: public TProtobufRawMultiOutputSpec { + static_assert( + std::conjunction_v<std::is_base_of<google::protobuf::Message, T>...>, + "all types should be derived from google::protobuf::Message"); + +public: + TProtobufMultiOutputSpec( + const TProtoSchemaOptions& options = {}, + TMaybe<TVector<google::protobuf::Arena*>> arenas = {}) + : TProtobufRawMultiOutputSpec({T::descriptor()...}, Nothing(), options, std::move(arenas)) + { + } +}; + +template <typename T> +struct TInputSpecTraits<TProtobufInputSpec<T>> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = true; + static const constexpr bool SupportPullListMode = true; + static const constexpr bool SupportPushStreamMode = true; + + using TConsumerType = THolder<IConsumer<T*>>; + + static void PreparePullStreamWorker(const TProtobufInputSpec<T>& inputSpec, IPullStreamWorker* worker, THolder<IStream<T*>> stream) { + auto raw = ConvertStream<google::protobuf::Message*>(std::move(stream)); + TInputSpecTraits<TProtobufRawInputSpec>::PreparePullStreamWorker(inputSpec, worker, std::move(raw)); + } + + static void PreparePullListWorker(const TProtobufInputSpec<T>& inputSpec, IPullListWorker* worker, THolder<IStream<T*>> stream) { + auto raw = ConvertStream<google::protobuf::Message*>(std::move(stream)); + TInputSpecTraits<TProtobufRawInputSpec>::PreparePullListWorker(inputSpec, worker, std::move(raw)); + } + + static TConsumerType MakeConsumer(const TProtobufInputSpec<T>& inputSpec, TWorkerHolder<IPushStreamWorker> worker) { + auto raw = TInputSpecTraits<TProtobufRawInputSpec>::MakeConsumer(inputSpec, std::move(worker)); + return ConvertConsumer<T*>(std::move(raw)); + } +}; + +template <typename T> +struct TOutputSpecTraits<TProtobufOutputSpec<T>> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = true; + static const constexpr bool SupportPullListMode = true; + static const constexpr bool SupportPushStreamMode = true; + + using TOutputItemType = T*; + using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; + using TPullListReturnType = THolder<IStream<TOutputItemType>>; + + static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufOutputSpec<T>& outputSpec, TWorkerHolder<IPullStreamWorker> worker) { + auto raw = TOutputSpecTraits<TProtobufRawOutputSpec>::ConvertPullStreamWorkerToOutputType(outputSpec, std::move(worker)); + return ConvertStreamUnsafe<TOutputItemType>(std::move(raw)); + } + + static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufOutputSpec<T>& outputSpec, TWorkerHolder<IPullListWorker> worker) { + auto raw = TOutputSpecTraits<TProtobufRawOutputSpec>::ConvertPullListWorkerToOutputType(outputSpec, std::move(worker)); + return ConvertStreamUnsafe<TOutputItemType>(std::move(raw)); + } + + static void SetConsumerToWorker(const TProtobufOutputSpec<T>& outputSpec, IPushStreamWorker* worker, THolder<IConsumer<T*>> consumer) { + auto raw = ConvertConsumerUnsafe<google::protobuf::Message*>(std::move(consumer)); + TOutputSpecTraits<TProtobufRawOutputSpec>::SetConsumerToWorker(outputSpec, worker, std::move(raw)); + } +}; + +template <typename... T> +struct TOutputSpecTraits<TProtobufMultiOutputSpec<T...>> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = true; + static const constexpr bool SupportPullListMode = true; + static const constexpr bool SupportPushStreamMode = true; + + using TOutputItemType = std::variant<T*...>; + using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; + using TPullListReturnType = THolder<IStream<TOutputItemType>>; + + static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufMultiOutputSpec<T...>& outputSpec, TWorkerHolder<IPullStreamWorker> worker) { + auto raw = TOutputSpecTraits<TProtobufRawMultiOutputSpec>::ConvertPullStreamWorkerToOutputType(outputSpec, std::move(worker)); + return THolder(new NPrivate::TProtobufsMappingStream<T...>(std::move(raw))); + } + + static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufMultiOutputSpec<T...>& outputSpec, TWorkerHolder<IPullListWorker> worker) { + auto raw = TOutputSpecTraits<TProtobufRawMultiOutputSpec>::ConvertPullListWorkerToOutputType(outputSpec, std::move(worker)); + return THolder(new NPrivate::TProtobufsMappingStream<T...>(std::move(raw))); + } + + static void SetConsumerToWorker(const TProtobufMultiOutputSpec<T...>& outputSpec, IPushStreamWorker* worker, THolder<IConsumer<TOutputItemType>> consumer) { + auto wrapper = MakeHolder<NPrivate::TProtobufsMappingConsumer<T...>>(std::move(consumer)); + TOutputSpecTraits<TProtobufRawMultiOutputSpec>::SetConsumerToWorker(outputSpec, worker, std::move(wrapper)); + } +}; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/io_specs/protobuf/ut/test_spec.cpp b/yql/essentials/public/purecalc/io_specs/protobuf/ut/test_spec.cpp index 5d479d2ef2c..981b8403754 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf/ut/test_spec.cpp +++ b/yql/essentials/public/purecalc/io_specs/protobuf/ut/test_spec.cpp @@ -7,71 +7,71 @@ #include <util/generic/xrange.h> namespace { - TMaybe<NPureCalcProto::TAllTypes> allTypesMessage; - - NPureCalcProto::TAllTypes& GetCanonicalMessage() { - if (!allTypesMessage) { - allTypesMessage = NPureCalcProto::TAllTypes(); - - allTypesMessage->SetFDouble(1); - allTypesMessage->SetFFloat(2); - allTypesMessage->SetFInt64(3); - allTypesMessage->SetFSfixed64(4); - allTypesMessage->SetFSint64(5); - allTypesMessage->SetFUint64(6); - allTypesMessage->SetFFixed64(7); - allTypesMessage->SetFInt32(8); - allTypesMessage->SetFSfixed32(9); - allTypesMessage->SetFSint32(10); - allTypesMessage->SetFUint32(11); - allTypesMessage->SetFFixed32(12); - allTypesMessage->SetFBool(true); - allTypesMessage->SetFString("asd"); - allTypesMessage->SetFBytes("dsa"); - } - - return allTypesMessage.GetRef(); +TMaybe<NPureCalcProto::TAllTypes> allTypesMessage; + +NPureCalcProto::TAllTypes& GetCanonicalMessage() { + if (!allTypesMessage) { + allTypesMessage = NPureCalcProto::TAllTypes(); + + allTypesMessage->SetFDouble(1); + allTypesMessage->SetFFloat(2); + allTypesMessage->SetFInt64(3); + allTypesMessage->SetFSfixed64(4); + allTypesMessage->SetFSint64(5); + allTypesMessage->SetFUint64(6); + allTypesMessage->SetFFixed64(7); + allTypesMessage->SetFInt32(8); + allTypesMessage->SetFSfixed32(9); + allTypesMessage->SetFSint32(10); + allTypesMessage->SetFUint32(11); + allTypesMessage->SetFFixed32(12); + allTypesMessage->SetFBool(true); + allTypesMessage->SetFString("asd"); + allTypesMessage->SetFBytes("dsa"); } - template <typename T1, typename T2> - void AssertEqualToCanonical(const T1& got, const T2& expected) { - UNIT_ASSERT_EQUAL(expected.GetFDouble(), got.GetFDouble()); - UNIT_ASSERT_EQUAL(expected.GetFFloat(), got.GetFFloat()); - UNIT_ASSERT_EQUAL(expected.GetFInt64(), got.GetFInt64()); - UNIT_ASSERT_EQUAL(expected.GetFSfixed64(), got.GetFSfixed64()); - UNIT_ASSERT_EQUAL(expected.GetFSint64(), got.GetFSint64()); - UNIT_ASSERT_EQUAL(expected.GetFUint64(), got.GetFUint64()); - UNIT_ASSERT_EQUAL(expected.GetFFixed64(), got.GetFFixed64()); - UNIT_ASSERT_EQUAL(expected.GetFInt32(), got.GetFInt32()); - UNIT_ASSERT_EQUAL(expected.GetFSfixed32(), got.GetFSfixed32()); - UNIT_ASSERT_EQUAL(expected.GetFSint32(), got.GetFSint32()); - UNIT_ASSERT_EQUAL(expected.GetFUint32(), got.GetFUint32()); - UNIT_ASSERT_EQUAL(expected.GetFFixed32(), got.GetFFixed32()); - UNIT_ASSERT_EQUAL(expected.GetFBool(), got.GetFBool()); - UNIT_ASSERT_EQUAL(expected.GetFString(), got.GetFString()); - UNIT_ASSERT_EQUAL(expected.GetFBytes(), got.GetFBytes()); - } + return allTypesMessage.GetRef(); +} - template <typename T> - void AssertEqualToCanonical(const T& got) { - AssertEqualToCanonical(got, GetCanonicalMessage()); - } +template <typename T1, typename T2> +void AssertEqualToCanonical(const T1& got, const T2& expected) { + UNIT_ASSERT_EQUAL(expected.GetFDouble(), got.GetFDouble()); + UNIT_ASSERT_EQUAL(expected.GetFFloat(), got.GetFFloat()); + UNIT_ASSERT_EQUAL(expected.GetFInt64(), got.GetFInt64()); + UNIT_ASSERT_EQUAL(expected.GetFSfixed64(), got.GetFSfixed64()); + UNIT_ASSERT_EQUAL(expected.GetFSint64(), got.GetFSint64()); + UNIT_ASSERT_EQUAL(expected.GetFUint64(), got.GetFUint64()); + UNIT_ASSERT_EQUAL(expected.GetFFixed64(), got.GetFFixed64()); + UNIT_ASSERT_EQUAL(expected.GetFInt32(), got.GetFInt32()); + UNIT_ASSERT_EQUAL(expected.GetFSfixed32(), got.GetFSfixed32()); + UNIT_ASSERT_EQUAL(expected.GetFSint32(), got.GetFSint32()); + UNIT_ASSERT_EQUAL(expected.GetFUint32(), got.GetFUint32()); + UNIT_ASSERT_EQUAL(expected.GetFFixed32(), got.GetFFixed32()); + UNIT_ASSERT_EQUAL(expected.GetFBool(), got.GetFBool()); + UNIT_ASSERT_EQUAL(expected.GetFString(), got.GetFString()); + UNIT_ASSERT_EQUAL(expected.GetFBytes(), got.GetFBytes()); +} - TString SerializeToTextFormatAsString(const google::protobuf::Message& message) { - TString result; - { - TStringOutput output(result); - SerializeToTextFormat(message, output); - } - return result; - } +template <typename T> +void AssertEqualToCanonical(const T& got) { + AssertEqualToCanonical(got, GetCanonicalMessage()); +} - template <typename T> - void AssertProtoEqual(const T& actual, const T& expected) { - UNIT_ASSERT_VALUES_EQUAL(SerializeToTextFormatAsString(actual), SerializeToTextFormatAsString(expected)); +TString SerializeToTextFormatAsString(const google::protobuf::Message& message) { + TString result; + { + TStringOutput output(result); + SerializeToTextFormat(message, output); } + return result; } +template <typename T> +void AssertProtoEqual(const T& actual, const T& expected) { + UNIT_ASSERT_VALUES_EQUAL(SerializeToTextFormatAsString(actual), SerializeToTextFormatAsString(expected)); +} +} // namespace + class TAllTypesStreamImpl: public NYql::NPureCalc::IStream<NPureCalcProto::TAllTypes*> { private: int I_ = 0; @@ -256,7 +256,7 @@ private: ui32 I_ = 0; }; -template<typename T> +template <typename T> struct TVectorConsumer: public NYql::NPureCalc::IConsumer<T*> { TVector<T> Data; @@ -280,494 +280,476 @@ public: }; Y_UNIT_TEST_SUITE(TestProtoIO) { - Y_UNIT_TEST(TestAllTypes) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestAllTypes) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(); + auto factory = MakeProgramFactory(); - { - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TAllTypes>(), - TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), - "SELECT * FROM Input", - ETranslationMode::SQL - ); + { + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TAllTypes>(), + TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), + "SELECT * FROM Input", + ETranslationMode::SQL); - auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); + auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); - NPureCalcProto::TAllTypes* message; + NPureCalcProto::TAllTypes* message; - UNIT_ASSERT(message = stream->Fetch()); - AssertEqualToCanonical(*message); - UNIT_ASSERT(!stream->Fetch()); - } + UNIT_ASSERT(message = stream->Fetch()); + AssertEqualToCanonical(*message); + UNIT_ASSERT(!stream->Fetch()); + } - { - auto program = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TAllTypes>(), - TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), - "SELECT * FROM Input", - ETranslationMode::SQL - ); + { + auto program = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TAllTypes>(), + TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), + "SELECT * FROM Input", + ETranslationMode::SQL); - auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); + auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); - NPureCalcProto::TAllTypes* message; + NPureCalcProto::TAllTypes* message; - UNIT_ASSERT(message = stream->Fetch()); - AssertEqualToCanonical(*message); - UNIT_ASSERT(!stream->Fetch()); - } + UNIT_ASSERT(message = stream->Fetch()); + AssertEqualToCanonical(*message); + UNIT_ASSERT(!stream->Fetch()); + } - { - auto program = factory->MakePushStreamProgram( - TProtobufInputSpec<NPureCalcProto::TAllTypes>(), - TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), - "SELECT * FROM Input", - ETranslationMode::SQL - ); + { + auto program = factory->MakePushStreamProgram( + TProtobufInputSpec<NPureCalcProto::TAllTypes>(), + TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), + "SELECT * FROM Input", + ETranslationMode::SQL); - auto consumer = program->Apply(MakeHolder<TAllTypesConsumerImpl>()); + auto consumer = program->Apply(MakeHolder<TAllTypesConsumerImpl>()); - UNIT_ASSERT_NO_EXCEPTION([&](){ consumer->OnObject(&GetCanonicalMessage()); }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ consumer->OnFinish(); }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { consumer->OnObject(&GetCanonicalMessage()); }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { consumer->OnFinish(); }()); } +} - template <typename T> - void CheckPassThroughYql(T& testInput, google::protobuf::Arena* arena = nullptr) { - using namespace NYql::NPureCalc; +template <typename T> +void CheckPassThroughYql(T& testInput, google::protobuf::Arena* arena = nullptr) { + using namespace NYql::NPureCalc; - auto resetArena = [arena]() { - if (arena != nullptr) { - arena->Reset(); - } - }; + auto resetArena = [arena]() { + if (arena != nullptr) { + arena->Reset(); + } + }; - auto factory = MakeProgramFactory(); + auto factory = MakeProgramFactory(); - { - auto program = factory->MakePushStreamProgram( - TProtobufInputSpec<T>(), - TProtobufOutputSpec<T>({}, arena), - "SELECT * FROM Input", - ETranslationMode::SQL - ); - - auto resultConsumer = MakeHolder<TVectorConsumer<T>>(); - auto* resultConsumerPtr = resultConsumer.Get(); - auto sourceConsumer = program->Apply(std::move(resultConsumer)); - - sourceConsumer->OnObject(&testInput); - UNIT_ASSERT_VALUES_EQUAL(1, resultConsumerPtr->Data.size()); - AssertProtoEqual(resultConsumerPtr->Data[0], testInput); - - resultConsumerPtr->Data.clear(); - sourceConsumer->OnObject(&testInput); - UNIT_ASSERT_VALUES_EQUAL(1, resultConsumerPtr->Data.size()); - AssertProtoEqual(resultConsumerPtr->Data[0], testInput); - } - resetArena(); + { + auto program = factory->MakePushStreamProgram( + TProtobufInputSpec<T>(), + TProtobufOutputSpec<T>({}, arena), + "SELECT * FROM Input", + ETranslationMode::SQL); + + auto resultConsumer = MakeHolder<TVectorConsumer<T>>(); + auto* resultConsumerPtr = resultConsumer.Get(); + auto sourceConsumer = program->Apply(std::move(resultConsumer)); + + sourceConsumer->OnObject(&testInput); + UNIT_ASSERT_VALUES_EQUAL(1, resultConsumerPtr->Data.size()); + AssertProtoEqual(resultConsumerPtr->Data[0], testInput); + + resultConsumerPtr->Data.clear(); + sourceConsumer->OnObject(&testInput); + UNIT_ASSERT_VALUES_EQUAL(1, resultConsumerPtr->Data.size()); + AssertProtoEqual(resultConsumerPtr->Data[0], testInput); + } + resetArena(); - { - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<T>(), - TProtobufOutputSpec<T>({}, arena), - "SELECT * FROM Input", - ETranslationMode::SQL - ); - - auto sourceStream = MakeHolder<TVectorStream<T>>(); - auto* sourceStreamPtr = sourceStream.Get(); - auto resultStream = program->Apply(std::move(sourceStream)); - - sourceStreamPtr->Data.push_back(testInput); - T* resultMessage; - UNIT_ASSERT(resultMessage = resultStream->Fetch()); - AssertProtoEqual(*resultMessage, testInput); - UNIT_ASSERT(!resultStream->Fetch()); - - UNIT_ASSERT_VALUES_EQUAL(resultMessage->GetArena(), arena); - } - resetArena(); + { + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<T>(), + TProtobufOutputSpec<T>({}, arena), + "SELECT * FROM Input", + ETranslationMode::SQL); + + auto sourceStream = MakeHolder<TVectorStream<T>>(); + auto* sourceStreamPtr = sourceStream.Get(); + auto resultStream = program->Apply(std::move(sourceStream)); + + sourceStreamPtr->Data.push_back(testInput); + T* resultMessage; + UNIT_ASSERT(resultMessage = resultStream->Fetch()); + AssertProtoEqual(*resultMessage, testInput); + UNIT_ASSERT(!resultStream->Fetch()); + + UNIT_ASSERT_VALUES_EQUAL(resultMessage->GetArena(), arena); + } + resetArena(); - { - auto program = factory->MakePullListProgram( - TProtobufInputSpec<T>(), - TProtobufOutputSpec<T>({}, arena), - "SELECT * FROM Input", - ETranslationMode::SQL - ); - - auto sourceStream = MakeHolder<TVectorStream<T>>(); - auto* sourceStreamPtr = sourceStream.Get(); - auto resultStream = program->Apply(std::move(sourceStream)); - - sourceStreamPtr->Data.push_back(testInput); - T* resultMessage; - UNIT_ASSERT(resultMessage = resultStream->Fetch()); - AssertProtoEqual(*resultMessage, testInput); - UNIT_ASSERT(!resultStream->Fetch()); - - UNIT_ASSERT_VALUES_EQUAL(resultMessage->GetArena(), arena); - } - resetArena(); + { + auto program = factory->MakePullListProgram( + TProtobufInputSpec<T>(), + TProtobufOutputSpec<T>({}, arena), + "SELECT * FROM Input", + ETranslationMode::SQL); + + auto sourceStream = MakeHolder<TVectorStream<T>>(); + auto* sourceStreamPtr = sourceStream.Get(); + auto resultStream = program->Apply(std::move(sourceStream)); + + sourceStreamPtr->Data.push_back(testInput); + T* resultMessage; + UNIT_ASSERT(resultMessage = resultStream->Fetch()); + AssertProtoEqual(*resultMessage, testInput); + UNIT_ASSERT(!resultStream->Fetch()); + + UNIT_ASSERT_VALUES_EQUAL(resultMessage->GetArena(), arena); } + resetArena(); +} - template <typename T> - void CheckMessageIsInvalid(const TString& expectedExceptionMessage) { - using namespace NYql::NPureCalc; +template <typename T> +void CheckMessageIsInvalid(const TString& expectedExceptionMessage) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(); + auto factory = MakeProgramFactory(); - UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { - factory->MakePushStreamProgram(TProtobufInputSpec<T>(), TProtobufOutputSpec<T>(), "SELECT * FROM Input", ETranslationMode::SQL); - }(), yexception, expectedExceptionMessage); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(TProtobufInputSpec<T>(), TProtobufOutputSpec<T>(), "SELECT * FROM Input", ETranslationMode::SQL); + }(), yexception, expectedExceptionMessage); - UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { - factory->MakePullStreamProgram(TProtobufInputSpec<T>(), TProtobufOutputSpec<T>(), "SELECT * FROM Input", ETranslationMode::SQL); - }(), yexception, expectedExceptionMessage); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(TProtobufInputSpec<T>(), TProtobufOutputSpec<T>(), "SELECT * FROM Input", ETranslationMode::SQL); + }(), yexception, expectedExceptionMessage); - UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { - factory->MakePullListProgram(TProtobufInputSpec<T>(), TProtobufOutputSpec<T>(), "SELECT * FROM Input", ETranslationMode::SQL); - }(), yexception, expectedExceptionMessage); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullListProgram(TProtobufInputSpec<T>(), TProtobufOutputSpec<T>(), "SELECT * FROM Input", ETranslationMode::SQL); + }(), yexception, expectedExceptionMessage); +} - Y_UNIT_TEST(TestSimpleNested) { - NPureCalcProto::TSimpleNested input; - input.SetX(10); - { - auto* item = input.MutableY(); - *item = GetCanonicalMessage(); - item->SetFUint64(100); - } - CheckPassThroughYql(input); +Y_UNIT_TEST(TestSimpleNested) { + NPureCalcProto::TSimpleNested input; + input.SetX(10); + { + auto* item = input.MutableY(); + *item = GetCanonicalMessage(); + item->SetFUint64(100); } + CheckPassThroughYql(input); +} - Y_UNIT_TEST(TestOptionalNested) { - NPureCalcProto::TOptionalNested input; - { - auto* item = input.MutableX(); - *item = GetCanonicalMessage(); - item->SetFUint64(100); - } - CheckPassThroughYql(input); +Y_UNIT_TEST(TestOptionalNested) { + NPureCalcProto::TOptionalNested input; + { + auto* item = input.MutableX(); + *item = GetCanonicalMessage(); + item->SetFUint64(100); } + CheckPassThroughYql(input); +} - Y_UNIT_TEST(TestSimpleRepeated) { - NPureCalcProto::TSimpleRepeated input; - input.SetX(20); - input.AddY(100); - input.AddY(200); - input.AddY(300); - CheckPassThroughYql(input); - } +Y_UNIT_TEST(TestSimpleRepeated) { + NPureCalcProto::TSimpleRepeated input; + input.SetX(20); + input.AddY(100); + input.AddY(200); + input.AddY(300); + CheckPassThroughYql(input); +} - Y_UNIT_TEST(TestNestedRepeated) { - NPureCalcProto::TNestedRepeated input; - input.SetX(20); +Y_UNIT_TEST(TestNestedRepeated) { + NPureCalcProto::TNestedRepeated input; + input.SetX(20); + { + auto* item = input.MutableY()->Add(); + item->SetX(100); { - auto* item = input.MutableY()->Add(); - item->SetX(100); - { - auto* y = item->MutableY(); - *y = GetCanonicalMessage(); - y->SetFUint64(1000); - } + auto* y = item->MutableY(); + *y = GetCanonicalMessage(); + y->SetFUint64(1000); } + } + { + auto* item = input.MutableY()->Add(); + item->SetX(200); { - auto* item = input.MutableY()->Add(); - item->SetX(200); - { - auto* y = item->MutableY(); - *y = GetCanonicalMessage(); - y->SetFUint64(2000); - } + auto* y = item->MutableY(); + *y = GetCanonicalMessage(); + y->SetFUint64(2000); } - CheckPassThroughYql(input); } + CheckPassThroughYql(input); +} - Y_UNIT_TEST(TestMessageWithEnum) { - NPureCalcProto::TMessageWithEnum input; - input.AddEnumValue(NPureCalcProto::TMessageWithEnum::VALUE1); - input.AddEnumValue(NPureCalcProto::TMessageWithEnum::VALUE2); - CheckPassThroughYql(input); - } +Y_UNIT_TEST(TestMessageWithEnum) { + NPureCalcProto::TMessageWithEnum input; + input.AddEnumValue(NPureCalcProto::TMessageWithEnum::VALUE1); + input.AddEnumValue(NPureCalcProto::TMessageWithEnum::VALUE2); + CheckPassThroughYql(input); +} - Y_UNIT_TEST(TestRecursive) { - CheckMessageIsInvalid<NPureCalcProto::TRecursive>("NPureCalcProto.TRecursive->NPureCalcProto.TRecursive"); - } +Y_UNIT_TEST(TestRecursive) { + CheckMessageIsInvalid<NPureCalcProto::TRecursive>("NPureCalcProto.TRecursive->NPureCalcProto.TRecursive"); +} - Y_UNIT_TEST(TestRecursiveIndirectly) { - CheckMessageIsInvalid<NPureCalcProto::TRecursiveIndirectly>( - "NPureCalcProto.TRecursiveIndirectly->NPureCalcProto.TRecursiveIndirectly.TNested->NPureCalcProto.TRecursiveIndirectly"); - } +Y_UNIT_TEST(TestRecursiveIndirectly) { + CheckMessageIsInvalid<NPureCalcProto::TRecursiveIndirectly>( + "NPureCalcProto.TRecursiveIndirectly->NPureCalcProto.TRecursiveIndirectly.TNested->NPureCalcProto.TRecursiveIndirectly"); +} - Y_UNIT_TEST(TestColumnsFilter) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestColumnsFilter) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(); + auto factory = MakeProgramFactory(); - auto filter = THashSet<TString>({"FFixed64", "FBool", "FBytes"}); + auto filter = THashSet<TString>({"FFixed64", "FBool", "FBytes"}); - NPureCalcProto::TOptionalAllTypes canonicalMessage; - canonicalMessage.SetFFixed64(GetCanonicalMessage().GetFFixed64()); - canonicalMessage.SetFBool(GetCanonicalMessage().GetFBool()); - canonicalMessage.SetFBytes(GetCanonicalMessage().GetFBytes()); + NPureCalcProto::TOptionalAllTypes canonicalMessage; + canonicalMessage.SetFFixed64(GetCanonicalMessage().GetFFixed64()); + canonicalMessage.SetFBool(GetCanonicalMessage().GetFBool()); + canonicalMessage.SetFBytes(GetCanonicalMessage().GetFBytes()); - { - auto inputSpec = TProtobufInputSpec<NPureCalcProto::TAllTypes>(); - auto outputSpec = TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(); - outputSpec.SetOutputColumnsFilter(filter); + { + auto inputSpec = TProtobufInputSpec<NPureCalcProto::TAllTypes>(); + auto outputSpec = TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(); + outputSpec.SetOutputColumnsFilter(filter); - auto program = factory->MakePullStreamProgram( - inputSpec, - outputSpec, - "SELECT * FROM Input", - ETranslationMode::SQL - ); + auto program = factory->MakePullStreamProgram( + inputSpec, + outputSpec, + "SELECT * FROM Input", + ETranslationMode::SQL); - UNIT_ASSERT_EQUAL(program->GetUsedColumns(), filter); + UNIT_ASSERT_EQUAL(program->GetUsedColumns(), filter); - auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); + auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); - NPureCalcProto::TOptionalAllTypes* message; + NPureCalcProto::TOptionalAllTypes* message; - UNIT_ASSERT(message = stream->Fetch()); - AssertEqualToCanonical(*message, canonicalMessage); - UNIT_ASSERT(!stream->Fetch()); - } + UNIT_ASSERT(message = stream->Fetch()); + AssertEqualToCanonical(*message, canonicalMessage); + UNIT_ASSERT(!stream->Fetch()); } +} - Y_UNIT_TEST(TestColumnsFilterWithOptionalFields) { - using namespace NYql::NPureCalc; - - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestColumnsFilterWithOptionalFields) { + using namespace NYql::NPureCalc; - auto fields = THashSet<TString>({"FFixed64", "FBool", "FBytes"}); + auto factory = MakeProgramFactory(); - NPureCalcProto::TOptionalAllTypes canonicalMessage; - canonicalMessage.SetFFixed64(GetCanonicalMessage().GetFFixed64()); - canonicalMessage.SetFBool(GetCanonicalMessage().GetFBool()); - canonicalMessage.SetFBytes(GetCanonicalMessage().GetFBytes()); + auto fields = THashSet<TString>({"FFixed64", "FBool", "FBytes"}); - { - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TAllTypes>(), - TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(), - "SELECT FFixed64, FBool, FBytes FROM Input", - ETranslationMode::SQL - ); + NPureCalcProto::TOptionalAllTypes canonicalMessage; + canonicalMessage.SetFFixed64(GetCanonicalMessage().GetFFixed64()); + canonicalMessage.SetFBool(GetCanonicalMessage().GetFBool()); + canonicalMessage.SetFBytes(GetCanonicalMessage().GetFBytes()); - UNIT_ASSERT_EQUAL(program->GetUsedColumns(), fields); + { + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TAllTypes>(), + TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(), + "SELECT FFixed64, FBool, FBytes FROM Input", + ETranslationMode::SQL); - auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); + UNIT_ASSERT_EQUAL(program->GetUsedColumns(), fields); - NPureCalcProto::TOptionalAllTypes* message; + auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); - UNIT_ASSERT(message = stream->Fetch()); - AssertEqualToCanonical(*message, canonicalMessage); - UNIT_ASSERT(!stream->Fetch()); - } + NPureCalcProto::TOptionalAllTypes* message; - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TAllTypes>(), - TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), - "SELECT FFixed64, FBool, FBytes FROM Input", - ETranslationMode::SQL - ); - }(), TCompileError, "Failed to optimize"); + UNIT_ASSERT(message = stream->Fetch()); + AssertEqualToCanonical(*message, canonicalMessage); + UNIT_ASSERT(!stream->Fetch()); } - Y_UNIT_TEST(TestUsedColumns) { - using namespace NYql::NPureCalc; + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TAllTypes>(), + TProtobufOutputSpec<NPureCalcProto::TAllTypes>(), + "SELECT FFixed64, FBool, FBytes FROM Input", + ETranslationMode::SQL); + }(), TCompileError, "Failed to optimize"); +} - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestUsedColumns) { + using namespace NYql::NPureCalc; - auto allFields = THashSet<TString>(); + auto factory = MakeProgramFactory(); - for (auto i: xrange(NPureCalcProto::TOptionalAllTypes::descriptor()->field_count())) { - allFields.emplace(NPureCalcProto::TOptionalAllTypes::descriptor()->field(i)->name()); - } + auto allFields = THashSet<TString>(); - { - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TAllTypes>(), - TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(), - "SELECT * FROM Input", - ETranslationMode::SQL - ); - - UNIT_ASSERT_EQUAL(program->GetUsedColumns(), allFields); - } + for (auto i : xrange(NPureCalcProto::TOptionalAllTypes::descriptor()->field_count())) { + allFields.emplace(NPureCalcProto::TOptionalAllTypes::descriptor()->field(i)->name()); } - Y_UNIT_TEST(TestChaining) { - using namespace NYql::NPureCalc; + { + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TAllTypes>(), + TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(), + "SELECT * FROM Input", + ETranslationMode::SQL); - auto factory = MakeProgramFactory(); + UNIT_ASSERT_EQUAL(program->GetUsedColumns(), allFields); + } +} - TString sql1 = "SELECT UNWRAP(X || CAST(\"HI\" AS Utf8)) AS X FROM Input"; - TString sql2 = "SELECT LENGTH(X) AS X FROM Input"; +Y_UNIT_TEST(TestChaining) { + using namespace NYql::NPureCalc; - { - auto program1 = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql1, - ETranslationMode::SQL - ); - - auto program2 = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>(), - sql2, - ETranslationMode::SQL - ); - - auto input = MakeHolder<TStringMessageStreamImpl>(); - auto intermediate = program1->Apply(std::move(input)); - auto output = program2->Apply(std::move(intermediate)); - - TVector<int> expected = {2, 3, 4}; - TVector<int> actual{}; - - while (auto *x = output->Fetch()) { - actual.push_back(x->GetX()); - } + auto factory = MakeProgramFactory(); + + TString sql1 = "SELECT UNWRAP(X || CAST(\"HI\" AS Utf8)) AS X FROM Input"; + TString sql2 = "SELECT LENGTH(X) AS X FROM Input"; - UNIT_ASSERT_EQUAL(expected, actual); + { + auto program1 = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql1, + ETranslationMode::SQL); + + auto program2 = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>(), + sql2, + ETranslationMode::SQL); + + auto input = MakeHolder<TStringMessageStreamImpl>(); + auto intermediate = program1->Apply(std::move(input)); + auto output = program2->Apply(std::move(intermediate)); + + TVector<int> expected = {2, 3, 4}; + TVector<int> actual{}; + + while (auto* x = output->Fetch()) { + actual.push_back(x->GetX()); } - { - auto program1 = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql1, - ETranslationMode::SQL - ); - - auto program2 = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>(), - sql2, - ETranslationMode::SQL - ); - - auto input = MakeHolder<TStringMessageStreamImpl>(); - auto intermediate = program1->Apply(std::move(input)); - auto output = program2->Apply(std::move(intermediate)); - - TVector<int> expected = {2, 3, 4}; - TVector<int> actual{}; - - while (auto *x = output->Fetch()) { - actual.push_back(x->GetX()); - } + UNIT_ASSERT_EQUAL(expected, actual); + } - UNIT_ASSERT_EQUAL(expected, actual); + { + auto program1 = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql1, + ETranslationMode::SQL); + + auto program2 = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>(), + sql2, + ETranslationMode::SQL); + + auto input = MakeHolder<TStringMessageStreamImpl>(); + auto intermediate = program1->Apply(std::move(input)); + auto output = program2->Apply(std::move(intermediate)); + + TVector<int> expected = {2, 3, 4}; + TVector<int> actual{}; + + while (auto* x = output->Fetch()) { + actual.push_back(x->GetX()); } - { - auto program1 = factory->MakePushStreamProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql1, - ETranslationMode::SQL - ); + UNIT_ASSERT_EQUAL(expected, actual); + } + + { + auto program1 = factory->MakePushStreamProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql1, + ETranslationMode::SQL); - auto program2 = factory->MakePushStreamProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>(), - sql2, - ETranslationMode::SQL - ); + auto program2 = factory->MakePushStreamProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>(), + sql2, + ETranslationMode::SQL); - TVector<int> expected = {2, 3, 4, -100}; - TVector<int> actual{}; + TVector<int> expected = {2, 3, 4, -100}; + TVector<int> actual{}; - auto consumer = MakeHolder<TSimpleMessageConsumerImpl>(&actual); - auto intermediate = program2->Apply(std::move(consumer)); - auto input = program1->Apply(std::move(intermediate)); + auto consumer = MakeHolder<TSimpleMessageConsumerImpl>(&actual); + auto intermediate = program2->Apply(std::move(consumer)); + auto input = program1->Apply(std::move(intermediate)); - NPureCalcProto::TStringMessage Message; + NPureCalcProto::TStringMessage Message; - Message.SetX(""); - input->OnObject(&Message); + Message.SetX(""); + input->OnObject(&Message); - Message.SetX("1"); - input->OnObject(&Message); + Message.SetX("1"); + input->OnObject(&Message); - Message.SetX("22"); - input->OnObject(&Message); + Message.SetX("22"); + input->OnObject(&Message); - input->OnFinish(); + input->OnFinish(); - UNIT_ASSERT_EQUAL(expected, actual); - } + UNIT_ASSERT_EQUAL(expected, actual); } +} - Y_UNIT_TEST(TestTimestampColumn) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestTimestampColumn) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(TProgramFactoryOptions() - .SetDeterministicTimeProviderSeed(1)); // seconds + auto factory = MakeProgramFactory(TProgramFactoryOptions() + .SetDeterministicTimeProviderSeed(1)); // seconds - NPureCalcProto::TOptionalAllTypes canonicalMessage; + NPureCalcProto::TOptionalAllTypes canonicalMessage; - { - auto inputSpec = TProtobufInputSpec<NPureCalcProto::TAllTypes>("MyTimestamp"); - auto outputSpec = TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(); + { + auto inputSpec = TProtobufInputSpec<NPureCalcProto::TAllTypes>("MyTimestamp"); + auto outputSpec = TProtobufOutputSpec<NPureCalcProto::TOptionalAllTypes>(); - auto program = factory->MakePullStreamProgram( - inputSpec, - outputSpec, - "SELECT MyTimestamp AS FFixed64 FROM Input", - ETranslationMode::SQL - ); + auto program = factory->MakePullStreamProgram( + inputSpec, + outputSpec, + "SELECT MyTimestamp AS FFixed64 FROM Input", + ETranslationMode::SQL); - auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); + auto stream = program->Apply(MakeHolder<TAllTypesStreamImpl>()); - NPureCalcProto::TOptionalAllTypes* message; + NPureCalcProto::TOptionalAllTypes* message; - UNIT_ASSERT(message = stream->Fetch()); - UNIT_ASSERT_VALUES_EQUAL(message->GetFFixed64(), 1000000); // microseconds - UNIT_ASSERT(!stream->Fetch()); - } + UNIT_ASSERT(message = stream->Fetch()); + UNIT_ASSERT_VALUES_EQUAL(message->GetFFixed64(), 1000000); // microseconds + UNIT_ASSERT(!stream->Fetch()); } +} - Y_UNIT_TEST(TestTableNames) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestTableNames) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseSystemColumns(true)); + auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseSystemColumns(true)); - auto runTest = [&](TStringBuf tableName, i32 value) { - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TSimpleMessage>(), - TProtobufOutputSpec<NPureCalcProto::TNamedSimpleMessage>(), - TString::Join("SELECT TableName() AS Name, X FROM ", tableName), - ETranslationMode::SQL - ); + auto runTest = [&](TStringBuf tableName, i32 value) { + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TSimpleMessage>(), + TProtobufOutputSpec<NPureCalcProto::TNamedSimpleMessage>(), + TString::Join("SELECT TableName() AS Name, X FROM ", tableName), + ETranslationMode::SQL); - auto stream = program->Apply(MakeHolder<TSimpleMessageStreamImpl>(value)); - auto message = stream->Fetch(); + auto stream = program->Apply(MakeHolder<TSimpleMessageStreamImpl>(value)); + auto message = stream->Fetch(); - UNIT_ASSERT(message); - UNIT_ASSERT_VALUES_EQUAL(message->GetX(), value); - UNIT_ASSERT_VALUES_EQUAL(message->GetName(), tableName); - UNIT_ASSERT(!stream->Fetch()); - }; + UNIT_ASSERT(message); + UNIT_ASSERT_VALUES_EQUAL(message->GetX(), value); + UNIT_ASSERT_VALUES_EQUAL(message->GetName(), tableName); + UNIT_ASSERT(!stream->Fetch()); + }; - runTest("Input", 37); - runTest("Input0", -23); - } + runTest("Input", 37); + runTest("Input0", -23); +} - void CheckMultiOutputs(TMaybe<TVector<google::protobuf::Arena*>> arenas) { - using namespace NYql::NPureCalc; +void CheckMultiOutputs(TMaybe<TVector<google::protobuf::Arena*>> arenas) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(); - TString sExpr = R"( + auto factory = MakeProgramFactory(); + TString sExpr = R"( ( (let $type (ParseType '"Variant<Struct<BInt:Int32,BString:Utf8>, Struct<CUint:Uint32,CString:Utf8>, Struct<X:Utf8>>")) (let $stream (Self '0)) @@ -781,293 +763,276 @@ Y_UNIT_TEST_SUITE(TestProtoIO) { ) )"; - { - auto program = factory->MakePushStreamProgram( - TProtobufInputSpec<NPureCalcProto::TUnsplitted>(), - TProtobufMultiOutputSpec<NPureCalcProto::TSplitted1, NPureCalcProto::TSplitted2, NPureCalcProto::TStringMessage>( - {}, arenas - ), - sExpr, - ETranslationMode::SExpr - ); + { + auto program = factory->MakePushStreamProgram( + TProtobufInputSpec<NPureCalcProto::TUnsplitted>(), + TProtobufMultiOutputSpec<NPureCalcProto::TSplitted1, NPureCalcProto::TSplitted2, NPureCalcProto::TStringMessage>( + {}, arenas), + sExpr, + ETranslationMode::SExpr); - TVariantConsumerImpl::TType0 queue0; - TVariantConsumerImpl::TType1 queue1; - TVariantConsumerImpl::TType2 queue2; - int finalValue = 0; + TVariantConsumerImpl::TType0 queue0; + TVariantConsumerImpl::TType1 queue1; + TVariantConsumerImpl::TType2 queue2; + int finalValue = 0; - auto consumer = MakeHolder<TVariantConsumerImpl>(&queue0, &queue1, &queue2, &finalValue); - auto input = program->Apply(std::move(consumer)); + auto consumer = MakeHolder<TVariantConsumerImpl>(&queue0, &queue1, &queue2, &finalValue); + auto input = program->Apply(std::move(consumer)); - NPureCalcProto::TUnsplitted message; - message.SetAInt(-13); - message.SetAUint(47); - message.SetAString("first message"); - message.SetABool(true); + NPureCalcProto::TUnsplitted message; + message.SetAInt(-13); + message.SetAUint(47); + message.SetAString("first message"); + message.SetABool(true); - input->OnObject(&message); - UNIT_ASSERT(queue0.size() == 1 && queue1.empty() && queue2.empty() && finalValue == 0); + input->OnObject(&message); + UNIT_ASSERT(queue0.size() == 1 && queue1.empty() && queue2.empty() && finalValue == 0); - message.SetABool(false); - message.SetAString("second message"); + message.SetABool(false); + message.SetAString("second message"); - input->OnObject(&message); - UNIT_ASSERT(queue0.size() == 1 && queue1.size() == 1 && queue2.empty() && finalValue == 0); + input->OnObject(&message); + UNIT_ASSERT(queue0.size() == 1 && queue1.size() == 1 && queue2.empty() && finalValue == 0); - message.ClearABool(); + message.ClearABool(); - input->OnObject(&message); - UNIT_ASSERT(queue0.size() == 1 && queue1.size() == 1 && queue2.size() == 1 && finalValue == 0); + input->OnObject(&message); + UNIT_ASSERT(queue0.size() == 1 && queue1.size() == 1 && queue2.size() == 1 && finalValue == 0); - input->OnFinish(); - UNIT_ASSERT(queue0.size() == 1 && queue1.size() == 1 && queue2.size() == 1 && finalValue == 42); + input->OnFinish(); + UNIT_ASSERT(queue0.size() == 1 && queue1.size() == 1 && queue2.size() == 1 && finalValue == 42); - TVariantConsumerImpl::TType0 expected0 = {{-13, "first message"}}; - UNIT_ASSERT_EQUAL(queue0, expected0); + TVariantConsumerImpl::TType0 expected0 = {{-13, "first message"}}; + UNIT_ASSERT_EQUAL(queue0, expected0); - TVariantConsumerImpl::TType1 expected1 = {{47, "second message"}}; - UNIT_ASSERT_EQUAL(queue1, expected1); + TVariantConsumerImpl::TType1 expected1 = {{47, "second message"}}; + UNIT_ASSERT_EQUAL(queue1, expected1); - TVariantConsumerImpl::TType2 expected2 = {{"Error"}}; - UNIT_ASSERT_EQUAL(queue2, expected2); - } + TVariantConsumerImpl::TType2 expected2 = {{"Error"}}; + UNIT_ASSERT_EQUAL(queue2, expected2); + } - { - auto program1 = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TUnsplitted>(), - TProtobufMultiOutputSpec<NPureCalcProto::TSplitted1, NPureCalcProto::TSplitted2, NPureCalcProto::TStringMessage>( - {}, arenas - ), - sExpr, - ETranslationMode::SExpr - ); - - auto program2 = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TUnsplitted>(), - TProtobufMultiOutputSpec<NPureCalcProto::TSplitted1, NPureCalcProto::TSplitted2, NPureCalcProto::TStringMessage>( - {}, arenas - ), - sExpr, - ETranslationMode::SExpr - ); - - auto input1 = MakeHolder<TUnsplittedStreamImpl>(); - auto output1 = program1->Apply(std::move(input1)); - - auto input2 = MakeHolder<TUnsplittedStreamImpl>(); - auto output2 = program2->Apply(std::move(input2)); - - decltype(output1->Fetch()) variant1; - decltype(output2->Fetch()) variant2; - -#define ASSERT_EQUAL_FIELDS(X1, X2, I, F, E) \ - UNIT_ASSERT_EQUAL(X1.index(), I); \ - UNIT_ASSERT_EQUAL(X2.index(), I); \ - UNIT_ASSERT_EQUAL(std::get<I>(X1)->Get##F(), E); \ + { + auto program1 = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TUnsplitted>(), + TProtobufMultiOutputSpec<NPureCalcProto::TSplitted1, NPureCalcProto::TSplitted2, NPureCalcProto::TStringMessage>( + {}, arenas), + sExpr, + ETranslationMode::SExpr); + + auto program2 = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TUnsplitted>(), + TProtobufMultiOutputSpec<NPureCalcProto::TSplitted1, NPureCalcProto::TSplitted2, NPureCalcProto::TStringMessage>( + {}, arenas), + sExpr, + ETranslationMode::SExpr); + + auto input1 = MakeHolder<TUnsplittedStreamImpl>(); + auto output1 = program1->Apply(std::move(input1)); + + auto input2 = MakeHolder<TUnsplittedStreamImpl>(); + auto output2 = program2->Apply(std::move(input2)); + + decltype(output1->Fetch()) variant1; + decltype(output2->Fetch()) variant2; + +#define ASSERT_EQUAL_FIELDS(X1, X2, I, F, E) \ + UNIT_ASSERT_EQUAL(X1.index(), I); \ + UNIT_ASSERT_EQUAL(X2.index(), I); \ + UNIT_ASSERT_EQUAL(std::get<I>(X1)->Get##F(), E); \ UNIT_ASSERT_EQUAL(std::get<I>(X2)->Get##F(), E) - variant1 = output1->Fetch(); - variant2 = output2->Fetch(); - ASSERT_EQUAL_FIELDS(variant1, variant2, 2, X, "Error"); - ASSERT_EQUAL_FIELDS(variant1, variant2, 2, Arena, (arenas.Defined() ? arenas->at(2) : nullptr)); - - variant1 = output1->Fetch(); - variant2 = output2->Fetch(); - ASSERT_EQUAL_FIELDS(variant1, variant2, 1, CUint, 111); - ASSERT_EQUAL_FIELDS(variant1, variant2, 1, CString, "Hello!"); - ASSERT_EQUAL_FIELDS(variant1, variant2, 1, Arena, (arenas.Defined() ? arenas->at(1) : nullptr)); - - variant1 = output1->Fetch(); - variant2 = output2->Fetch(); - ASSERT_EQUAL_FIELDS(variant1, variant2, 0, BInt, -23); - ASSERT_EQUAL_FIELDS(variant1, variant2, 0, BString, "Hello!"); - ASSERT_EQUAL_FIELDS(variant1, variant2, 0, Arena, (arenas.Defined() ? arenas->at(0) : nullptr)); - - variant1 = output1->Fetch(); - variant2 = output2->Fetch(); - UNIT_ASSERT_EQUAL(variant1.index(), 0); - UNIT_ASSERT_EQUAL(variant2.index(), 0); - UNIT_ASSERT_EQUAL(std::get<0>(variant1), nullptr); - UNIT_ASSERT_EQUAL(std::get<0>(variant1), nullptr); + variant1 = output1->Fetch(); + variant2 = output2->Fetch(); + ASSERT_EQUAL_FIELDS(variant1, variant2, 2, X, "Error"); + ASSERT_EQUAL_FIELDS(variant1, variant2, 2, Arena, (arenas.Defined() ? arenas->at(2) : nullptr)); + + variant1 = output1->Fetch(); + variant2 = output2->Fetch(); + ASSERT_EQUAL_FIELDS(variant1, variant2, 1, CUint, 111); + ASSERT_EQUAL_FIELDS(variant1, variant2, 1, CString, "Hello!"); + ASSERT_EQUAL_FIELDS(variant1, variant2, 1, Arena, (arenas.Defined() ? arenas->at(1) : nullptr)); + + variant1 = output1->Fetch(); + variant2 = output2->Fetch(); + ASSERT_EQUAL_FIELDS(variant1, variant2, 0, BInt, -23); + ASSERT_EQUAL_FIELDS(variant1, variant2, 0, BString, "Hello!"); + ASSERT_EQUAL_FIELDS(variant1, variant2, 0, Arena, (arenas.Defined() ? arenas->at(0) : nullptr)); + + variant1 = output1->Fetch(); + variant2 = output2->Fetch(); + UNIT_ASSERT_EQUAL(variant1.index(), 0); + UNIT_ASSERT_EQUAL(variant2.index(), 0); + UNIT_ASSERT_EQUAL(std::get<0>(variant1), nullptr); + UNIT_ASSERT_EQUAL(std::get<0>(variant1), nullptr); #undef ASSERT_EQUAL_FIELDS - } - } - - Y_UNIT_TEST(TestMultiOutputs) { - CheckMultiOutputs(Nothing()); } +} - Y_UNIT_TEST(TestSupportedTypes) { +Y_UNIT_TEST(TestMultiOutputs) { + CheckMultiOutputs(Nothing()); +} - } +Y_UNIT_TEST(TestSupportedTypes) { +} - Y_UNIT_TEST(TestProtobufArena) { +Y_UNIT_TEST(TestProtobufArena) { + { + NPureCalcProto::TNestedRepeated input; + input.SetX(20); { - NPureCalcProto::TNestedRepeated input; - input.SetX(20); + auto* item = input.MutableY()->Add(); + item->SetX(100); { - auto* item = input.MutableY()->Add(); - item->SetX(100); - { - auto* y = item->MutableY(); - *y = GetCanonicalMessage(); - y->SetFUint64(1000); - } + auto* y = item->MutableY(); + *y = GetCanonicalMessage(); + y->SetFUint64(1000); } + } + { + auto* item = input.MutableY()->Add(); + item->SetX(200); { - auto* item = input.MutableY()->Add(); - item->SetX(200); - { - auto* y = item->MutableY(); - *y = GetCanonicalMessage(); - y->SetFUint64(2000); - } + auto* y = item->MutableY(); + *y = GetCanonicalMessage(); + y->SetFUint64(2000); } - - google::protobuf::Arena arena; - CheckPassThroughYql(input, &arena); } - { - google::protobuf::Arena arena1; - google::protobuf::Arena arena2; - TVector<google::protobuf::Arena*> arenas{&arena1, &arena2, &arena1}; - CheckMultiOutputs(arenas); - } + google::protobuf::Arena arena; + CheckPassThroughYql(input, &arena); } - Y_UNIT_TEST(TestFieldRenames) { - using namespace NYql::NPureCalc; + { + google::protobuf::Arena arena1; + google::protobuf::Arena arena2; + TVector<google::protobuf::Arena*> arenas{&arena1, &arena2, &arena1}; + CheckMultiOutputs(arenas); + } +} - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestFieldRenames) { + using namespace NYql::NPureCalc; - TString query = "SELECT InputAlias AS OutputAlias FROM Input"; + auto factory = MakeProgramFactory(); - auto inputProtoOptions = TProtoSchemaOptions(); - inputProtoOptions.SetFieldRenames({{"X", "InputAlias"}}); + TString query = "SELECT InputAlias AS OutputAlias FROM Input"; - auto inputSpec = TProtobufInputSpec<NPureCalcProto::TSimpleMessage>( - Nothing(), std::move(inputProtoOptions) - ); + auto inputProtoOptions = TProtoSchemaOptions(); + inputProtoOptions.SetFieldRenames({{"X", "InputAlias"}}); - auto outputProtoOptions = TProtoSchemaOptions(); - outputProtoOptions.SetFieldRenames({{"X", "OutputAlias"}}); + auto inputSpec = TProtobufInputSpec<NPureCalcProto::TSimpleMessage>( + Nothing(), std::move(inputProtoOptions)); - auto outputSpec = TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>( - std::move(outputProtoOptions) - ); + auto outputProtoOptions = TProtoSchemaOptions(); + outputProtoOptions.SetFieldRenames({{"X", "OutputAlias"}}); - { - auto program = factory->MakePullStreamProgram( - inputSpec, outputSpec, query, ETranslationMode::SQL - ); + auto outputSpec = TProtobufOutputSpec<NPureCalcProto::TSimpleMessage>( + std::move(outputProtoOptions)); - auto input = MakeHolder<TSimpleMessageStreamImpl>(1); - auto output = program->Apply(std::move(input)); + { + auto program = factory->MakePullStreamProgram( + inputSpec, outputSpec, query, ETranslationMode::SQL); - TVector<int> expected = {1}; - TVector<int> actual; + auto input = MakeHolder<TSimpleMessageStreamImpl>(1); + auto output = program->Apply(std::move(input)); - while (auto* x = output->Fetch()) { - actual.push_back(x->GetX()); - } + TVector<int> expected = {1}; + TVector<int> actual; - UNIT_ASSERT_VALUES_EQUAL(expected, actual); + while (auto* x = output->Fetch()) { + actual.push_back(x->GetX()); } - { - auto program = factory->MakePullListProgram( - inputSpec, outputSpec, query, ETranslationMode::SQL - ); + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + } - auto input = MakeHolder<TSimpleMessageStreamImpl>(1); - auto output = program->Apply(std::move(input)); + { + auto program = factory->MakePullListProgram( + inputSpec, outputSpec, query, ETranslationMode::SQL); - TVector<int> expected = {1}; - TVector<int> actual; + auto input = MakeHolder<TSimpleMessageStreamImpl>(1); + auto output = program->Apply(std::move(input)); - while (auto* x = output->Fetch()) { - actual.push_back(x->GetX()); - } + TVector<int> expected = {1}; + TVector<int> actual; - UNIT_ASSERT_VALUES_EQUAL(expected, actual); + while (auto* x = output->Fetch()) { + actual.push_back(x->GetX()); } - { - auto program = factory->MakePushStreamProgram( - inputSpec, outputSpec, query, ETranslationMode::SQL - ); + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + } - TVector<int> expected = {1, -100}; - TVector<int> actual; + { + auto program = factory->MakePushStreamProgram( + inputSpec, outputSpec, query, ETranslationMode::SQL); - auto consumer = MakeHolder<TSimpleMessageConsumerImpl>(&actual); - auto input = program->Apply(std::move(consumer)); + TVector<int> expected = {1, -100}; + TVector<int> actual; - NPureCalcProto::TSimpleMessage Message; + auto consumer = MakeHolder<TSimpleMessageConsumerImpl>(&actual); + auto input = program->Apply(std::move(consumer)); - Message.SetX(1); - input->OnObject(&Message); + NPureCalcProto::TSimpleMessage Message; - input->OnFinish(); + Message.SetX(1); + input->OnObject(&Message); - UNIT_ASSERT_VALUES_EQUAL(expected, actual); - } - } + input->OnFinish(); - Y_UNIT_TEST(TestNestedFieldRenames) { - using namespace NYql::NPureCalc; + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + } +} - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestNestedFieldRenames) { + using namespace NYql::NPureCalc; - TString query = "SELECT InputAlias AS OutputAlias, X FROM Input"; + auto factory = MakeProgramFactory(); - auto inputProtoOptions = TProtoSchemaOptions(); - inputProtoOptions.SetFieldRenames({{"Y", "InputAlias"}, {"FInt64", "NestedField"}}); - inputProtoOptions.SetEnableRecursiveRenaming(true); + TString query = "SELECT InputAlias AS OutputAlias, X FROM Input"; - auto inputSpec = TProtobufInputSpec<NPureCalcProto::TSimpleNested>( - Nothing(), std::move(inputProtoOptions) - ); + auto inputProtoOptions = TProtoSchemaOptions(); + inputProtoOptions.SetFieldRenames({{"Y", "InputAlias"}, {"FInt64", "NestedField"}}); + inputProtoOptions.SetEnableRecursiveRenaming(true); - auto outputProtoOptions = TProtoSchemaOptions(); - outputProtoOptions.SetEnableRecursiveRenaming(true); + auto inputSpec = TProtobufInputSpec<NPureCalcProto::TSimpleNested>( + Nothing(), std::move(inputProtoOptions)); - outputProtoOptions.SetFieldRenames({{"Y", "OutputAlias"}, {"FInt64", "NestedField"}}); - auto outputSpecWithNestedRename = TProtobufOutputSpec<NPureCalcProto::TSimpleNested>( - outputProtoOptions - ); + auto outputProtoOptions = TProtoSchemaOptions(); + outputProtoOptions.SetEnableRecursiveRenaming(true); - outputProtoOptions.SetFieldRenames({{"Y", "OutputAlias"}}); - auto outputSpecWithoutNestedRename = TProtobufOutputSpec<NPureCalcProto::TSimpleNested>( - std::move(outputProtoOptions) - ); + outputProtoOptions.SetFieldRenames({{"Y", "OutputAlias"}, {"FInt64", "NestedField"}}); + auto outputSpecWithNestedRename = TProtobufOutputSpec<NPureCalcProto::TSimpleNested>( + outputProtoOptions); - { - auto program = factory->MakePullStreamProgram( - inputSpec, outputSpecWithNestedRename, query, ETranslationMode::SQL - ); + outputProtoOptions.SetFieldRenames({{"Y", "OutputAlias"}}); + auto outputSpecWithoutNestedRename = TProtobufOutputSpec<NPureCalcProto::TSimpleNested>( + std::move(outputProtoOptions)); - auto input = MakeHolder<TTSimpleNestedStreamImpl>(); - auto output = program->Apply(std::move(input)); + { + auto program = factory->MakePullStreamProgram( + inputSpec, outputSpecWithNestedRename, query, ETranslationMode::SQL); - TVector<int> expected = {3}; - TVector<int> actual; + auto input = MakeHolder<TTSimpleNestedStreamImpl>(); + auto output = program->Apply(std::move(input)); - while (auto* x = output->Fetch()) { - actual.push_back(x->GetY().GetFInt64()); - } + TVector<int> expected = {3}; + TVector<int> actual; - UNIT_ASSERT_VALUES_EQUAL(expected, actual); + while (auto* x = output->Fetch()) { + actual.push_back(x->GetY().GetFInt64()); } - { - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram( - inputSpec, outputSpecWithoutNestedRename, query, ETranslationMode::SQL - ); - }(), TCompileError, "Failed to optimize"); - } + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + } + + { + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram( + inputSpec, outputSpecWithoutNestedRename, query, ETranslationMode::SQL); + }(), TCompileError, "Failed to optimize"); } } +} // Y_UNIT_TEST_SUITE(TestProtoIO) diff --git a/yql/essentials/public/purecalc/io_specs/protobuf/ut/ya.make b/yql/essentials/public/purecalc/io_specs/protobuf/ut/ya.make index 2519816d02e..69e36132340 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf/ut/ya.make +++ b/yql/essentials/public/purecalc/io_specs/protobuf/ut/ya.make @@ -2,6 +2,8 @@ IF (NOT SANITIZER_TYPE) UNITTEST() +ENABLE(YQL_STYLE_CPP) + PEERDIR( library/cpp/protobuf/util yql/essentials/public/udf/service/exception_policy diff --git a/yql/essentials/public/purecalc/io_specs/protobuf/ya.make b/yql/essentials/public/purecalc/io_specs/protobuf/ya.make index b9441ceecf4..600bf9df61d 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf/ya.make +++ b/yql/essentials/public/purecalc/io_specs/protobuf/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + PEERDIR( yql/essentials/public/purecalc/common yql/essentials/public/purecalc/io_specs/protobuf_raw diff --git a/yql/essentials/public/purecalc/io_specs/protobuf_raw/proto_holder.h b/yql/essentials/public/purecalc/io_specs/protobuf_raw/proto_holder.h index 7d4d843bfcf..6808a150945 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf_raw/proto_holder.h +++ b/yql/essentials/public/purecalc/io_specs/protobuf_raw/proto_holder.h @@ -7,25 +7,25 @@ #include <type_traits> namespace NYql::NPureCalc { - class TProtoDestroyer { - public: - template <typename T> - static inline void Destroy(T* t) noexcept { - if (t->GetArena() == nullptr) { - CheckedDelete(t); - } +class TProtoDestroyer { +public: + template <typename T> + static inline void Destroy(T* t) noexcept { + if (t->GetArena() == nullptr) { + CheckedDelete(t); } - }; + } +}; - template <typename TProto> - concept IsProtoMessage = std::is_base_of_v<NProtoBuf::Message, TProto>; +template <typename TProto> +concept IsProtoMessage = std::is_base_of_v<NProtoBuf::Message, TProto>; - template <IsProtoMessage TProto> - using TProtoHolder = THolder<TProto, TProtoDestroyer>; +template <IsProtoMessage TProto> +using TProtoHolder = THolder<TProto, TProtoDestroyer>; - template <IsProtoMessage TProto, typename... TArgs> - TProtoHolder<TProto> MakeProtoHolder(NProtoBuf::Arena* arena, TArgs&&... args) { - auto* ptr = NProtoBuf::Arena::CreateMessage<TProto>(arena, std::forward<TArgs>(args)...); - return TProtoHolder<TProto>(ptr); - } +template <IsProtoMessage TProto, typename... TArgs> +TProtoHolder<TProto> MakeProtoHolder(NProtoBuf::Arena* arena, TArgs&&... args) { + auto* ptr = NProtoBuf::Arena::CreateMessage<TProto>(arena, std::forward<TArgs>(args)...); + return TProtoHolder<TProto>(ptr); } +} // namespace NYql::NPureCalc diff --git a/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.cpp b/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.cpp index 150ebfa80a8..af4561a49b8 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.cpp +++ b/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.cpp @@ -17,8 +17,7 @@ using namespace NKikimr::NMiniKQL; TProtobufRawInputSpec::TProtobufRawInputSpec( const Descriptor& descriptor, const TMaybe<TString>& timestampColumn, - const TProtoSchemaOptions& options -) + const TProtoSchemaOptions& options) : Descriptor_(descriptor) , TimestampColumn_(timestampColumn) , SchemaOptions_(options) @@ -58,8 +57,7 @@ TProtobufRawOutputSpec::TProtobufRawOutputSpec( const Descriptor& descriptor, MessageFactory* factory, const TProtoSchemaOptions& options, - Arena* arena -) + Arena* arena) : Descriptor_(descriptor) , Factory_(factory) , SchemaOptions_(options) @@ -104,8 +102,7 @@ TProtobufRawMultiOutputSpec::TProtobufRawMultiOutputSpec( TVector<const Descriptor*> descriptors, TMaybe<TVector<MessageFactory*>> factories, const TProtoSchemaOptions& options, - TMaybe<TVector<Arena*>> arenas -) + TMaybe<TVector<Arena*>> arenas) : Descriptors_(std::move(descriptors)) , SchemaOptions_(options) { @@ -171,408 +168,325 @@ const TProtoSchemaOptions& TProtobufRawMultiOutputSpec::GetSchemaOptions() const } namespace { - struct TFieldMapping { - TString Name; - const FieldDescriptor* Field; - TVector<TFieldMapping> NestedFields; - }; - - void FillFieldMappingsImpl( - const TStructType* fromType, - const Descriptor& toType, - TVector<TFieldMapping>& mappings, - const TMaybe<TString>& timestampColumn, - bool listIsOptional, - bool enableRecursiveRenaming, - const THashMap<TString, TString>& inverseFieldRenames - ) { - static const THashMap<TString, TString> emptyInverseFieldRenames; - mappings.resize(fromType->GetMembersCount()); - for (ui32 i = 0; i < fromType->GetMembersCount(); ++i) { - TString fieldName(fromType->GetMemberName(i)); - if (auto fieldRenamePtr = inverseFieldRenames.FindPtr(fieldName)) { - fieldName = *fieldRenamePtr; - } +struct TFieldMapping { + TString Name; + const FieldDescriptor* Field; + TVector<TFieldMapping> NestedFields; +}; + +void FillFieldMappingsImpl( + const TStructType* fromType, + const Descriptor& toType, + TVector<TFieldMapping>& mappings, + const TMaybe<TString>& timestampColumn, + bool listIsOptional, + bool enableRecursiveRenaming, + const THashMap<TString, TString>& inverseFieldRenames) { + static const THashMap<TString, TString> emptyInverseFieldRenames; + mappings.resize(fromType->GetMembersCount()); + for (ui32 i = 0; i < fromType->GetMembersCount(); ++i) { + TString fieldName(fromType->GetMemberName(i)); + if (auto fieldRenamePtr = inverseFieldRenames.FindPtr(fieldName)) { + fieldName = *fieldRenamePtr; + } - mappings[i].Name = fieldName; - mappings[i].Field = toType.FindFieldByName(fieldName); - YQL_ENSURE( - mappings[i].Field || timestampColumn && *timestampColumn == fieldName, - "Missing field: " << fieldName); - - const auto* fieldType = fromType->GetMemberType(i); - if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::List) { - const auto* listType = static_cast<const NKikimr::NMiniKQL::TListType*>(fieldType); - fieldType = listType->GetItemType(); - } else if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Optional) { - const auto* optionalType = static_cast<const NKikimr::NMiniKQL::TOptionalType*>(fieldType); - fieldType = optionalType->GetItemType(); - - if (listIsOptional) { - if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::List) { - const auto* listType = static_cast<const NKikimr::NMiniKQL::TListType*>(fieldType); - fieldType = listType->GetItemType(); - } + mappings[i].Name = fieldName; + mappings[i].Field = toType.FindFieldByName(fieldName); + YQL_ENSURE( + mappings[i].Field || timestampColumn && *timestampColumn == fieldName, + "Missing field: " << fieldName); + + const auto* fieldType = fromType->GetMemberType(i); + if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::List) { + const auto* listType = static_cast<const NKikimr::NMiniKQL::TListType*>(fieldType); + fieldType = listType->GetItemType(); + } else if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Optional) { + const auto* optionalType = static_cast<const NKikimr::NMiniKQL::TOptionalType*>(fieldType); + fieldType = optionalType->GetItemType(); + + if (listIsOptional) { + if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::List) { + const auto* listType = static_cast<const NKikimr::NMiniKQL::TListType*>(fieldType); + fieldType = listType->GetItemType(); } } - YQL_ENSURE(fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Struct || + } + YQL_ENSURE(fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Struct || fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Data, - "unsupported field kind [" << fieldType->GetKindAsStr() << "], field [" << fieldName << "]"); - if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Struct) { - FillFieldMappingsImpl( - static_cast<const NKikimr::NMiniKQL::TStructType*>(fieldType), - *mappings[i].Field->message_type(), - mappings[i].NestedFields, - Nothing(), - listIsOptional, - enableRecursiveRenaming, - enableRecursiveRenaming ? inverseFieldRenames : emptyInverseFieldRenames - ); - } + "unsupported field kind [" << fieldType->GetKindAsStr() << "], field [" << fieldName << "]"); + if (fieldType->GetKind() == NKikimr::NMiniKQL::TType::EKind::Struct) { + FillFieldMappingsImpl( + static_cast<const NKikimr::NMiniKQL::TStructType*>(fieldType), + *mappings[i].Field->message_type(), + mappings[i].NestedFields, + Nothing(), + listIsOptional, + enableRecursiveRenaming, + enableRecursiveRenaming ? inverseFieldRenames : emptyInverseFieldRenames); } } +} - /** - * Fills a tree of field mappings from the given yql struct type to protobuf message. - * - * @param fromType source yql type. - * @param toType target protobuf message type. - * @param mappings destination vector will be filled with field descriptors. Order of descriptors will match - * the order of field names. - */ - void FillFieldMappings( - const TStructType* fromType, - const Descriptor& toType, - TVector<TFieldMapping>& mappings, - const TMaybe<TString>& timestampColumn, - bool listIsOptional, - bool enableRecursiveRenaming, - const THashMap<TString, TString>& fieldRenames - ) { - THashMap<TString, TString> inverseFieldRenames; - for (const auto& [source, target]: fieldRenames) { - auto [iterator, emplaced] = inverseFieldRenames.emplace(target, source); - Y_ENSURE(emplaced, "Duplicate rename field found: " << source << " -> " << target); - } - - FillFieldMappingsImpl(fromType, toType, mappings, timestampColumn, listIsOptional, enableRecursiveRenaming, inverseFieldRenames); +/** + * Fills a tree of field mappings from the given yql struct type to protobuf message. + * + * @param fromType source yql type. + * @param toType target protobuf message type. + * @param mappings destination vector will be filled with field descriptors. Order of descriptors will match + * the order of field names. + */ +void FillFieldMappings( + const TStructType* fromType, + const Descriptor& toType, + TVector<TFieldMapping>& mappings, + const TMaybe<TString>& timestampColumn, + bool listIsOptional, + bool enableRecursiveRenaming, + const THashMap<TString, TString>& fieldRenames) { + THashMap<TString, TString> inverseFieldRenames; + for (const auto& [source, target] : fieldRenames) { + auto [iterator, emplaced] = inverseFieldRenames.emplace(target, source); + Y_ENSURE(emplaced, "Duplicate rename field found: " << source << " -> " << target); } - /** - * Extract field values from the given protobuf message into an array of unboxed values. - * - * @param factory to create nested unboxed values. - * @param source source protobuf message. - * @param destination destination array of unboxed values. Each element in the array corresponds to a field - * in the protobuf message. - * @param mappings vector of protobuf field descriptors which denotes relation between fields of the - * source message and elements of the destination array. - * @param scratch temporary string which will be used during conversion. - */ - void FillInputValue( - const THolderFactory& factory, - const Message* source, - TUnboxedValue* destination, - const TVector<TFieldMapping>& mappings, - const TMaybe<TString>& timestampColumn, - ITimeProvider* timeProvider, - EEnumPolicy enumPolicy - ) { - TString scratch; - auto reflection = source->GetReflection(); - for (ui32 i = 0; i < mappings.size(); ++i) { - auto mapping = mappings[i]; - if (!mapping.Field) { - YQL_ENSURE(timestampColumn && mapping.Name == *timestampColumn); - destination[i] = TUnboxedValuePod(timeProvider->Now().MicroSeconds()); - continue; - } - - const auto type = mapping.Field->type(); - if (mapping.Field->label() == FieldDescriptor::LABEL_REPEATED) { - const auto size = static_cast<ui32>(reflection->FieldSize(*source, mapping.Field)); - if (size == 0) { - destination[i] = factory.GetEmptyContainerLazy(); - } else { - TUnboxedValue* inplace = nullptr; - destination[i] = factory.CreateDirectArrayHolder(size, inplace); - for (ui32 j = 0; j < size; ++j) { - switch (type) { - case FieldDescriptor::TYPE_DOUBLE: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedDouble(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_FLOAT: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedFloat(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_INT64: - case FieldDescriptor::TYPE_SFIXED64: - case FieldDescriptor::TYPE_SINT64: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedInt64(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_ENUM: - switch (EnumFormatType(*mapping.Field, enumPolicy)) { - case EEnumFormatType::Int32: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedEnumValue(*source, mapping.Field, j)); - break; - case EEnumFormatType::String: - inplace[j] = MakeString(reflection->GetRepeatedEnum(*source, mapping.Field, j)->name()); - break; - } - break; - - case FieldDescriptor::TYPE_UINT64: - case FieldDescriptor::TYPE_FIXED64: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedUInt64(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_INT32: - case FieldDescriptor::TYPE_SFIXED32: - case FieldDescriptor::TYPE_SINT32: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedInt32(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_UINT32: - case FieldDescriptor::TYPE_FIXED32: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedUInt32(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_BOOL: - inplace[j] = TUnboxedValuePod(reflection->GetRepeatedBool(*source, mapping.Field, j)); - break; - - case FieldDescriptor::TYPE_STRING: - inplace[j] = MakeString(reflection->GetRepeatedStringReference(*source, mapping.Field, j, &scratch)); - break; - - case FieldDescriptor::TYPE_BYTES: - inplace[j] = MakeString(reflection->GetRepeatedStringReference(*source, mapping.Field, j, &scratch)); - break; - - case FieldDescriptor::TYPE_MESSAGE: - { - const Message& nestedMessage = reflection->GetRepeatedMessage(*source, mapping.Field, j); - TUnboxedValue* nestedValues = nullptr; - inplace[j] = factory.CreateDirectArrayHolder(static_cast<ui32>(mapping.NestedFields.size()), - nestedValues); - FillInputValue(factory, &nestedMessage, nestedValues, mapping.NestedFields, Nothing(), timeProvider, enumPolicy); - } - break; - - default: - ythrow yexception() << "Unsupported protobuf type: " << mapping.Field->type_name() << ", field: " << mapping.Field->name(); - } - } - } - } else { - if (!reflection->HasField(*source, mapping.Field)) { - continue; - } - - switch (type) { - case FieldDescriptor::TYPE_DOUBLE: - destination[i] = TUnboxedValuePod(reflection->GetDouble(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_FLOAT: - destination[i] = TUnboxedValuePod(reflection->GetFloat(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_INT64: - case FieldDescriptor::TYPE_SFIXED64: - case FieldDescriptor::TYPE_SINT64: - destination[i] = TUnboxedValuePod(reflection->GetInt64(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_ENUM: - switch (EnumFormatType(*mapping.Field, enumPolicy)) { - case EEnumFormatType::Int32: - destination[i] = TUnboxedValuePod(reflection->GetEnumValue(*source, mapping.Field)); - break; - case EEnumFormatType::String: - destination[i] = MakeString(reflection->GetEnum(*source, mapping.Field)->name()); - break; - } - break; - - case FieldDescriptor::TYPE_UINT64: - case FieldDescriptor::TYPE_FIXED64: - destination[i] = TUnboxedValuePod(reflection->GetUInt64(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_INT32: - case FieldDescriptor::TYPE_SFIXED32: - case FieldDescriptor::TYPE_SINT32: - destination[i] = TUnboxedValuePod(reflection->GetInt32(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_UINT32: - case FieldDescriptor::TYPE_FIXED32: - destination[i] = TUnboxedValuePod(reflection->GetUInt32(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_BOOL: - destination[i] = TUnboxedValuePod(reflection->GetBool(*source, mapping.Field)); - break; - - case FieldDescriptor::TYPE_STRING: - destination[i] = MakeString(reflection->GetStringReference(*source, mapping.Field, &scratch)); - break; - - case FieldDescriptor::TYPE_BYTES: - destination[i] = MakeString(reflection->GetStringReference(*source, mapping.Field, &scratch)); - break; - case FieldDescriptor::TYPE_MESSAGE: - { - const Message& nestedMessage = reflection->GetMessage(*source, mapping.Field); - TUnboxedValue* nestedValues = nullptr; - destination[i] = factory.CreateDirectArrayHolder(static_cast<ui32>(mapping.NestedFields.size()), - nestedValues); - FillInputValue(factory, &nestedMessage, nestedValues, mapping.NestedFields, Nothing(), timeProvider, enumPolicy); - } - break; + FillFieldMappingsImpl(fromType, toType, mappings, timestampColumn, listIsOptional, enableRecursiveRenaming, inverseFieldRenames); +} - default: - ythrow yexception() << "Unsupported protobuf type: " << mapping.Field->type_name() - << ", field: " << mapping.Field->name(); - } - } +/** + * Extract field values from the given protobuf message into an array of unboxed values. + * + * @param factory to create nested unboxed values. + * @param source source protobuf message. + * @param destination destination array of unboxed values. Each element in the array corresponds to a field + * in the protobuf message. + * @param mappings vector of protobuf field descriptors which denotes relation between fields of the + * source message and elements of the destination array. + * @param scratch temporary string which will be used during conversion. + */ +void FillInputValue( + const THolderFactory& factory, + const Message* source, + TUnboxedValue* destination, + const TVector<TFieldMapping>& mappings, + const TMaybe<TString>& timestampColumn, + ITimeProvider* timeProvider, + EEnumPolicy enumPolicy) { + TString scratch; + auto reflection = source->GetReflection(); + for (ui32 i = 0; i < mappings.size(); ++i) { + auto mapping = mappings[i]; + if (!mapping.Field) { + YQL_ENSURE(timestampColumn && mapping.Name == *timestampColumn); + destination[i] = TUnboxedValuePod(timeProvider->Now().MicroSeconds()); + continue; } - } - - /** - * Convert unboxed value to protobuf. - * - * @param source unboxed value to extract data from. Type of the value should be struct. It's UB to pass - * a non-struct value here. - * @param destination destination message. Data in this message will be overwritten - * by data from unboxed value. - * @param mappings vector of protobuf field descriptors which denotes relation between struct fields - * and message fields. For any i-th element of this vector, type of the i-th element of - * the unboxed structure must match type of the field pointed by descriptor. Size of this - * vector should match the number of fields in the struct. - */ - void FillOutputMessage( - const TUnboxedValue& source, - Message* destination, - const TVector<TFieldMapping>& mappings, - EEnumPolicy enumPolicy - ) { - auto reflection = destination->GetReflection(); - for (ui32 i = 0; i < mappings.size(); ++i) { - const auto& mapping = mappings[i]; - const auto& cell = source.GetElement(i); - if (!cell) { - reflection->ClearField(destination, mapping.Field); - continue; - } - const auto type = mapping.Field->type(); - if (mapping.Field->label() == FieldDescriptor::LABEL_REPEATED) { - const auto iter = cell.GetListIterator(); - reflection->ClearField(destination, mapping.Field); - for (TUnboxedValue item; iter.Next(item);) { - switch (mapping.Field->type()) { + const auto type = mapping.Field->type(); + if (mapping.Field->label() == FieldDescriptor::LABEL_REPEATED) { + const auto size = static_cast<ui32>(reflection->FieldSize(*source, mapping.Field)); + if (size == 0) { + destination[i] = factory.GetEmptyContainerLazy(); + } else { + TUnboxedValue* inplace = nullptr; + destination[i] = factory.CreateDirectArrayHolder(size, inplace); + for (ui32 j = 0; j < size; ++j) { + switch (type) { case FieldDescriptor::TYPE_DOUBLE: - reflection->AddDouble(destination, mapping.Field, item.Get<double>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedDouble(*source, mapping.Field, j)); break; case FieldDescriptor::TYPE_FLOAT: - reflection->AddFloat(destination, mapping.Field, item.Get<float>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedFloat(*source, mapping.Field, j)); break; case FieldDescriptor::TYPE_INT64: case FieldDescriptor::TYPE_SFIXED64: case FieldDescriptor::TYPE_SINT64: - reflection->AddInt64(destination, mapping.Field, item.Get<i64>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedInt64(*source, mapping.Field, j)); break; - case FieldDescriptor::TYPE_ENUM: { + case FieldDescriptor::TYPE_ENUM: switch (EnumFormatType(*mapping.Field, enumPolicy)) { case EEnumFormatType::Int32: - reflection->AddEnumValue(destination, mapping.Field, item.Get<i32>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedEnumValue(*source, mapping.Field, j)); break; - case EEnumFormatType::String: { - auto enumValueDescriptor = mapping.Field->enum_type()->FindValueByName(TString(item.AsStringRef())); - if (!enumValueDescriptor) { - enumValueDescriptor = mapping.Field->default_value_enum(); - } - reflection->AddEnum(destination, mapping.Field, enumValueDescriptor); + case EEnumFormatType::String: + inplace[j] = MakeString(reflection->GetRepeatedEnum(*source, mapping.Field, j)->name()); break; - } } break; - } case FieldDescriptor::TYPE_UINT64: case FieldDescriptor::TYPE_FIXED64: - reflection->AddUInt64(destination, mapping.Field, item.Get<ui64>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedUInt64(*source, mapping.Field, j)); break; case FieldDescriptor::TYPE_INT32: case FieldDescriptor::TYPE_SFIXED32: case FieldDescriptor::TYPE_SINT32: - reflection->AddInt32(destination, mapping.Field, item.Get<i32>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedInt32(*source, mapping.Field, j)); break; case FieldDescriptor::TYPE_UINT32: case FieldDescriptor::TYPE_FIXED32: - reflection->AddUInt32(destination, mapping.Field, item.Get<ui32>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedUInt32(*source, mapping.Field, j)); break; case FieldDescriptor::TYPE_BOOL: - reflection->AddBool(destination, mapping.Field, item.Get<bool>()); + inplace[j] = TUnboxedValuePod(reflection->GetRepeatedBool(*source, mapping.Field, j)); break; case FieldDescriptor::TYPE_STRING: - reflection->AddString(destination, mapping.Field, TString(item.AsStringRef())); + inplace[j] = MakeString(reflection->GetRepeatedStringReference(*source, mapping.Field, j, &scratch)); break; case FieldDescriptor::TYPE_BYTES: - reflection->AddString(destination, mapping.Field, TString(item.AsStringRef())); + inplace[j] = MakeString(reflection->GetRepeatedStringReference(*source, mapping.Field, j, &scratch)); break; - case FieldDescriptor::TYPE_MESSAGE: - { - auto* nestedMessage = reflection->AddMessage(destination, mapping.Field); - FillOutputMessage(item, nestedMessage, mapping.NestedFields, enumPolicy); - } - break; + case FieldDescriptor::TYPE_MESSAGE: { + const Message& nestedMessage = reflection->GetRepeatedMessage(*source, mapping.Field, j); + TUnboxedValue* nestedValues = nullptr; + inplace[j] = factory.CreateDirectArrayHolder(static_cast<ui32>(mapping.NestedFields.size()), + nestedValues); + FillInputValue(factory, &nestedMessage, nestedValues, mapping.NestedFields, Nothing(), timeProvider, enumPolicy); + } break; default: - ythrow yexception() << "Unsupported protobuf type: " - << mapping.Field->type_name() << ", field: " << mapping.Field->name(); + ythrow yexception() << "Unsupported protobuf type: " << mapping.Field->type_name() << ", field: " << mapping.Field->name(); } } - } else { - switch (type) { + } + } else { + if (!reflection->HasField(*source, mapping.Field)) { + continue; + } + + switch (type) { + case FieldDescriptor::TYPE_DOUBLE: + destination[i] = TUnboxedValuePod(reflection->GetDouble(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_FLOAT: + destination[i] = TUnboxedValuePod(reflection->GetFloat(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_INT64: + case FieldDescriptor::TYPE_SFIXED64: + case FieldDescriptor::TYPE_SINT64: + destination[i] = TUnboxedValuePod(reflection->GetInt64(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_ENUM: + switch (EnumFormatType(*mapping.Field, enumPolicy)) { + case EEnumFormatType::Int32: + destination[i] = TUnboxedValuePod(reflection->GetEnumValue(*source, mapping.Field)); + break; + case EEnumFormatType::String: + destination[i] = MakeString(reflection->GetEnum(*source, mapping.Field)->name()); + break; + } + break; + + case FieldDescriptor::TYPE_UINT64: + case FieldDescriptor::TYPE_FIXED64: + destination[i] = TUnboxedValuePod(reflection->GetUInt64(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_INT32: + case FieldDescriptor::TYPE_SFIXED32: + case FieldDescriptor::TYPE_SINT32: + destination[i] = TUnboxedValuePod(reflection->GetInt32(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_UINT32: + case FieldDescriptor::TYPE_FIXED32: + destination[i] = TUnboxedValuePod(reflection->GetUInt32(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_BOOL: + destination[i] = TUnboxedValuePod(reflection->GetBool(*source, mapping.Field)); + break; + + case FieldDescriptor::TYPE_STRING: + destination[i] = MakeString(reflection->GetStringReference(*source, mapping.Field, &scratch)); + break; + + case FieldDescriptor::TYPE_BYTES: + destination[i] = MakeString(reflection->GetStringReference(*source, mapping.Field, &scratch)); + break; + case FieldDescriptor::TYPE_MESSAGE: { + const Message& nestedMessage = reflection->GetMessage(*source, mapping.Field); + TUnboxedValue* nestedValues = nullptr; + destination[i] = factory.CreateDirectArrayHolder(static_cast<ui32>(mapping.NestedFields.size()), + nestedValues); + FillInputValue(factory, &nestedMessage, nestedValues, mapping.NestedFields, Nothing(), timeProvider, enumPolicy); + } break; + + default: + ythrow yexception() << "Unsupported protobuf type: " << mapping.Field->type_name() + << ", field: " << mapping.Field->name(); + } + } + } +} + +/** + * Convert unboxed value to protobuf. + * + * @param source unboxed value to extract data from. Type of the value should be struct. It's UB to pass + * a non-struct value here. + * @param destination destination message. Data in this message will be overwritten + * by data from unboxed value. + * @param mappings vector of protobuf field descriptors which denotes relation between struct fields + * and message fields. For any i-th element of this vector, type of the i-th element of + * the unboxed structure must match type of the field pointed by descriptor. Size of this + * vector should match the number of fields in the struct. + */ +void FillOutputMessage( + const TUnboxedValue& source, + Message* destination, + const TVector<TFieldMapping>& mappings, + EEnumPolicy enumPolicy) { + auto reflection = destination->GetReflection(); + for (ui32 i = 0; i < mappings.size(); ++i) { + const auto& mapping = mappings[i]; + const auto& cell = source.GetElement(i); + if (!cell) { + reflection->ClearField(destination, mapping.Field); + continue; + } + const auto type = mapping.Field->type(); + if (mapping.Field->label() == FieldDescriptor::LABEL_REPEATED) { + const auto iter = cell.GetListIterator(); + reflection->ClearField(destination, mapping.Field); + for (TUnboxedValue item; iter.Next(item);) { + switch (mapping.Field->type()) { case FieldDescriptor::TYPE_DOUBLE: - reflection->SetDouble(destination, mapping.Field, cell.Get<double>()); + reflection->AddDouble(destination, mapping.Field, item.Get<double>()); break; case FieldDescriptor::TYPE_FLOAT: - reflection->SetFloat(destination, mapping.Field, cell.Get<float>()); + reflection->AddFloat(destination, mapping.Field, item.Get<float>()); break; case FieldDescriptor::TYPE_INT64: case FieldDescriptor::TYPE_SFIXED64: case FieldDescriptor::TYPE_SINT64: - reflection->SetInt64(destination, mapping.Field, cell.Get<i64>()); + reflection->AddInt64(destination, mapping.Field, item.Get<i64>()); break; case FieldDescriptor::TYPE_ENUM: { switch (EnumFormatType(*mapping.Field, enumPolicy)) { case EEnumFormatType::Int32: - reflection->SetEnumValue(destination, mapping.Field, cell.Get<i32>()); + reflection->AddEnumValue(destination, mapping.Field, item.Get<i32>()); break; case EEnumFormatType::String: { - auto enumValueDescriptor = mapping.Field->enum_type()->FindValueByName(TString(cell.AsStringRef())); + auto enumValueDescriptor = mapping.Field->enum_type()->FindValueByName(TString(item.AsStringRef())); if (!enumValueDescriptor) { enumValueDescriptor = mapping.Field->default_value_enum(); } - reflection->SetEnum(destination, mapping.Field, enumValueDescriptor); + reflection->AddEnum(destination, mapping.Field, enumValueDescriptor); break; } } @@ -581,438 +495,497 @@ namespace { case FieldDescriptor::TYPE_UINT64: case FieldDescriptor::TYPE_FIXED64: - reflection->SetUInt64(destination, mapping.Field, cell.Get<ui64>()); + reflection->AddUInt64(destination, mapping.Field, item.Get<ui64>()); break; case FieldDescriptor::TYPE_INT32: case FieldDescriptor::TYPE_SFIXED32: case FieldDescriptor::TYPE_SINT32: - reflection->SetInt32(destination, mapping.Field, cell.Get<i32>()); + reflection->AddInt32(destination, mapping.Field, item.Get<i32>()); break; case FieldDescriptor::TYPE_UINT32: case FieldDescriptor::TYPE_FIXED32: - reflection->SetUInt32(destination, mapping.Field, cell.Get<ui32>()); + reflection->AddUInt32(destination, mapping.Field, item.Get<ui32>()); break; case FieldDescriptor::TYPE_BOOL: - reflection->SetBool(destination, mapping.Field, cell.Get<bool>()); + reflection->AddBool(destination, mapping.Field, item.Get<bool>()); break; case FieldDescriptor::TYPE_STRING: - reflection->SetString(destination, mapping.Field, TString(cell.AsStringRef())); + reflection->AddString(destination, mapping.Field, TString(item.AsStringRef())); break; case FieldDescriptor::TYPE_BYTES: - reflection->SetString(destination, mapping.Field, TString(cell.AsStringRef())); + reflection->AddString(destination, mapping.Field, TString(item.AsStringRef())); break; - case FieldDescriptor::TYPE_MESSAGE: - { - auto* nestedMessage = reflection->MutableMessage(destination, mapping.Field); - FillOutputMessage(cell, nestedMessage, mapping.NestedFields, enumPolicy); - } - break; + case FieldDescriptor::TYPE_MESSAGE: { + auto* nestedMessage = reflection->AddMessage(destination, mapping.Field); + FillOutputMessage(item, nestedMessage, mapping.NestedFields, enumPolicy); + } break; default: ythrow yexception() << "Unsupported protobuf type: " << mapping.Field->type_name() << ", field: " << mapping.Field->name(); } } + } else { + switch (type) { + case FieldDescriptor::TYPE_DOUBLE: + reflection->SetDouble(destination, mapping.Field, cell.Get<double>()); + break; + + case FieldDescriptor::TYPE_FLOAT: + reflection->SetFloat(destination, mapping.Field, cell.Get<float>()); + break; + + case FieldDescriptor::TYPE_INT64: + case FieldDescriptor::TYPE_SFIXED64: + case FieldDescriptor::TYPE_SINT64: + reflection->SetInt64(destination, mapping.Field, cell.Get<i64>()); + break; + + case FieldDescriptor::TYPE_ENUM: { + switch (EnumFormatType(*mapping.Field, enumPolicy)) { + case EEnumFormatType::Int32: + reflection->SetEnumValue(destination, mapping.Field, cell.Get<i32>()); + break; + case EEnumFormatType::String: { + auto enumValueDescriptor = mapping.Field->enum_type()->FindValueByName(TString(cell.AsStringRef())); + if (!enumValueDescriptor) { + enumValueDescriptor = mapping.Field->default_value_enum(); + } + reflection->SetEnum(destination, mapping.Field, enumValueDescriptor); + break; + } + } + break; + } + + case FieldDescriptor::TYPE_UINT64: + case FieldDescriptor::TYPE_FIXED64: + reflection->SetUInt64(destination, mapping.Field, cell.Get<ui64>()); + break; + + case FieldDescriptor::TYPE_INT32: + case FieldDescriptor::TYPE_SFIXED32: + case FieldDescriptor::TYPE_SINT32: + reflection->SetInt32(destination, mapping.Field, cell.Get<i32>()); + break; + + case FieldDescriptor::TYPE_UINT32: + case FieldDescriptor::TYPE_FIXED32: + reflection->SetUInt32(destination, mapping.Field, cell.Get<ui32>()); + break; + + case FieldDescriptor::TYPE_BOOL: + reflection->SetBool(destination, mapping.Field, cell.Get<bool>()); + break; + + case FieldDescriptor::TYPE_STRING: + reflection->SetString(destination, mapping.Field, TString(cell.AsStringRef())); + break; + + case FieldDescriptor::TYPE_BYTES: + reflection->SetString(destination, mapping.Field, TString(cell.AsStringRef())); + break; + + case FieldDescriptor::TYPE_MESSAGE: { + auto* nestedMessage = reflection->MutableMessage(destination, mapping.Field); + FillOutputMessage(cell, nestedMessage, mapping.NestedFields, enumPolicy); + } break; + + default: + ythrow yexception() << "Unsupported protobuf type: " + << mapping.Field->type_name() << ", field: " << mapping.Field->name(); + } } } +} - /** - * Converts input messages to unboxed values. - */ - class TInputConverter { - protected: - IWorker* Worker_; - TVector<TFieldMapping> Mappings_; - TPlainContainerCache Cache_; - TMaybe<TString> TimestampColumn_; - EEnumPolicy EnumPolicy_ = EEnumPolicy::Int32; - - public: - explicit TInputConverter(const TProtobufRawInputSpec& inputSpec, IWorker* worker) - : Worker_(worker) - , TimestampColumn_(inputSpec.GetTimestampColumn()) - , EnumPolicy_(inputSpec.GetSchemaOptions().EnumPolicy) - { - FillFieldMappings( - Worker_->GetInputType(), inputSpec.GetDescriptor(), - Mappings_, TimestampColumn_, - inputSpec.GetSchemaOptions().ListIsOptional, - inputSpec.GetSchemaOptions().EnableRecursiveRenaming, - inputSpec.GetSchemaOptions().FieldRenames - ); - } +/** + * Converts input messages to unboxed values. + */ +class TInputConverter { +protected: + IWorker* Worker_; + TVector<TFieldMapping> Mappings_; + TPlainContainerCache Cache_; + TMaybe<TString> TimestampColumn_; + EEnumPolicy EnumPolicy_ = EEnumPolicy::Int32; + +public: + explicit TInputConverter(const TProtobufRawInputSpec& inputSpec, IWorker* worker) + : Worker_(worker) + , TimestampColumn_(inputSpec.GetTimestampColumn()) + , EnumPolicy_(inputSpec.GetSchemaOptions().EnumPolicy) + { + FillFieldMappings( + Worker_->GetInputType(), inputSpec.GetDescriptor(), + Mappings_, TimestampColumn_, + inputSpec.GetSchemaOptions().ListIsOptional, + inputSpec.GetSchemaOptions().EnableRecursiveRenaming, + inputSpec.GetSchemaOptions().FieldRenames); + } + +public: + void DoConvert(const Message* message, TUnboxedValue& result) { + auto& holderFactory = Worker_->GetGraph().GetHolderFactory(); + TUnboxedValue* items = nullptr; + result = Cache_.NewArray(holderFactory, static_cast<ui32>(Mappings_.size()), items); + FillInputValue(holderFactory, message, items, Mappings_, TimestampColumn_, Worker_->GetTimeProvider(), EnumPolicy_); + } - public: - void DoConvert(const Message* message, TUnboxedValue& result) { - auto& holderFactory = Worker_->GetGraph().GetHolderFactory(); - TUnboxedValue* items = nullptr; - result = Cache_.NewArray(holderFactory, static_cast<ui32>(Mappings_.size()), items); - FillInputValue(holderFactory, message, items, Mappings_, TimestampColumn_, Worker_->GetTimeProvider(), EnumPolicy_); + void ClearCache() { + Cache_.Clear(); + } +}; + +template <typename TOutputSpec> +using OutputItemType = typename TOutputSpecTraits<TOutputSpec>::TOutputItemType; + +template <typename TOutputSpec> +class TOutputConverter; + +/** + * Converts unboxed values to output messages (single-output program case). + */ +template <> +class TOutputConverter<TProtobufRawOutputSpec> { +protected: + IWorker* Worker_; + TVector<TFieldMapping> OutputColumns_; + TProtoHolder<Message> Message_; + EEnumPolicy EnumPolicy_ = EEnumPolicy::Int32; + +public: + explicit TOutputConverter(const TProtobufRawOutputSpec& outputSpec, IWorker* worker) + : Worker_(worker) + , EnumPolicy_(outputSpec.GetSchemaOptions().EnumPolicy) + { + if (!Worker_->GetOutputType()->IsStruct()) { + ythrow yexception() << "protobuf output spec does not support multiple outputs"; } - void ClearCache() { - Cache_.Clear(); + FillFieldMappings( + static_cast<const NKikimr::NMiniKQL::TStructType*>(Worker_->GetOutputType()), + outputSpec.GetDescriptor(), + OutputColumns_, + Nothing(), + outputSpec.GetSchemaOptions().ListIsOptional, + outputSpec.GetSchemaOptions().EnableRecursiveRenaming, + outputSpec.GetSchemaOptions().FieldRenames); + + auto* factory = outputSpec.GetFactory(); + + if (!factory) { + factory = MessageFactory::generated_factory(); } - }; - - template <typename TOutputSpec> - using OutputItemType = typename TOutputSpecTraits<TOutputSpec>::TOutputItemType; - - template <typename TOutputSpec> - class TOutputConverter; - - /** - * Converts unboxed values to output messages (single-output program case). - */ - template <> - class TOutputConverter<TProtobufRawOutputSpec> { - protected: - IWorker* Worker_; - TVector<TFieldMapping> OutputColumns_; - TProtoHolder<Message> Message_; - EEnumPolicy EnumPolicy_ = EEnumPolicy::Int32; - - public: - explicit TOutputConverter(const TProtobufRawOutputSpec& outputSpec, IWorker* worker) - : Worker_(worker) - , EnumPolicy_(outputSpec.GetSchemaOptions().EnumPolicy) - { - if (!Worker_->GetOutputType()->IsStruct()) { - ythrow yexception() << "protobuf output spec does not support multiple outputs"; - } + + Message_.Reset(factory->GetPrototype(&outputSpec.GetDescriptor())->New(outputSpec.GetArena())); + } + + OutputItemType<TProtobufRawOutputSpec> DoConvert(TUnboxedValue value) { + FillOutputMessage(value, Message_.Get(), OutputColumns_, EnumPolicy_); + return Message_.Get(); + } +}; + +/* + * Converts unboxed values to output type (multi-output programs case). + */ +template <> +class TOutputConverter<TProtobufRawMultiOutputSpec> { +protected: + IWorker* Worker_; + TVector<TVector<TFieldMapping>> OutputColumns_; + TVector<TProtoHolder<Message>> Messages_; + EEnumPolicy EnumPolicy_ = EEnumPolicy::Int32; + +public: + explicit TOutputConverter(const TProtobufRawMultiOutputSpec& outputSpec, IWorker* worker) + : Worker_(worker) + , EnumPolicy_(outputSpec.GetSchemaOptions().EnumPolicy) + { + const auto* outputType = Worker_->GetOutputType(); + Y_ENSURE(outputType->IsVariant(), "protobuf multi-output spec requires multi-output program"); + const auto* variantType = static_cast<const NKikimr::NMiniKQL::TVariantType*>(outputType); + Y_ENSURE( + variantType->GetUnderlyingType()->IsTuple(), + "protobuf multi-output spec requires variant over tuple as program output type"); + Y_ENSURE( + outputSpec.GetOutputsNumber() == variantType->GetAlternativesCount(), + "number of outputs provided by spec does not match number of variant alternatives"); + + auto defaultFactory = MessageFactory::generated_factory(); + + for (ui32 i = 0; i < variantType->GetAlternativesCount(); ++i) { + const auto* type = variantType->GetAlternativeType(i); + Y_ASSERT(type->IsStruct()); + Y_ASSERT(OutputColumns_.size() == i && Messages_.size() == i); + + OutputColumns_.push_back({}); FillFieldMappings( - static_cast<const NKikimr::NMiniKQL::TStructType*>(Worker_->GetOutputType()), - outputSpec.GetDescriptor(), - OutputColumns_, + static_cast<const NKikimr::NMiniKQL::TStructType*>(type), + outputSpec.GetDescriptor(i), + OutputColumns_.back(), Nothing(), outputSpec.GetSchemaOptions().ListIsOptional, - outputSpec.GetSchemaOptions().EnableRecursiveRenaming, - outputSpec.GetSchemaOptions().FieldRenames - ); - - auto* factory = outputSpec.GetFactory(); + false, + {}); + auto factory = outputSpec.GetFactory(i); if (!factory) { - factory = MessageFactory::generated_factory(); + factory = defaultFactory; } - Message_.Reset(factory->GetPrototype(&outputSpec.GetDescriptor())->New(outputSpec.GetArena())); - } - - OutputItemType<TProtobufRawOutputSpec> DoConvert(TUnboxedValue value) { - FillOutputMessage(value, Message_.Get(), OutputColumns_, EnumPolicy_); - return Message_.Get(); + Messages_.push_back(TProtoHolder<Message>( + factory->GetPrototype(&outputSpec.GetDescriptor(i))->New(outputSpec.GetArena(i)))); } - }; - - /* - * Converts unboxed values to output type (multi-output programs case). - */ - template <> - class TOutputConverter<TProtobufRawMultiOutputSpec> { - protected: - IWorker* Worker_; - TVector<TVector<TFieldMapping>> OutputColumns_; - TVector<TProtoHolder<Message>> Messages_; - EEnumPolicy EnumPolicy_ = EEnumPolicy::Int32; - - public: - explicit TOutputConverter(const TProtobufRawMultiOutputSpec& outputSpec, IWorker* worker) - : Worker_(worker) - , EnumPolicy_(outputSpec.GetSchemaOptions().EnumPolicy) - { - const auto* outputType = Worker_->GetOutputType(); - Y_ENSURE(outputType->IsVariant(), "protobuf multi-output spec requires multi-output program"); - const auto* variantType = static_cast<const NKikimr::NMiniKQL::TVariantType*>(outputType); - Y_ENSURE( - variantType->GetUnderlyingType()->IsTuple(), - "protobuf multi-output spec requires variant over tuple as program output type" - ); - Y_ENSURE( - outputSpec.GetOutputsNumber() == variantType->GetAlternativesCount(), - "number of outputs provided by spec does not match number of variant alternatives" - ); - - auto defaultFactory = MessageFactory::generated_factory(); - - for (ui32 i = 0; i < variantType->GetAlternativesCount(); ++i) { - const auto* type = variantType->GetAlternativeType(i); - Y_ASSERT(type->IsStruct()); - Y_ASSERT(OutputColumns_.size() == i && Messages_.size() == i); - - OutputColumns_.push_back({}); - - FillFieldMappings( - static_cast<const NKikimr::NMiniKQL::TStructType*>(type), - outputSpec.GetDescriptor(i), - OutputColumns_.back(), - Nothing(), - outputSpec.GetSchemaOptions().ListIsOptional, - false, - {} - ); - - auto factory = outputSpec.GetFactory(i); - if (!factory) { - factory = defaultFactory; - } + } - Messages_.push_back(TProtoHolder<Message>( - factory->GetPrototype(&outputSpec.GetDescriptor(i))->New(outputSpec.GetArena(i)) - )); - } - } + OutputItemType<TProtobufRawMultiOutputSpec> DoConvert(TUnboxedValue value) { + auto index = value.GetVariantIndex(); + auto msgPtr = Messages_[index].Get(); + FillOutputMessage(value.GetVariantItem(), msgPtr, OutputColumns_[index], EnumPolicy_); + return {index, msgPtr}; + } +}; + +/** + * List (or, better, stream) of unboxed values. Used as an input value in pull workers. + */ +class TProtoListValue final: public TCustomListValue { +private: + mutable bool HasIterator_ = false; + THolder<IStream<Message*>> Underlying_; + TInputConverter Converter_; + IWorker* Worker_; + TScopedAlloc& ScopedAlloc_; + +public: + TProtoListValue( + TMemoryUsageInfo* memInfo, + const TProtobufRawInputSpec& inputSpec, + THolder<IStream<Message*>> underlying, + IWorker* worker) + : TCustomListValue(memInfo) + , Underlying_(std::move(underlying)) + , Converter_(inputSpec, worker) + , Worker_(worker) + , ScopedAlloc_(Worker_->GetScopedAlloc()) + { + } - OutputItemType<TProtobufRawMultiOutputSpec> DoConvert(TUnboxedValue value) { - auto index = value.GetVariantIndex(); - auto msgPtr = Messages_[index].Get(); - FillOutputMessage(value.GetVariantItem(), msgPtr, OutputColumns_[index], EnumPolicy_); - return {index, msgPtr}; - } - }; - - /** - * List (or, better, stream) of unboxed values. Used as an input value in pull workers. - */ - class TProtoListValue final: public TCustomListValue { - private: - mutable bool HasIterator_ = false; - THolder<IStream<Message*>> Underlying_; - TInputConverter Converter_; - IWorker* Worker_; - TScopedAlloc& ScopedAlloc_; - - public: - TProtoListValue( - TMemoryUsageInfo* memInfo, - const TProtobufRawInputSpec& inputSpec, - THolder<IStream<Message*>> underlying, - IWorker* worker - ) - : TCustomListValue(memInfo) - , Underlying_(std::move(underlying)) - , Converter_(inputSpec, worker) - , Worker_(worker) - , ScopedAlloc_(Worker_->GetScopedAlloc()) + ~TProtoListValue() override { { + // This list value stored in the worker's computation graph and destroyed upon the computation + // graph's destruction. This brings us to an interesting situation: scoped alloc is acquired, + // worker and computation graph are half-way destroyed, and now it's our turn to die. The problem is, + // the underlying stream may own another worker. This happens when chaining programs. Now, to destroy + // that worker correctly, we need to release our scoped alloc (because that worker has its own + // computation graph and scoped alloc). + // By the way, note that we shouldn't interact with the worker here because worker is in the middle of + // its own destruction. So we're using our own reference to the scoped alloc. That reference is alive + // because scoped alloc destroyed after computation graph. + auto unguard = Unguard(ScopedAlloc_); + Underlying_.Destroy(); } + } - ~TProtoListValue() override { - { - // This list value stored in the worker's computation graph and destroyed upon the computation - // graph's destruction. This brings us to an interesting situation: scoped alloc is acquired, - // worker and computation graph are half-way destroyed, and now it's our turn to die. The problem is, - // the underlying stream may own another worker. This happens when chaining programs. Now, to destroy - // that worker correctly, we need to release our scoped alloc (because that worker has its own - // computation graph and scoped alloc). - // By the way, note that we shouldn't interact with the worker here because worker is in the middle of - // its own destruction. So we're using our own reference to the scoped alloc. That reference is alive - // because scoped alloc destroyed after computation graph. - auto unguard = Unguard(ScopedAlloc_); - Underlying_.Destroy(); - } - } +public: + TUnboxedValue GetListIterator() const override { + YQL_ENSURE(!HasIterator_, "Only one pass over input is supported"); + HasIterator_ = true; + return TUnboxedValuePod(const_cast<TProtoListValue*>(this)); + } - public: - TUnboxedValue GetListIterator() const override { - YQL_ENSURE(!HasIterator_, "Only one pass over input is supported"); - HasIterator_ = true; - return TUnboxedValuePod(const_cast<TProtoListValue*>(this)); + bool Next(TUnboxedValue& result) override { + const Message* message; + { + auto unguard = Unguard(ScopedAlloc_); + message = Underlying_->Fetch(); } - bool Next(TUnboxedValue& result) override { - const Message* message; - { - auto unguard = Unguard(ScopedAlloc_); - message = Underlying_->Fetch(); - } - - if (!message) { - return false; - } + if (!message) { + return false; + } - Converter_.DoConvert(message, result); + Converter_.DoConvert(message, result); - return true; - } + return true; + } - EFetchStatus Fetch(TUnboxedValue& result) override { - if (Next(result)) { - return EFetchStatus::Ok; - } else { - return EFetchStatus::Finish; - } - } - }; - - /** - * Consumer which converts messages to unboxed values and relays them to the worker. Used as a return value - * of the push processor's Process function. - */ - class TProtoConsumerImpl final: public IConsumer<Message*> { - private: - TWorkerHolder<IPushStreamWorker> WorkerHolder_; - TInputConverter Converter_; - - public: - explicit TProtoConsumerImpl( - const TProtobufRawInputSpec& inputSpec, - TWorkerHolder<IPushStreamWorker> worker - ) - : WorkerHolder_(std::move(worker)) - , Converter_(inputSpec, WorkerHolder_.Get()) - { + EFetchStatus Fetch(TUnboxedValue& result) override { + if (Next(result)) { + return EFetchStatus::Ok; + } else { + return EFetchStatus::Finish; } + } +}; + +/** + * Consumer which converts messages to unboxed values and relays them to the worker. Used as a return value + * of the push processor's Process function. + */ +class TProtoConsumerImpl final: public IConsumer<Message*> { +private: + TWorkerHolder<IPushStreamWorker> WorkerHolder_; + TInputConverter Converter_; + +public: + explicit TProtoConsumerImpl( + const TProtobufRawInputSpec& inputSpec, + TWorkerHolder<IPushStreamWorker> worker) + : WorkerHolder_(std::move(worker)) + , Converter_(inputSpec, WorkerHolder_.Get()) + { + } - ~TProtoConsumerImpl() override { - with_lock(WorkerHolder_->GetScopedAlloc()) { - Converter_.ClearCache(); - } + ~TProtoConsumerImpl() override { + with_lock (WorkerHolder_->GetScopedAlloc()) { + Converter_.ClearCache(); } + } - public: - void OnObject(Message* message) override { - TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); +public: + void OnObject(Message* message) override { + TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - with_lock(WorkerHolder_->GetScopedAlloc()) { - TUnboxedValue result; - Converter_.DoConvert(message, result); - WorkerHolder_->Push(std::move(result)); - } + with_lock (WorkerHolder_->GetScopedAlloc()) { + TUnboxedValue result; + Converter_.DoConvert(message, result); + WorkerHolder_->Push(std::move(result)); } + } - void OnFinish() override { - TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); + void OnFinish() override { + TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - with_lock(WorkerHolder_->GetScopedAlloc()) { - WorkerHolder_->OnFinish(); - } - } - }; - - /** - * Protobuf input stream for unboxed value streams. - */ - template <typename TOutputSpec> - class TRawProtoStreamImpl final: public IStream<OutputItemType<TOutputSpec>> { - protected: - TWorkerHolder<IPullStreamWorker> WorkerHolder_; - TOutputConverter<TOutputSpec> Converter_; - - public: - explicit TRawProtoStreamImpl(const TOutputSpec& outputSpec, TWorkerHolder<IPullStreamWorker> worker) - : WorkerHolder_(std::move(worker)) - , Converter_(outputSpec, WorkerHolder_.Get()) - { + with_lock (WorkerHolder_->GetScopedAlloc()) { + WorkerHolder_->OnFinish(); } + } +}; - public: - OutputItemType<TOutputSpec> Fetch() override { - TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); +/** + * Protobuf input stream for unboxed value streams. + */ +template <typename TOutputSpec> +class TRawProtoStreamImpl final: public IStream<OutputItemType<TOutputSpec>> { +protected: + TWorkerHolder<IPullStreamWorker> WorkerHolder_; + TOutputConverter<TOutputSpec> Converter_; + +public: + explicit TRawProtoStreamImpl(const TOutputSpec& outputSpec, TWorkerHolder<IPullStreamWorker> worker) + : WorkerHolder_(std::move(worker)) + , Converter_(outputSpec, WorkerHolder_.Get()) + { + } - with_lock(WorkerHolder_->GetScopedAlloc()) { - TUnboxedValue value; +public: + OutputItemType<TOutputSpec> Fetch() override { + TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - auto status = WorkerHolder_->GetOutput().Fetch(value); + with_lock (WorkerHolder_->GetScopedAlloc()) { + TUnboxedValue value; - YQL_ENSURE(status != EFetchStatus::Yield, "Yield is not supported in pull mode"); + auto status = WorkerHolder_->GetOutput().Fetch(value); - if (status == EFetchStatus::Finish) { - return TOutputSpecTraits<TOutputSpec>::StreamSentinel; - } + YQL_ENSURE(status != EFetchStatus::Yield, "Yield is not supported in pull mode"); - return Converter_.DoConvert(value); + if (status == EFetchStatus::Finish) { + return TOutputSpecTraits<TOutputSpec>::StreamSentinel; } + + return Converter_.DoConvert(value); } - }; - - /** - * Protobuf input stream for unboxed value lists. - */ - template <typename TOutputSpec> - class TRawProtoListImpl final: public IStream<OutputItemType<TOutputSpec>> { - protected: - TWorkerHolder<IPullListWorker> WorkerHolder_; - TOutputConverter<TOutputSpec> Converter_; - - public: - explicit TRawProtoListImpl(const TOutputSpec& outputSpec, TWorkerHolder<IPullListWorker> worker) - : WorkerHolder_(std::move(worker)) - , Converter_(outputSpec, WorkerHolder_.Get()) - { - } + } +}; - public: - OutputItemType<TOutputSpec> Fetch() override { - TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); +/** + * Protobuf input stream for unboxed value lists. + */ +template <typename TOutputSpec> +class TRawProtoListImpl final: public IStream<OutputItemType<TOutputSpec>> { +protected: + TWorkerHolder<IPullListWorker> WorkerHolder_; + TOutputConverter<TOutputSpec> Converter_; + +public: + explicit TRawProtoListImpl(const TOutputSpec& outputSpec, TWorkerHolder<IPullListWorker> worker) + : WorkerHolder_(std::move(worker)) + , Converter_(outputSpec, WorkerHolder_.Get()) + { + } - with_lock(WorkerHolder_->GetScopedAlloc()) { - TUnboxedValue value; +public: + OutputItemType<TOutputSpec> Fetch() override { + TBindTerminator bind(WorkerHolder_->GetGraph().GetTerminator()); - if (!WorkerHolder_->GetOutputIterator().Next(value)) { - return TOutputSpecTraits<TOutputSpec>::StreamSentinel; - } + with_lock (WorkerHolder_->GetScopedAlloc()) { + TUnboxedValue value; - return Converter_.DoConvert(value); + if (!WorkerHolder_->GetOutputIterator().Next(value)) { + return TOutputSpecTraits<TOutputSpec>::StreamSentinel; } + + return Converter_.DoConvert(value); } - }; - - /** - * Push relay used to convert generated unboxed value to a message and push it to the user's consumer. - */ - template <typename TOutputSpec> - class TPushRelayImpl: public IConsumer<const TUnboxedValue*> { - private: - THolder<IConsumer<OutputItemType<TOutputSpec>>> Underlying_; - TOutputConverter<TOutputSpec> Converter_; - IWorker* Worker_; - - public: - TPushRelayImpl( - const TOutputSpec& outputSpec, - IPushStreamWorker* worker, - THolder<IConsumer<OutputItemType<TOutputSpec>>> underlying - ) - : Underlying_(std::move(underlying)) - , Converter_(outputSpec, worker) - , Worker_(worker) - { - } + } +}; + +/** + * Push relay used to convert generated unboxed value to a message and push it to the user's consumer. + */ +template <typename TOutputSpec> +class TPushRelayImpl: public IConsumer<const TUnboxedValue*> { +private: + THolder<IConsumer<OutputItemType<TOutputSpec>>> Underlying_; + TOutputConverter<TOutputSpec> Converter_; + IWorker* Worker_; + +public: + TPushRelayImpl( + const TOutputSpec& outputSpec, + IPushStreamWorker* worker, + THolder<IConsumer<OutputItemType<TOutputSpec>>> underlying) + : Underlying_(std::move(underlying)) + , Converter_(outputSpec, worker) + , Worker_(worker) + { + } - // If you've read a comment in the TProtoListValue's destructor, you may be wondering why don't we do the - // same trick here. Well, that's because in push mode, consumer is destroyed before acquiring scoped alloc and - // destroying computation graph. + // If you've read a comment in the TProtoListValue's destructor, you may be wondering why don't we do the + // same trick here. Well, that's because in push mode, consumer is destroyed before acquiring scoped alloc and + // destroying computation graph. - public: - void OnObject(const TUnboxedValue* value) override { - OutputItemType<TOutputSpec> message = Converter_.DoConvert(*value); - auto unguard = Unguard(Worker_->GetScopedAlloc()); - Underlying_->OnObject(message); - } +public: + void OnObject(const TUnboxedValue* value) override { + OutputItemType<TOutputSpec> message = Converter_.DoConvert(*value); + auto unguard = Unguard(Worker_->GetScopedAlloc()); + Underlying_->OnObject(message); + } - void OnFinish() override { - auto unguard = Unguard(Worker_->GetScopedAlloc()); - Underlying_->OnFinish(); - } - }; -} + void OnFinish() override { + auto unguard = Unguard(Worker_->GetScopedAlloc()); + Underlying_->OnFinish(); + } +}; +} // namespace using ConsumerType = TInputSpecTraits<TProtobufRawInputSpec>::TConsumerType; void TInputSpecTraits<TProtobufRawInputSpec>::PreparePullStreamWorker( const TProtobufRawInputSpec& inputSpec, IPullStreamWorker* worker, - THolder<IStream<Message*>> stream -) { - with_lock(worker->GetScopedAlloc()) { + THolder<IStream<Message*>> stream) { + with_lock (worker->GetScopedAlloc()) { worker->SetInput( worker->GetGraph().GetHolderFactory().Create<TProtoListValue>(inputSpec, std::move(stream), worker), 0); } @@ -1021,9 +994,8 @@ void TInputSpecTraits<TProtobufRawInputSpec>::PreparePullStreamWorker( void TInputSpecTraits<TProtobufRawInputSpec>::PreparePullListWorker( const TProtobufRawInputSpec& inputSpec, IPullListWorker* worker, - THolder<IStream<Message*>> stream -) { - with_lock(worker->GetScopedAlloc()) { + THolder<IStream<Message*>> stream) { + with_lock (worker->GetScopedAlloc()) { worker->SetInput( worker->GetGraph().GetHolderFactory().Create<TProtoListValue>(inputSpec, std::move(stream), worker), 0); } @@ -1031,8 +1003,7 @@ void TInputSpecTraits<TProtobufRawInputSpec>::PreparePullListWorker( ConsumerType TInputSpecTraits<TProtobufRawInputSpec>::MakeConsumer( const TProtobufRawInputSpec& inputSpec, - TWorkerHolder<IPushStreamWorker> worker -) { + TWorkerHolder<IPushStreamWorker> worker) { return MakeHolder<TProtoConsumerImpl>(inputSpec, std::move(worker)); } @@ -1043,44 +1014,38 @@ using PullListReturnType = typename TOutputSpecTraits<TOutputSpec>::TPullListRet PullStreamReturnType<TProtobufRawOutputSpec> TOutputSpecTraits<TProtobufRawOutputSpec>::ConvertPullStreamWorkerToOutputType( const TProtobufRawOutputSpec& outputSpec, - TWorkerHolder<IPullStreamWorker> worker -) { + TWorkerHolder<IPullStreamWorker> worker) { return MakeHolder<TRawProtoStreamImpl<TProtobufRawOutputSpec>>(outputSpec, std::move(worker)); } PullListReturnType<TProtobufRawOutputSpec> TOutputSpecTraits<TProtobufRawOutputSpec>::ConvertPullListWorkerToOutputType( const TProtobufRawOutputSpec& outputSpec, - TWorkerHolder<IPullListWorker> worker -) { + TWorkerHolder<IPullListWorker> worker) { return MakeHolder<TRawProtoListImpl<TProtobufRawOutputSpec>>(outputSpec, std::move(worker)); } void TOutputSpecTraits<TProtobufRawOutputSpec>::SetConsumerToWorker( const TProtobufRawOutputSpec& outputSpec, IPushStreamWorker* worker, - THolder<IConsumer<TOutputItemType>> consumer -) { + THolder<IConsumer<TOutputItemType>> consumer) { worker->SetConsumer(MakeHolder<TPushRelayImpl<TProtobufRawOutputSpec>>(outputSpec, worker, std::move(consumer))); } PullStreamReturnType<TProtobufRawMultiOutputSpec> TOutputSpecTraits<TProtobufRawMultiOutputSpec>::ConvertPullStreamWorkerToOutputType( const TProtobufRawMultiOutputSpec& outputSpec, - TWorkerHolder<IPullStreamWorker> worker -) { + TWorkerHolder<IPullStreamWorker> worker) { return MakeHolder<TRawProtoStreamImpl<TProtobufRawMultiOutputSpec>>(outputSpec, std::move(worker)); } PullListReturnType<TProtobufRawMultiOutputSpec> TOutputSpecTraits<TProtobufRawMultiOutputSpec>::ConvertPullListWorkerToOutputType( const TProtobufRawMultiOutputSpec& outputSpec, - TWorkerHolder<IPullListWorker> worker -) { + TWorkerHolder<IPullListWorker> worker) { return MakeHolder<TRawProtoListImpl<TProtobufRawMultiOutputSpec>>(outputSpec, std::move(worker)); } void TOutputSpecTraits<TProtobufRawMultiOutputSpec>::SetConsumerToWorker( const TProtobufRawMultiOutputSpec& outputSpec, IPushStreamWorker* worker, - THolder<IConsumer<TOutputItemType>> consumer -) { + THolder<IConsumer<TOutputItemType>> consumer) { worker->SetConsumer(MakeHolder<TPushRelayImpl<TProtobufRawMultiOutputSpec>>(outputSpec, worker, std::move(consumer))); } diff --git a/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.h b/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.h index 436b243bffd..a2ef2829a10 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.h +++ b/yql/essentials/public/purecalc/io_specs/protobuf_raw/spec.h @@ -8,250 +8,247 @@ #include <util/generic/maybe.h> namespace NYql { - namespace NPureCalc { - /** - * Processing mode for working with raw protobuf message inputs. - * - * In this mode purecalc accept pointers to abstract protobuf messages and processes them using the reflection - * mechanism. All passed messages should have the same descriptor (the one you pass to the constructor - * of the input spec). - * - * All working modes are implemented. In pull stream and pull list modes a program would accept a single object - * stream of const protobuf messages. In push mode, a program will return a consumer of const protobuf messages. - * - * The program synopsis follows: - * - * @code - * ... TPullStreamProgram::Apply(IStream<google::protobuf::Message*>); - * ... TPullListProgram::Apply(IStream<google::protobuf::Message*>); - * TConsumer<google::protobuf::Message*> TPushStreamProgram::Apply(...); - * @endcode - */ - class TProtobufRawInputSpec: public TInputSpecBase { - private: - const google::protobuf::Descriptor& Descriptor_; - const TMaybe<TString> TimestampColumn_; - const TProtoSchemaOptions SchemaOptions_; - mutable TVector<NYT::TNode> SavedSchemas_; - - public: - /** - * Build input spec and associate the given message descriptor. - */ - explicit TProtobufRawInputSpec( - const google::protobuf::Descriptor& descriptor, - const TMaybe<TString>& timestampColumn = Nothing(), - const TProtoSchemaOptions& options = {} - ); - - public: - const TVector<NYT::TNode>& GetSchemas() const override; - - /** - * Get the descriptor associated with this spec. - */ - const google::protobuf::Descriptor& GetDescriptor() const; - - const TMaybe<TString>& GetTimestampColumn() const; - - /* - * Get options that customize input struct type building. - */ - const TProtoSchemaOptions& GetSchemaOptions() const; - }; - - /** - * Processing mode for working with raw protobuf message outputs. - * - * In this mode purecalc yields pointers to abstract protobuf messages. All generated messages share the same - * descriptor so they can be safely converted into an appropriate message type. - * - * Note that one should not expect that the returned pointer will be valid forever; in can (and will) become - * outdated once a new output is requested/pushed. - * - * All working modes are implemented. In pull stream and pull list modes a program will return an object - * stream of non-const protobuf messages. In push mode, it will accept a single consumer of non-const - * messages. - * - * The program synopsis follows: - * - * @code - * IStream<google::protobuf::Message*> TPullStreamProgram::Apply(...); - * IStream<google::protobuf::Message*> TPullListProgram::Apply(...); - * ... TPushStreamProgram::Apply(TConsumer<google::protobuf::Message*>); - * @endcode - */ - class TProtobufRawOutputSpec: public TOutputSpecBase { - private: - const google::protobuf::Descriptor& Descriptor_; - google::protobuf::MessageFactory* Factory_; - TProtoSchemaOptions SchemaOptions_; - google::protobuf::Arena* Arena_; - mutable TMaybe<NYT::TNode> SavedSchema_; - - public: - /** - * Build output spec and associate the given message descriptor and maybe the given message factory. - */ - explicit TProtobufRawOutputSpec( - const google::protobuf::Descriptor& descriptor, - google::protobuf::MessageFactory* = nullptr, - const TProtoSchemaOptions& options = {}, - google::protobuf::Arena* arena = nullptr - ); - - public: - const NYT::TNode& GetSchema() const override; - - /** - * Get the descriptor associated with this spec. - */ - const google::protobuf::Descriptor& GetDescriptor() const; - - /** - * Set a new message factory which will be used to generate messages. Pass a null pointer to use the - * default factory. - */ - void SetFactory(google::protobuf::MessageFactory*); - - /** - * Get the message factory which is currently associated with this spec. - */ - google::protobuf::MessageFactory* GetFactory() const; - - /** - * Set a new arena which will be used to generate messages. Pass a null pointer to create on the heap. - */ - void SetArena(google::protobuf::Arena*); - - /** - * Get the arena which is currently associated with this spec. - */ - google::protobuf::Arena* GetArena() const; - - /** - * Get options that customize output struct type building. - */ - const TProtoSchemaOptions& GetSchemaOptions() const; - }; - - /** - * Processing mode for working with raw protobuf messages and several outputs. - * - * The program synopsis follows: - * - * @code - * IStream<std::pair<ui32, google::protobuf::Message*>> TPullStreamProgram::Apply(...); - * IStream<std::pair<ui32, google::protobuf::Message*>> TPullListProgram::Apply(...); - * ... TPushStreamProgram::Apply(TConsumer<std::pair<ui32, google::protobuf::Message*>>); - * @endcode - */ - class TProtobufRawMultiOutputSpec: public TOutputSpecBase { - private: - TVector<const google::protobuf::Descriptor*> Descriptors_; - TVector<google::protobuf::MessageFactory*> Factories_; - const TProtoSchemaOptions SchemaOptions_; - TVector<google::protobuf::Arena*> Arenas_; - mutable NYT::TNode SavedSchema_; - - public: - TProtobufRawMultiOutputSpec( - TVector<const google::protobuf::Descriptor*>, - TMaybe<TVector<google::protobuf::MessageFactory*>> = {}, - const TProtoSchemaOptions& options = {}, - TMaybe<TVector<google::protobuf::Arena*>> arenas = {} - ); - - public: - const NYT::TNode& GetSchema() const override; - - /** - * Get the descriptor associated with given output. - */ - const google::protobuf::Descriptor& GetDescriptor(ui32) const; - - /** - * Set a new message factory for given output. It will be used to generate messages for this output. - */ - void SetFactory(ui32, google::protobuf::MessageFactory*); - - /** - * Get the message factory which is currently associated with given output. - */ - google::protobuf::MessageFactory* GetFactory(ui32) const; - - /** - * Set a new arena for given output. It will be used to generate messages for this output. - */ - void SetArena(ui32, google::protobuf::Arena*); - - /** - * Get the arena which is currently associated with given output. - */ - google::protobuf::Arena* GetArena(ui32) const; - - /** - * Get number of outputs for this spec. - */ - ui32 GetOutputsNumber() const; - - /** - * Get options that customize output struct type building. - */ - const TProtoSchemaOptions& GetSchemaOptions() const; - }; - - template <> - struct TInputSpecTraits<TProtobufRawInputSpec> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = true; - static const constexpr bool SupportPullListMode = true; - static const constexpr bool SupportPushStreamMode = true; - - using TConsumerType = THolder<IConsumer<google::protobuf::Message*>>; - - static void PreparePullStreamWorker(const TProtobufRawInputSpec&, IPullStreamWorker*, THolder<IStream<google::protobuf::Message*>>); - static void PreparePullListWorker(const TProtobufRawInputSpec&, IPullListWorker*, THolder<IStream<google::protobuf::Message*>>); - static TConsumerType MakeConsumer(const TProtobufRawInputSpec&, TWorkerHolder<IPushStreamWorker>); - }; - - template <> - struct TOutputSpecTraits<TProtobufRawOutputSpec> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = true; - static const constexpr bool SupportPullListMode = true; - static const constexpr bool SupportPushStreamMode = true; - - using TOutputItemType = google::protobuf::Message*; - using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; - using TPullListReturnType = THolder<IStream<TOutputItemType>>; - - static const constexpr TOutputItemType StreamSentinel = nullptr; - - static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufRawOutputSpec&, TWorkerHolder<IPullStreamWorker>); - static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufRawOutputSpec&, TWorkerHolder<IPullListWorker>); - static void SetConsumerToWorker(const TProtobufRawOutputSpec&, IPushStreamWorker*, THolder<IConsumer<TOutputItemType>>); - }; - - template <> - struct TOutputSpecTraits<TProtobufRawMultiOutputSpec> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = true; - static const constexpr bool SupportPullListMode = true; - static const constexpr bool SupportPushStreamMode = true; - - using TOutputItemType = std::pair<ui32, google::protobuf::Message*>; - using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; - using TPullListReturnType = THolder<IStream<TOutputItemType>>; - - static const constexpr TOutputItemType StreamSentinel = {0, nullptr}; - - static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufRawMultiOutputSpec&, TWorkerHolder<IPullStreamWorker>); - static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufRawMultiOutputSpec&, TWorkerHolder<IPullListWorker>); - static void SetConsumerToWorker(const TProtobufRawMultiOutputSpec&, IPushStreamWorker*, THolder<IConsumer<TOutputItemType>>); - }; - } -} +namespace NPureCalc { +/** + * Processing mode for working with raw protobuf message inputs. + * + * In this mode purecalc accept pointers to abstract protobuf messages and processes them using the reflection + * mechanism. All passed messages should have the same descriptor (the one you pass to the constructor + * of the input spec). + * + * All working modes are implemented. In pull stream and pull list modes a program would accept a single object + * stream of const protobuf messages. In push mode, a program will return a consumer of const protobuf messages. + * + * The program synopsis follows: + * + * @code + * ... TPullStreamProgram::Apply(IStream<google::protobuf::Message*>); + * ... TPullListProgram::Apply(IStream<google::protobuf::Message*>); + * TConsumer<google::protobuf::Message*> TPushStreamProgram::Apply(...); + * @endcode + */ +class TProtobufRawInputSpec: public TInputSpecBase { +private: + const google::protobuf::Descriptor& Descriptor_; + const TMaybe<TString> TimestampColumn_; + const TProtoSchemaOptions SchemaOptions_; + mutable TVector<NYT::TNode> SavedSchemas_; + +public: + /** + * Build input spec and associate the given message descriptor. + */ + explicit TProtobufRawInputSpec( + const google::protobuf::Descriptor& descriptor, + const TMaybe<TString>& timestampColumn = Nothing(), + const TProtoSchemaOptions& options = {}); + +public: + const TVector<NYT::TNode>& GetSchemas() const override; + + /** + * Get the descriptor associated with this spec. + */ + const google::protobuf::Descriptor& GetDescriptor() const; + + const TMaybe<TString>& GetTimestampColumn() const; + + /* + * Get options that customize input struct type building. + */ + const TProtoSchemaOptions& GetSchemaOptions() const; +}; + +/** + * Processing mode for working with raw protobuf message outputs. + * + * In this mode purecalc yields pointers to abstract protobuf messages. All generated messages share the same + * descriptor so they can be safely converted into an appropriate message type. + * + * Note that one should not expect that the returned pointer will be valid forever; in can (and will) become + * outdated once a new output is requested/pushed. + * + * All working modes are implemented. In pull stream and pull list modes a program will return an object + * stream of non-const protobuf messages. In push mode, it will accept a single consumer of non-const + * messages. + * + * The program synopsis follows: + * + * @code + * IStream<google::protobuf::Message*> TPullStreamProgram::Apply(...); + * IStream<google::protobuf::Message*> TPullListProgram::Apply(...); + * ... TPushStreamProgram::Apply(TConsumer<google::protobuf::Message*>); + * @endcode + */ +class TProtobufRawOutputSpec: public TOutputSpecBase { +private: + const google::protobuf::Descriptor& Descriptor_; + google::protobuf::MessageFactory* Factory_; + TProtoSchemaOptions SchemaOptions_; + google::protobuf::Arena* Arena_; + mutable TMaybe<NYT::TNode> SavedSchema_; + +public: + /** + * Build output spec and associate the given message descriptor and maybe the given message factory. + */ + explicit TProtobufRawOutputSpec( + const google::protobuf::Descriptor& descriptor, + google::protobuf::MessageFactory* = nullptr, + const TProtoSchemaOptions& options = {}, + google::protobuf::Arena* arena = nullptr); + +public: + const NYT::TNode& GetSchema() const override; + + /** + * Get the descriptor associated with this spec. + */ + const google::protobuf::Descriptor& GetDescriptor() const; + + /** + * Set a new message factory which will be used to generate messages. Pass a null pointer to use the + * default factory. + */ + void SetFactory(google::protobuf::MessageFactory*); + + /** + * Get the message factory which is currently associated with this spec. + */ + google::protobuf::MessageFactory* GetFactory() const; + + /** + * Set a new arena which will be used to generate messages. Pass a null pointer to create on the heap. + */ + void SetArena(google::protobuf::Arena*); + + /** + * Get the arena which is currently associated with this spec. + */ + google::protobuf::Arena* GetArena() const; + + /** + * Get options that customize output struct type building. + */ + const TProtoSchemaOptions& GetSchemaOptions() const; +}; + +/** + * Processing mode for working with raw protobuf messages and several outputs. + * + * The program synopsis follows: + * + * @code + * IStream<std::pair<ui32, google::protobuf::Message*>> TPullStreamProgram::Apply(...); + * IStream<std::pair<ui32, google::protobuf::Message*>> TPullListProgram::Apply(...); + * ... TPushStreamProgram::Apply(TConsumer<std::pair<ui32, google::protobuf::Message*>>); + * @endcode + */ +class TProtobufRawMultiOutputSpec: public TOutputSpecBase { +private: + TVector<const google::protobuf::Descriptor*> Descriptors_; + TVector<google::protobuf::MessageFactory*> Factories_; + const TProtoSchemaOptions SchemaOptions_; + TVector<google::protobuf::Arena*> Arenas_; + mutable NYT::TNode SavedSchema_; + +public: + TProtobufRawMultiOutputSpec( + TVector<const google::protobuf::Descriptor*>, + TMaybe<TVector<google::protobuf::MessageFactory*>> = {}, + const TProtoSchemaOptions& options = {}, + TMaybe<TVector<google::protobuf::Arena*>> arenas = {}); + +public: + const NYT::TNode& GetSchema() const override; + + /** + * Get the descriptor associated with given output. + */ + const google::protobuf::Descriptor& GetDescriptor(ui32) const; + + /** + * Set a new message factory for given output. It will be used to generate messages for this output. + */ + void SetFactory(ui32, google::protobuf::MessageFactory*); + + /** + * Get the message factory which is currently associated with given output. + */ + google::protobuf::MessageFactory* GetFactory(ui32) const; + + /** + * Set a new arena for given output. It will be used to generate messages for this output. + */ + void SetArena(ui32, google::protobuf::Arena*); + + /** + * Get the arena which is currently associated with given output. + */ + google::protobuf::Arena* GetArena(ui32) const; + + /** + * Get number of outputs for this spec. + */ + ui32 GetOutputsNumber() const; + + /** + * Get options that customize output struct type building. + */ + const TProtoSchemaOptions& GetSchemaOptions() const; +}; + +template <> +struct TInputSpecTraits<TProtobufRawInputSpec> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = true; + static const constexpr bool SupportPullListMode = true; + static const constexpr bool SupportPushStreamMode = true; + + using TConsumerType = THolder<IConsumer<google::protobuf::Message*>>; + + static void PreparePullStreamWorker(const TProtobufRawInputSpec&, IPullStreamWorker*, THolder<IStream<google::protobuf::Message*>>); + static void PreparePullListWorker(const TProtobufRawInputSpec&, IPullListWorker*, THolder<IStream<google::protobuf::Message*>>); + static TConsumerType MakeConsumer(const TProtobufRawInputSpec&, TWorkerHolder<IPushStreamWorker>); +}; + +template <> +struct TOutputSpecTraits<TProtobufRawOutputSpec> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = true; + static const constexpr bool SupportPullListMode = true; + static const constexpr bool SupportPushStreamMode = true; + + using TOutputItemType = google::protobuf::Message*; + using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; + using TPullListReturnType = THolder<IStream<TOutputItemType>>; + + static const constexpr TOutputItemType StreamSentinel = nullptr; + + static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufRawOutputSpec&, TWorkerHolder<IPullStreamWorker>); + static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufRawOutputSpec&, TWorkerHolder<IPullListWorker>); + static void SetConsumerToWorker(const TProtobufRawOutputSpec&, IPushStreamWorker*, THolder<IConsumer<TOutputItemType>>); +}; + +template <> +struct TOutputSpecTraits<TProtobufRawMultiOutputSpec> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = true; + static const constexpr bool SupportPullListMode = true; + static const constexpr bool SupportPushStreamMode = true; + + using TOutputItemType = std::pair<ui32, google::protobuf::Message*>; + using TPullStreamReturnType = THolder<IStream<TOutputItemType>>; + using TPullListReturnType = THolder<IStream<TOutputItemType>>; + + static const constexpr TOutputItemType StreamSentinel = {0, nullptr}; + + static TPullStreamReturnType ConvertPullStreamWorkerToOutputType(const TProtobufRawMultiOutputSpec&, TWorkerHolder<IPullStreamWorker>); + static TPullListReturnType ConvertPullListWorkerToOutputType(const TProtobufRawMultiOutputSpec&, TWorkerHolder<IPullListWorker>); + static void SetConsumerToWorker(const TProtobufRawMultiOutputSpec&, IPushStreamWorker*, THolder<IConsumer<TOutputItemType>>); +}; +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/io_specs/protobuf_raw/ya.make b/yql/essentials/public/purecalc/io_specs/protobuf_raw/ya.make index db3fab7e7a5..7b917148e2e 100644 --- a/yql/essentials/public/purecalc/io_specs/protobuf_raw/ya.make +++ b/yql/essentials/public/purecalc/io_specs/protobuf_raw/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + PEERDIR( yql/essentials/public/purecalc/common yql/essentials/public/purecalc/helpers/protobuf diff --git a/yql/essentials/public/purecalc/no_llvm/purecalc.h b/yql/essentials/public/purecalc/no_llvm/purecalc.h index 9b281a7caa7..83bd8a7b842 100644 --- a/yql/essentials/public/purecalc/no_llvm/purecalc.h +++ b/yql/essentials/public/purecalc/no_llvm/purecalc.h @@ -1,4 +1,3 @@ #pragma once #include "common/interface.h" - diff --git a/yql/essentials/public/purecalc/no_llvm/ya.make b/yql/essentials/public/purecalc/no_llvm/ya.make index 1cff51e09eb..84a73440c5c 100644 --- a/yql/essentials/public/purecalc/no_llvm/ya.make +++ b/yql/essentials/public/purecalc/no_llvm/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + ADDINCL( yql/essentials/public/purecalc ) diff --git a/yql/essentials/public/purecalc/ut/empty_stream.h b/yql/essentials/public/purecalc/ut/empty_stream.h index 8d10e647aee..64c3954f8e7 100644 --- a/yql/essentials/public/purecalc/ut/empty_stream.h +++ b/yql/essentials/public/purecalc/ut/empty_stream.h @@ -3,18 +3,18 @@ #include <yql/essentials/public/purecalc/purecalc.h> namespace NYql { - namespace NPureCalc { - template <typename T> - class TEmptyStreamImpl: public IStream<T> { - public: - T Fetch() override { - return nullptr; - } - }; - - template <typename T> - THolder<IStream<T>> EmptyStream() { - return MakeHolder<TEmptyStreamImpl<T>>(); - } +namespace NPureCalc { +template <typename T> +class TEmptyStreamImpl: public IStream<T> { +public: + T Fetch() override { + return nullptr; } +}; + +template <typename T> +THolder<IStream<T>> EmptyStream() { + return MakeHolder<TEmptyStreamImpl<T>>(); } +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/ut/fake_spec.cpp b/yql/essentials/public/purecalc/ut/fake_spec.cpp index 8acdbcfb347..12bbeb51340 100644 --- a/yql/essentials/public/purecalc/ut/fake_spec.cpp +++ b/yql/essentials/public/purecalc/ut/fake_spec.cpp @@ -1,82 +1,80 @@ #include "fake_spec.h" namespace NYql { - namespace NPureCalc { - NYT::TNode MakeFakeSchema(bool pg) { - auto itemType = NYT::TNode::CreateList(); - itemType.Add(pg ? "PgType" : "DataType"); - itemType.Add(pg ? "int4" : "Int32"); +namespace NPureCalc { +NYT::TNode MakeFakeSchema(bool pg) { + auto itemType = NYT::TNode::CreateList(); + itemType.Add(pg ? "PgType" : "DataType"); + itemType.Add(pg ? "int4" : "Int32"); - auto itemNode = NYT::TNode::CreateList(); - itemNode.Add("Name"); - itemNode.Add(std::move(itemType)); + auto itemNode = NYT::TNode::CreateList(); + itemNode.Add("Name"); + itemNode.Add(std::move(itemType)); - auto items = NYT::TNode::CreateList(); - items.Add(std::move(itemNode)); + auto items = NYT::TNode::CreateList(); + items.Add(std::move(itemNode)); - auto schema = NYT::TNode::CreateList(); - schema.Add("StructType"); - schema.Add(std::move(items)); + auto schema = NYT::TNode::CreateList(); + schema.Add("StructType"); + schema.Add(std::move(items)); - return schema; - } + return schema; +} - TFakeInputSpec FakeIS(ui32 inputsNumber, bool pg) { - auto spec = TFakeInputSpec(); - spec.Schemas = TVector<NYT::TNode>(inputsNumber, MakeFakeSchema(pg)); - return spec; - } +TFakeInputSpec FakeIS(ui32 inputsNumber, bool pg) { + auto spec = TFakeInputSpec(); + spec.Schemas = TVector<NYT::TNode>(inputsNumber, MakeFakeSchema(pg)); + return spec; +} - TFakeOutputSpec FakeOS(bool pg) { - auto spec = TFakeOutputSpec(); - spec.Schema = MakeFakeSchema(pg); - return spec; - } +TFakeOutputSpec FakeOS(bool pg) { + auto spec = TFakeOutputSpec(); + spec.Schema = MakeFakeSchema(pg); + return spec; +} - NYT::TNode CreateTypeNode(const TString& fieldType) { - return NYT::TNode::CreateList() - .Add("DataType") - .Add(fieldType); - } +NYT::TNode CreateTypeNode(const TString& fieldType) { + return NYT::TNode::CreateList() + .Add("DataType") + .Add(fieldType); +} - NYT::TNode CreateOptionalTypeNode(const TString& fieldType) { - return NYT::TNode::CreateList() - .Add("OptionalType") - .Add(CreateTypeNode(fieldType)); - } +NYT::TNode CreateOptionalTypeNode(const TString& fieldType) { + return NYT::TNode::CreateList() + .Add("OptionalType") + .Add(CreateTypeNode(fieldType)); +} - void AddField(NYT::TNode& node, const TString& fieldName, const TString& fieldType) { - node.Add( - NYT::TNode::CreateList() - .Add(fieldName) - .Add(CreateOptionalTypeNode(fieldType)) - ); - } +void AddField(NYT::TNode& node, const TString& fieldName, const TString& fieldType) { + node.Add( + NYT::TNode::CreateList() + .Add(fieldName) + .Add(CreateOptionalTypeNode(fieldType))); +} - NYT::TNode MakeFakeStructSchema() { - auto structMembers = NYT::TNode::CreateList(); - AddField(structMembers, "Id", "Uint32"); - AddField(structMembers, "Name", "Utf8"); - AddField(structMembers, "Body", "String"); +NYT::TNode MakeFakeStructSchema() { + auto structMembers = NYT::TNode::CreateList(); + AddField(structMembers, "Id", "Uint32"); + AddField(structMembers, "Name", "Utf8"); + AddField(structMembers, "Body", "String"); - auto rootMembers = NYT::TNode::CreateList(); - rootMembers.Add( - NYT::TNode::CreateList() - .Add("_r") - .Add(NYT::TNode::CreateList() - .Add("StructType") - .Add(std::move(structMembers))) - ); + auto rootMembers = NYT::TNode::CreateList(); + rootMembers.Add( + NYT::TNode::CreateList() + .Add("_r") + .Add(NYT::TNode::CreateList() + .Add("StructType") + .Add(std::move(structMembers)))); - return NYT::TNode::CreateList() - .Add("StructType") - .Add(std::move(rootMembers)); - } + return NYT::TNode::CreateList() + .Add("StructType") + .Add(std::move(rootMembers)); +} - TFakeOutputSpec FakeStructOS() { - auto spec = TFakeOutputSpec(); - spec.Schema = MakeFakeStructSchema(); - return spec; - } - } +TFakeOutputSpec FakeStructOS() { + auto spec = TFakeOutputSpec(); + spec.Schema = MakeFakeStructSchema(); + return spec; } +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/ut/fake_spec.h b/yql/essentials/public/purecalc/ut/fake_spec.h index 5f2ea310821..9e54c6a4139 100644 --- a/yql/essentials/public/purecalc/ut/fake_spec.h +++ b/yql/essentials/public/purecalc/ut/fake_spec.h @@ -3,53 +3,53 @@ #include <yql/essentials/public/purecalc/purecalc.h> namespace NYql { - namespace NPureCalc { - class TFakeInputSpec: public TInputSpecBase { - public: - TVector<NYT::TNode> Schemas = {NYT::TNode::CreateList()}; - - public: - const TVector<NYT::TNode>& GetSchemas() const override { - return Schemas; - } - }; - - class TFakeOutputSpec: public TOutputSpecBase { - public: - NYT::TNode Schema = NYT::TNode::CreateList(); - - public: - const NYT::TNode& GetSchema() const override { - return Schema; - } - }; - - template <> - struct TInputSpecTraits<TFakeInputSpec> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = false; - static const constexpr bool SupportPullListMode = false; - static const constexpr bool SupportPushStreamMode = false; - - using TConsumerType = void; - }; - - template <> - struct TOutputSpecTraits<TFakeOutputSpec> { - static const constexpr bool IsPartial = false; - - static const constexpr bool SupportPullStreamMode = false; - static const constexpr bool SupportPullListMode = false; - static const constexpr bool SupportPushStreamMode = false; - - using TPullStreamReturnType = void; - using TPullListReturnType = void; - }; - - NYT::TNode MakeFakeSchema(bool pg = false); - TFakeInputSpec FakeIS(ui32 inputsNumber = 1, bool pg = false); - TFakeOutputSpec FakeOS(bool pg = false); - TFakeOutputSpec FakeStructOS(); +namespace NPureCalc { +class TFakeInputSpec: public TInputSpecBase { +public: + TVector<NYT::TNode> Schemas = {NYT::TNode::CreateList()}; + +public: + const TVector<NYT::TNode>& GetSchemas() const override { + return Schemas; } -} +}; + +class TFakeOutputSpec: public TOutputSpecBase { +public: + NYT::TNode Schema = NYT::TNode::CreateList(); + +public: + const NYT::TNode& GetSchema() const override { + return Schema; + } +}; + +template <> +struct TInputSpecTraits<TFakeInputSpec> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = false; + static const constexpr bool SupportPullListMode = false; + static const constexpr bool SupportPushStreamMode = false; + + using TConsumerType = void; +}; + +template <> +struct TOutputSpecTraits<TFakeOutputSpec> { + static const constexpr bool IsPartial = false; + + static const constexpr bool SupportPullStreamMode = false; + static const constexpr bool SupportPullListMode = false; + static const constexpr bool SupportPushStreamMode = false; + + using TPullStreamReturnType = void; + using TPullListReturnType = void; +}; + +NYT::TNode MakeFakeSchema(bool pg = false); +TFakeInputSpec FakeIS(ui32 inputsNumber = 1, bool pg = false); +TFakeOutputSpec FakeOS(bool pg = false); +TFakeOutputSpec FakeStructOS(); +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/ut/lib/helpers.cpp b/yql/essentials/public/purecalc/ut/lib/helpers.cpp index cef9a995235..59383454e67 100644 --- a/yql/essentials/public/purecalc/ut/lib/helpers.cpp +++ b/yql/essentials/public/purecalc/ut/lib/helpers.cpp @@ -7,49 +7,45 @@ #include <util/string/ascii.h> #include <util/generic/hash_set.h> - namespace NYql { - namespace NPureCalc { - namespace NPrivate { - NYT::TNode GetSchema( - const TVector<TString>& fields, - const TVector<TString>& optionalFields - ) { - THashSet<TString> optionalFilter { - optionalFields.begin(), optionalFields.end() - }; - - NYT::TNode members {NYT::TNode::CreateList()}; - - auto addField = [&] (const TString& name, const TString& type) { - auto typeNode = NYT::TNode::CreateList() - .Add("DataType") - .Add(type); - - if (optionalFilter.contains(name)) { - typeNode = NYT::TNode::CreateList() - .Add("OptionalType") - .Add(typeNode); - } - - members.Add(NYT::TNode::CreateList() - .Add(name) - .Add(typeNode) - ); - }; - - for (const auto& field: fields) { - TString type {field}; - type[0] = AsciiToUpper(type[0]); - addField(field, type); - } - - NYT::TNode schema = NYT::TNode::CreateList() - .Add("StructType") - .Add(members); - - return schema; - } +namespace NPureCalc { +namespace NPrivate { +NYT::TNode GetSchema( + const TVector<TString>& fields, + const TVector<TString>& optionalFields) { + THashSet<TString> optionalFilter{ + optionalFields.begin(), optionalFields.end()}; + + NYT::TNode members{NYT::TNode::CreateList()}; + + auto addField = [&](const TString& name, const TString& type) { + auto typeNode = NYT::TNode::CreateList() + .Add("DataType") + .Add(type); + + if (optionalFilter.contains(name)) { + typeNode = NYT::TNode::CreateList() + .Add("OptionalType") + .Add(typeNode); } + + members.Add(NYT::TNode::CreateList() + .Add(name) + .Add(typeNode)); + }; + + for (const auto& field : fields) { + TString type{field}; + type[0] = AsciiToUpper(type[0]); + addField(field, type); } + + NYT::TNode schema = NYT::TNode::CreateList() + .Add("StructType") + .Add(members); + + return schema; } +} // namespace NPrivate +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/ut/lib/helpers.h b/yql/essentials/public/purecalc/ut/lib/helpers.h index 53a22661ec3..b6fd110ae46 100644 --- a/yql/essentials/public/purecalc/ut/lib/helpers.h +++ b/yql/essentials/public/purecalc/ut/lib/helpers.h @@ -5,14 +5,12 @@ #include <util/generic/vector.h> #include <util/stream/str.h> - namespace NYql { - namespace NPureCalc { - namespace NPrivate { - NYT::TNode GetSchema( - const TVector<TString>& fields, - const TVector<TString>& optionalFields = {} - ); - } - } -} +namespace NPureCalc { +namespace NPrivate { +NYT::TNode GetSchema( + const TVector<TString>& fields, + const TVector<TString>& optionalFields = {}); +} // namespace NPrivate +} // namespace NPureCalc +} // namespace NYql diff --git a/yql/essentials/public/purecalc/ut/lib/ya.make b/yql/essentials/public/purecalc/ut/lib/ya.make index 36134a2940b..bff80e2cb97 100644 --- a/yql/essentials/public/purecalc/ut/lib/ya.make +++ b/yql/essentials/public/purecalc/ut/lib/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + PEERDIR( contrib/libs/apache/arrow library/cpp/yson diff --git a/yql/essentials/public/purecalc/ut/test_eval.cpp b/yql/essentials/public/purecalc/ut/test_eval.cpp index 9ca2579301b..0c2cdc10808 100644 --- a/yql/essentials/public/purecalc/ut/test_eval.cpp +++ b/yql/essentials/public/purecalc/ut/test_eval.cpp @@ -6,72 +6,69 @@ #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TestEval) { - Y_UNIT_TEST(TestEvalExpr) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestEvalExpr) { + using namespace NYql::NPureCalc; - auto options = TProgramFactoryOptions(); - auto factory = MakeProgramFactory(options); + auto options = TProgramFactoryOptions(); + auto factory = MakeProgramFactory(options); + auto program = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + "SELECT Unwrap(cast(EvaluateExpr('foo' || 'bar') as Utf8)) AS X", + ETranslationMode::SQL); + + auto stream = program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); + + NPureCalcProto::TStringMessage* message; + + UNIT_ASSERT(message = stream->Fetch()); + UNIT_ASSERT_EQUAL(message->GetX(), "foobar"); + UNIT_ASSERT(!stream->Fetch()); +} + +Y_UNIT_TEST(TestSelfType) { + using namespace NYql::NPureCalc; + + auto options = TProgramFactoryOptions(); + auto factory = MakeProgramFactory(options); + + try { auto program = factory->MakePullListProgram( TProtobufInputSpec<NPureCalcProto::TStringMessage>(), TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - "SELECT Unwrap(cast(EvaluateExpr('foo' || 'bar') as Utf8)) AS X", - ETranslationMode::SQL - ); + "$input = PROCESS Input;select unwrap(cast(FormatType(EvaluateType(TypeHandle(TypeOf($input)))) AS Utf8)) AS X", + ETranslationMode::SQL); auto stream = program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); NPureCalcProto::TStringMessage* message; UNIT_ASSERT(message = stream->Fetch()); - UNIT_ASSERT_EQUAL(message->GetX(), "foobar"); + UNIT_ASSERT_VALUES_EQUAL(message->GetX(), "List<Struct<'X':Utf8>>"); UNIT_ASSERT(!stream->Fetch()); + } catch (const TCompileError& e) { + UNIT_FAIL(e.GetIssues()); } +} - Y_UNIT_TEST(TestSelfType) { - using namespace NYql::NPureCalc; - - auto options = TProgramFactoryOptions(); - auto factory = MakeProgramFactory(options); - - try { - auto program = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - "$input = PROCESS Input;select unwrap(cast(FormatType(EvaluateType(TypeHandle(TypeOf($input)))) AS Utf8)) AS X", - ETranslationMode::SQL - ); - - auto stream = program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); +Y_UNIT_TEST(CantUseSelfInsideEvaluation) { + using namespace NYql::NPureCalc; - NPureCalcProto::TStringMessage* message; + auto options = TProgramFactoryOptions(); + auto factory = MakeProgramFactory(options); - UNIT_ASSERT(message = stream->Fetch()); - UNIT_ASSERT_VALUES_EQUAL(message->GetX(), "List<Struct<'X':Utf8>>"); - UNIT_ASSERT(!stream->Fetch()); - } catch (const TCompileError& e) { - UNIT_FAIL(e.GetIssues()); - } - } + try { + auto program = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + "$x = SELECT count(*) FROM Input;select unwrap(cast(EvaluateExpr($x) AS Utf8)) AS X", + ETranslationMode::SQL); - Y_UNIT_TEST(CantUseSelfInsideEvaluation) { - using namespace NYql::NPureCalc; - - auto options = TProgramFactoryOptions(); - auto factory = MakeProgramFactory(options); - - try { - auto program = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - "$x = SELECT count(*) FROM Input;select unwrap(cast(EvaluateExpr($x) AS Utf8)) AS X", - ETranslationMode::SQL - ); - - program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); - UNIT_FAIL("Exception is expected"); - } catch (const TCompileError& e) { - UNIT_ASSERT_C(TString(e.GetIssues()).Contains("Inputs aren't available during evaluation"), e.GetIssues()); - } + program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); + UNIT_FAIL("Exception is expected"); + } catch (const TCompileError& e) { + UNIT_ASSERT_C(TString(e.GetIssues()).Contains("Inputs aren't available during evaluation"), e.GetIssues()); } } +} // Y_UNIT_TEST_SUITE(TestEval) diff --git a/yql/essentials/public/purecalc/ut/test_fatal_err.cpp b/yql/essentials/public/purecalc/ut/test_fatal_err.cpp index bb1452b16e3..54cac5b5874 100644 --- a/yql/essentials/public/purecalc/ut/test_fatal_err.cpp +++ b/yql/essentials/public/purecalc/ut/test_fatal_err.cpp @@ -6,22 +6,21 @@ #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TestFatalError) { - Y_UNIT_TEST(TestFailType) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestFailType) { + using namespace NYql::NPureCalc; - auto options = TProgramFactoryOptions(); - auto factory = MakeProgramFactory(options); + auto options = TProgramFactoryOptions(); + auto factory = MakeProgramFactory(options); - try { - factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - "pragma warning(\"disable\",\"4510\");select unwrap(cast(Yql::FailMe(AsAtom('type')) as Utf8)) as X;", - ETranslationMode::SQL - ); - UNIT_FAIL("Exception is expected"); - } catch (const TCompileError& e) { - UNIT_ASSERT_C(e.GetIssues().Contains("abnormal"), e.GetIssues()); - } + try { + factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + "pragma warning(\"disable\",\"4510\");select unwrap(cast(Yql::FailMe(AsAtom('type')) as Utf8)) as X;", + ETranslationMode::SQL); + UNIT_FAIL("Exception is expected"); + } catch (const TCompileError& e) { + UNIT_ASSERT_C(e.GetIssues().Contains("abnormal"), e.GetIssues()); } } +} // Y_UNIT_TEST_SUITE(TestFatalError) diff --git a/yql/essentials/public/purecalc/ut/test_langver.cpp b/yql/essentials/public/purecalc/ut/test_langver.cpp index 1b3cfa2ff46..3fe55b862f6 100644 --- a/yql/essentials/public/purecalc/ut/test_langver.cpp +++ b/yql/essentials/public/purecalc/ut/test_langver.cpp @@ -8,22 +8,22 @@ #include "fake_spec.h" Y_UNIT_TEST_SUITE(TestLangVer) { - Y_UNIT_TEST(TooHighLangVer) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TooHighLangVer) { + using namespace NYql::NPureCalc; - auto options = TProgramFactoryOptions(); - options.SetLanguageVersion(NYql::GetMaxLangVersion()); - auto factory = MakeProgramFactory(options); + auto options = TProgramFactoryOptions(); + options.SetLanguageVersion(NYql::GetMaxLangVersion()); + auto factory = MakeProgramFactory(options); - try { - auto sql = TString(R"( + try { + auto sql = TString(R"( SELECT * FROM Input; )"); - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - UNIT_FAIL("Exception is expected"); - } catch (const TCompileError& e) { - UNIT_ASSERT_C(e.GetIssues().Contains("version"), e.GetIssues()); - } + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + UNIT_FAIL("Exception is expected"); + } catch (const TCompileError& e) { + UNIT_ASSERT_C(e.GetIssues().Contains("version"), e.GetIssues()); } } +} // Y_UNIT_TEST_SUITE(TestLangVer) diff --git a/yql/essentials/public/purecalc/ut/test_mixed_allocators.cpp b/yql/essentials/public/purecalc/ut/test_mixed_allocators.cpp index 2932538f78f..3dad746a045 100644 --- a/yql/essentials/public/purecalc/ut/test_mixed_allocators.cpp +++ b/yql/essentials/public/purecalc/ut/test_mixed_allocators.cpp @@ -9,83 +9,81 @@ using namespace NYql::NPureCalc; namespace { - class TStatelessInputSpec : public TInputSpecBase { - public: - TStatelessInputSpec() - : Schemas_({NYT::TNode::CreateList() - .Add("StructType") - .Add(NYT::TNode::CreateList() - .Add(NYT::TNode::CreateList() - .Add("InputValue") +class TStatelessInputSpec: public TInputSpecBase { +public: + TStatelessInputSpec() + : Schemas_({NYT::TNode::CreateList() + .Add("StructType") .Add(NYT::TNode::CreateList() - .Add("DataType") - .Add("Utf8") - ) - ) - ) - }) - {}; - - const TVector<NYT::TNode>& GetSchemas() const override { - return Schemas_; - } + .Add(NYT::TNode::CreateList() + .Add("InputValue") + .Add(NYT::TNode::CreateList() + .Add("DataType") + .Add("Utf8"))))}) + {}; + + const TVector<NYT::TNode>& GetSchemas() const override { + return Schemas_; + } - private: - const TVector<NYT::TNode> Schemas_; - }; +private: + const TVector<NYT::TNode> Schemas_; +}; - class TStatelessInputConsumer : public IConsumer<const NYql::NUdf::TUnboxedValue&> { - public: - TStatelessInputConsumer(TWorkerHolder<IPushStreamWorker> worker) - : Worker_(std::move(worker)) - {} +class TStatelessInputConsumer: public IConsumer<const NYql::NUdf::TUnboxedValue&> { +public: + TStatelessInputConsumer(TWorkerHolder<IPushStreamWorker> worker) + : Worker_(std::move(worker)) + { + } - void OnObject(const NYql::NUdf::TUnboxedValue& value) override { - with_lock (Worker_->GetScopedAlloc()) { - NYql::NUdf::TUnboxedValue* items = nullptr; - NYql::NUdf::TUnboxedValue result = Worker_->GetGraph().GetHolderFactory().CreateDirectArrayHolder(1, items); + void OnObject(const NYql::NUdf::TUnboxedValue& value) override { + with_lock (Worker_->GetScopedAlloc()) { + NYql::NUdf::TUnboxedValue* items = nullptr; + NYql::NUdf::TUnboxedValue result = Worker_->GetGraph().GetHolderFactory().CreateDirectArrayHolder(1, items); - items[0] = value; + items[0] = value; - Worker_->Push(std::move(result)); + Worker_->Push(std::move(result)); - // Clear graph after each object because - // values allocated on another allocator and should be released - Worker_->Invalidate(); - } + // Clear graph after each object because + // values allocated on another allocator and should be released + Worker_->Invalidate(); } + } - void OnFinish() override { - with_lock(Worker_->GetScopedAlloc()) { - Worker_->OnFinish(); - } + void OnFinish() override { + with_lock (Worker_->GetScopedAlloc()) { + Worker_->OnFinish(); } + } - private: - TWorkerHolder<IPushStreamWorker> Worker_; - }; +private: + TWorkerHolder<IPushStreamWorker> Worker_; +}; - class TStatelessConsumer : public IConsumer<NPureCalcProto::TStringMessage*> { - const TString ExpectedData_; - const ui64 ExpectedRows_; - ui64 RowId_ = 0; +class TStatelessConsumer: public IConsumer<NPureCalcProto::TStringMessage*> { + const TString ExpectedData_; + const ui64 ExpectedRows_; + ui64 RowId_ = 0; - public: - TStatelessConsumer(const TString& expectedData, ui64 expectedRows) - : ExpectedData_(expectedData) - , ExpectedRows_(expectedRows) - {} +public: + TStatelessConsumer(const TString& expectedData, ui64 expectedRows) + : ExpectedData_(expectedData) + , ExpectedRows_(expectedRows) + { + } - void OnObject(NPureCalcProto::TStringMessage* message) override { - UNIT_ASSERT_VALUES_EQUAL_C(ExpectedData_, message->GetX(), RowId_); - RowId_++; - } + void OnObject(NPureCalcProto::TStringMessage* message) override { + UNIT_ASSERT_VALUES_EQUAL_C(ExpectedData_, message->GetX(), RowId_); + RowId_++; + } - void OnFinish() override { - UNIT_ASSERT_VALUES_EQUAL(ExpectedRows_, RowId_); - } - }; -} + void OnFinish() override { + UNIT_ASSERT_VALUES_EQUAL(ExpectedRows_, RowId_); + } +}; +} // namespace template <> struct TInputSpecTraits<TStatelessInputSpec> { @@ -100,40 +98,39 @@ struct TInputSpecTraits<TStatelessInputSpec> { }; Y_UNIT_TEST_SUITE(TestMixedAllocators) { - Y_UNIT_TEST(TestPushStream) { - const auto targetString = "large string >= 14 bytes"; - const auto factory = MakeProgramFactory(); - const auto sql = TStringBuilder() << "SELECT InputValue AS X FROM Input WHERE InputValue = \"" << targetString << "\";"; - - const auto program = factory->MakePushStreamProgram( - TStatelessInputSpec(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql - ); - - const ui64 numberRows = 5; - const auto inputConsumer = program->Apply(MakeHolder<TStatelessConsumer>(targetString, numberRows)); - NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), true, false); - - const auto pushString = [&](TString inputValue) { - NYql::NUdf::TUnboxedValue stringValue; - with_lock(alloc) { - stringValue = NKikimr::NMiniKQL::MakeString(inputValue); - alloc.Ref().LockObject(stringValue); - } - - inputConsumer->OnObject(stringValue); - - with_lock(alloc) { - alloc.Ref().UnlockObject(stringValue); - stringValue.Clear(); - } - }; - - for (ui64 i = 0; i < numberRows; ++i) { - pushString(targetString); - pushString("another large string >= 14 bytes"); +Y_UNIT_TEST(TestPushStream) { + const auto targetString = "large string >= 14 bytes"; + const auto factory = MakeProgramFactory(); + const auto sql = TStringBuilder() << "SELECT InputValue AS X FROM Input WHERE InputValue = \"" << targetString << "\";"; + + const auto program = factory->MakePushStreamProgram( + TStatelessInputSpec(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql); + + const ui64 numberRows = 5; + const auto inputConsumer = program->Apply(MakeHolder<TStatelessConsumer>(targetString, numberRows)); + NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), true, false); + + const auto pushString = [&](TString inputValue) { + NYql::NUdf::TUnboxedValue stringValue; + with_lock (alloc) { + stringValue = NKikimr::NMiniKQL::MakeString(inputValue); + alloc.Ref().LockObject(stringValue); } - inputConsumer->OnFinish(); + + inputConsumer->OnObject(stringValue); + + with_lock (alloc) { + alloc.Ref().UnlockObject(stringValue); + stringValue.Clear(); + } + }; + + for (ui64 i = 0; i < numberRows; ++i) { + pushString(targetString); + pushString("another large string >= 14 bytes"); } + inputConsumer->OnFinish(); } +} // Y_UNIT_TEST_SUITE(TestMixedAllocators) diff --git a/yql/essentials/public/purecalc/ut/test_pg.cpp b/yql/essentials/public/purecalc/ut/test_pg.cpp index 3d26cfbd1be..c45e2ed569d 100644 --- a/yql/essentials/public/purecalc/ut/test_pg.cpp +++ b/yql/essentials/public/purecalc/ut/test_pg.cpp @@ -7,65 +7,65 @@ #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TestPg) { - using namespace NYql::NPureCalc; +using namespace NYql::NPureCalc; - Y_UNIT_TEST(TestPgCompile) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestPgCompile) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( SELECT * FROM "Input"; )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(1,true), FakeOS(true), sql, ETranslationMode::PG); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }()); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram(FakeIS(1,true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "PullList mode"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "PullList mode"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePushStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "PullList mode"); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "PullList mode"); +} - Y_UNIT_TEST(TestSqlWrongTableName) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlWrongTableName) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( SELECT * FROM WrongTable; )"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullListProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "Failed to optimize"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullListProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "Failed to optimize"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "PullList mode"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "PullList mode"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePushStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "PullList mode"); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "PullList mode"); +} - Y_UNIT_TEST(TestInvalidSql) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestInvalidSql) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( Just some invalid SQL; )"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullListProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "failed to parse PG"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullListProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "failed to parse PG"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "PullList mode"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "PullList mode"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePushStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); - }(), TCompileError, "PullList mode"); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(FakeIS(1, true), FakeOS(true), sql, ETranslationMode::PG); + }(), TCompileError, "PullList mode"); } +} // Y_UNIT_TEST_SUITE(TestPg) diff --git a/yql/essentials/public/purecalc/ut/test_pool.cpp b/yql/essentials/public/purecalc/ut/test_pool.cpp index b3de36cbf5f..90ae69b8bc2 100644 --- a/yql/essentials/public/purecalc/ut/test_pool.cpp +++ b/yql/essentials/public/purecalc/ut/test_pool.cpp @@ -10,175 +10,172 @@ using namespace NYql::NPureCalc; namespace { - class TStringMessageStreamImpl: public IStream<NPureCalcProto::TStringMessage*> { - private: - ui32 I_ = 0; - NPureCalcProto::TStringMessage Message_{}; - - public: - NPureCalcProto::TStringMessage* Fetch() override { - if (I_ >= 3) { - return nullptr; - } else { - Message_.SetX(ToString(I_)); - ++I_; - return &Message_; - } +class TStringMessageStreamImpl: public IStream<NPureCalcProto::TStringMessage*> { +private: + ui32 I_ = 0; + NPureCalcProto::TStringMessage Message_{}; + +public: + NPureCalcProto::TStringMessage* Fetch() override { + if (I_ >= 3) { + return nullptr; + } else { + Message_.SetX(ToString(I_)); + ++I_; + return &Message_; } - }; + } +}; - class TStringMessageConsumerImpl: public IConsumer<NPureCalcProto::TStringMessage*> { - private: - TVector<TString>* Buf_; +class TStringMessageConsumerImpl: public IConsumer<NPureCalcProto::TStringMessage*> { +private: + TVector<TString>* Buf_; - public: - TStringMessageConsumerImpl(TVector<TString>* buf) - : Buf_(buf) - { - } +public: + TStringMessageConsumerImpl(TVector<TString>* buf) + : Buf_(buf) + { + } - public: - void OnObject(NPureCalcProto::TStringMessage* t) override { - Buf_->push_back(t->GetX()); - } +public: + void OnObject(NPureCalcProto::TStringMessage* t) override { + Buf_->push_back(t->GetX()); + } - void OnFinish() override { - } - }; + void OnFinish() override { + } +}; -} +} // namespace Y_UNIT_TEST_SUITE(TestWorkerPool) { - static TString sql = "SELECT 'abc'u || X AS X FROM Input"; +static TString sql = "SELECT 'abc'u || X AS X FROM Input"; - static TVector<TString> expected{"abc0", "abc1", "abc2"}; +static TVector<TString> expected{"abc0", "abc1", "abc2"}; - void TestPullStreamImpl(bool useWorkerPool) { - auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseWorkerPool(useWorkerPool)); +void TestPullStreamImpl(bool useWorkerPool) { + auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseWorkerPool(useWorkerPool)); - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql, - ETranslationMode::SQL - ); + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql, + ETranslationMode::SQL); - auto check = [](IStream<NPureCalcProto::TStringMessage*>* output) { - TVector<TString> actual; - while (auto *x = output->Fetch()) { - actual.push_back(x->GetX()); - } + auto check = [](IStream<NPureCalcProto::TStringMessage*>* output) { + TVector<TString> actual; + while (auto* x = output->Fetch()) { + actual.push_back(x->GetX()); + } - UNIT_ASSERT_VALUES_EQUAL(expected, actual); - }; + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + }; - // Sequential use - for (size_t i = 0; i < 2; ++i) { - auto output = program->Apply(MakeHolder<TStringMessageStreamImpl>()); - check(output.Get()); - } - // Parallel use - { - auto output1 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); - auto output2 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); - check(output1.Get()); - check(output2.Get()); - } + // Sequential use + for (size_t i = 0; i < 2; ++i) { + auto output = program->Apply(MakeHolder<TStringMessageStreamImpl>()); + check(output.Get()); } - - Y_UNIT_TEST(TestPullStreamUseWorkerPool) { - TestPullStreamImpl(true); + // Parallel use + { + auto output1 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); + auto output2 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); + check(output1.Get()); + check(output2.Get()); } +} - Y_UNIT_TEST(TestPullStreamNoWorkerPool) { - TestPullStreamImpl(false); - } +Y_UNIT_TEST(TestPullStreamUseWorkerPool) { + TestPullStreamImpl(true); +} - void TestPullListImpl(bool useWorkerPool) { - auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseWorkerPool(useWorkerPool)); - - auto program = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql, - ETranslationMode::SQL - ); - - auto check = [](IStream<NPureCalcProto::TStringMessage*>* output) { - TVector<TString> actual; - while (auto *x = output->Fetch()) { - actual.push_back(x->GetX()); - } - - UNIT_ASSERT_VALUES_EQUAL(expected, actual); - }; - - // Sequential use - for (size_t i = 0; i < 2; ++i) { - auto output = program->Apply(MakeHolder<TStringMessageStreamImpl>()); - check(output.Get()); - } - // Parallel use - { - auto output1 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); - auto output2 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); - check(output1.Get()); - check(output2.Get()); +Y_UNIT_TEST(TestPullStreamNoWorkerPool) { + TestPullStreamImpl(false); +} + +void TestPullListImpl(bool useWorkerPool) { + auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseWorkerPool(useWorkerPool)); + + auto program = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql, + ETranslationMode::SQL); + + auto check = [](IStream<NPureCalcProto::TStringMessage*>* output) { + TVector<TString> actual; + while (auto* x = output->Fetch()) { + actual.push_back(x->GetX()); } - } - Y_UNIT_TEST(TestPullListUseWorkerPool) { - TestPullListImpl(true); - } + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + }; - Y_UNIT_TEST(TestPullListNoWorkerPool) { - TestPullListImpl(false); + // Sequential use + for (size_t i = 0; i < 2; ++i) { + auto output = program->Apply(MakeHolder<TStringMessageStreamImpl>()); + check(output.Get()); } + // Parallel use + { + auto output1 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); + auto output2 = program->Apply(MakeHolder<TStringMessageStreamImpl>()); + check(output1.Get()); + check(output2.Get()); + } +} - void TestPushStreamImpl(bool useWorkerPool) { - auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseWorkerPool(useWorkerPool)); - - auto program = factory->MakePushStreamProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - sql, - ETranslationMode::SQL - ); - - auto check = [](IConsumer<NPureCalcProto::TStringMessage*>* input, const TVector<TString>& result) { - NPureCalcProto::TStringMessage message; - for (auto s: {"0", "1", "2"}) { - message.SetX(s); - input->OnObject(&message); - } - input->OnFinish(); - - UNIT_ASSERT_VALUES_EQUAL(expected, result); - }; - - // Sequential use - for (size_t i = 0; i < 2; ++i) { - TVector<TString> actual; - auto input = program->Apply(MakeHolder<TStringMessageConsumerImpl>(&actual)); - check(input.Get(), actual); - } +Y_UNIT_TEST(TestPullListUseWorkerPool) { + TestPullListImpl(true); +} + +Y_UNIT_TEST(TestPullListNoWorkerPool) { + TestPullListImpl(false); +} + +void TestPushStreamImpl(bool useWorkerPool) { + auto factory = MakeProgramFactory(TProgramFactoryOptions().SetUseWorkerPool(useWorkerPool)); + + auto program = factory->MakePushStreamProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + sql, + ETranslationMode::SQL); - // Parallel use - { - TVector<TString> actual1; - auto input1 = program->Apply(MakeHolder<TStringMessageConsumerImpl>(&actual1)); - TVector<TString> actual2; - auto input2 = program->Apply(MakeHolder<TStringMessageConsumerImpl>(&actual2)); - check(input1.Get(), actual1); - check(input2.Get(), actual2); + auto check = [](IConsumer<NPureCalcProto::TStringMessage*>* input, const TVector<TString>& result) { + NPureCalcProto::TStringMessage message; + for (auto s : {"0", "1", "2"}) { + message.SetX(s); + input->OnObject(&message); } - } + input->OnFinish(); + + UNIT_ASSERT_VALUES_EQUAL(expected, result); + }; - Y_UNIT_TEST(TestPushStreamUseWorkerPool) { - TestPushStreamImpl(true); + // Sequential use + for (size_t i = 0; i < 2; ++i) { + TVector<TString> actual; + auto input = program->Apply(MakeHolder<TStringMessageConsumerImpl>(&actual)); + check(input.Get(), actual); } - Y_UNIT_TEST(TestPushStreamNoWorkerPool) { - TestPushStreamImpl(false); + // Parallel use + { + TVector<TString> actual1; + auto input1 = program->Apply(MakeHolder<TStringMessageConsumerImpl>(&actual1)); + TVector<TString> actual2; + auto input2 = program->Apply(MakeHolder<TStringMessageConsumerImpl>(&actual2)); + check(input1.Get(), actual1); + check(input2.Get(), actual2); } } + +Y_UNIT_TEST(TestPushStreamUseWorkerPool) { + TestPushStreamImpl(true); +} + +Y_UNIT_TEST(TestPushStreamNoWorkerPool) { + TestPushStreamImpl(false); +} +} // Y_UNIT_TEST_SUITE(TestWorkerPool) diff --git a/yql/essentials/public/purecalc/ut/test_sexpr.cpp b/yql/essentials/public/purecalc/ut/test_sexpr.cpp index 9c50dd1f291..5b46f685148 100644 --- a/yql/essentials/public/purecalc/ut/test_sexpr.cpp +++ b/yql/essentials/public/purecalc/ut/test_sexpr.cpp @@ -7,49 +7,49 @@ #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TestSExpr) { - Y_UNIT_TEST(TestSExprCompile) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestSExprCompile) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(); + auto factory = MakeProgramFactory(); - auto expr = TString(R"( + auto expr = TString(R"( ( (return (Self '0)) ) )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), expr, ETranslationMode::SExpr); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), expr, ETranslationMode::SExpr); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), expr, ETranslationMode::SExpr); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), expr, ETranslationMode::SExpr); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), expr, ETranslationMode::SExpr); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), expr, ETranslationMode::SExpr); + }()); +} - Y_UNIT_TEST(TestInvalidSExpr) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestInvalidSExpr) { + using namespace NYql::NPureCalc; - auto factory = MakeProgramFactory(); + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( Some totally invalid SExpr )"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SExpr); - }(), TCompileError, "failed to parse s-expression"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SExpr); + }(), TCompileError, "failed to parse s-expression"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SExpr); - }(), TCompileError, "failed to parse s-expression"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SExpr); + }(), TCompileError, "failed to parse s-expression"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SExpr); - }(), TCompileError, "failed to parse s-expression"); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SExpr); + }(), TCompileError, "failed to parse s-expression"); } +} // Y_UNIT_TEST_SUITE(TestSExpr) diff --git a/yql/essentials/public/purecalc/ut/test_sql.cpp b/yql/essentials/public/purecalc/ut/test_sql.cpp index 1b6b69279a8..7b7917751a4 100644 --- a/yql/essentials/public/purecalc/ut/test_sql.cpp +++ b/yql/essentials/public/purecalc/ut/test_sql.cpp @@ -7,36 +7,36 @@ #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TestSql) { - using namespace NYql::NPureCalc; +using namespace NYql::NPureCalc; - Y_UNIT_TEST(TestSqlCompile) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlCompile) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( SELECT * FROM Input; )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - auto program = factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - auto expectedIssues = TString(R"(<main>: Warning: Type annotation, code: 1030 + auto program = factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + auto expectedIssues = TString(R"(<main>: Warning: Type annotation, code: 1030 generated.sql:2:13: Warning: At function: PersistableRepr generated.sql:2:13: Warning: Persistable required. Atom, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable, code: 1104 )"); - UNIT_ASSERT_VALUES_EQUAL(expectedIssues, program->GetIssues().ToString()); - } + UNIT_ASSERT_VALUES_EQUAL(expectedIssues, program->GetIssues().ToString()); +} - Y_UNIT_TEST(TestStructCastMessage) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestStructCastMessage) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( $l = ($x) -> { return <| Id: $x.Name, Name: CAST($x.Name AS String), Body: Just('foo') |>; }; @@ -44,54 +44,54 @@ Y_UNIT_TEST_SUITE(TestSql) { SELECT $l(TableRow()) AS _r FROM Input )"); - try { - factory->MakePullListProgram(FakeIS(), FakeStructOS(), sql, ETranslationMode::SQL); - UNIT_ASSERT_C(false, "Unreachable"); - } catch (const NYql::NPureCalc::TCompileError& error) { - auto issue = error.GetIssues(); - UNIT_ASSERT_C(issue.Contains("Failed to convert 'Id': Int32 to Optional<Uint32>"), issue); - UNIT_ASSERT_C(!issue.Contains("Body"), issue); - } + try { + factory->MakePullListProgram(FakeIS(), FakeStructOS(), sql, ETranslationMode::SQL); + UNIT_ASSERT_C(false, "Unreachable"); + } catch (const NYql::NPureCalc::TCompileError& error) { + auto issue = error.GetIssues(); + UNIT_ASSERT_C(issue.Contains("Failed to convert 'Id': Int32 to Optional<Uint32>"), issue); + UNIT_ASSERT_C(!issue.Contains("Body"), issue); } +} - Y_UNIT_TEST(TestSqlCompileSingleUnnamedInput) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlCompileSingleUnnamedInput) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( SELECT * FROM TABLES() )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); +} - Y_UNIT_TEST(TestSqlCompileNamedMultiinputs) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlCompileNamedMultiinputs) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( SELECT * FROM Input0 UNION ALL SELECT * FROM Input1 )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(2), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(2), FakeOS(), sql, ETranslationMode::SQL); + }()); +} - Y_UNIT_TEST(TestSqlCompileUnnamedMultiinputs) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlCompileUnnamedMultiinputs) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( $t0, $t1, $t2 = PROCESS TABLES(); SELECT * FROM $t0 UNION ALL @@ -100,117 +100,117 @@ Y_UNIT_TEST_SUITE(TestSql) { SELECT * FROM $t2 )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(3), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(3), FakeOS(), sql, ETranslationMode::SQL); + }()); +} - Y_UNIT_TEST(TestSqlCompileWithWarning) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlCompileWithWarning) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( $x = 1; $y = 2; SELECT $x as Name FROM Input; )"); - auto expectedIssues = TString(R"(generated.sql:3:13: Warning: Symbol $y is not used, code: 4527 + auto expectedIssues = TString(R"(generated.sql:3:13: Warning: Symbol $y is not used, code: 4527 <main>: Warning: Type annotation, code: 1030 generated.sql:4:13: Warning: At function: PersistableRepr generated.sql:4:13: Warning: Persistable required. Atom, key, world, datasink, datasource, callable, resource, stream and lambda are not persistable, code: 1104 )"); - auto program = factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - UNIT_ASSERT_VALUES_EQUAL(expectedIssues, program->GetIssues().ToString()); - } + auto program = factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + UNIT_ASSERT_VALUES_EQUAL(expectedIssues, program->GetIssues().ToString()); +} - Y_UNIT_TEST(TestSqlWrongTableName) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestSqlWrongTableName) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( SELECT * FROM WrongTable; )"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }(), TCompileError, "Failed to optimize"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }(), TCompileError, "Failed to optimize"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }(), TCompileError, "Failed to optimize"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }(), TCompileError, "Failed to optimize"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }(), TCompileError, "Failed to optimize"); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }(), TCompileError, "Failed to optimize"); +} - Y_UNIT_TEST(TestAllocateLargeStringOnEvaluate) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestAllocateLargeStringOnEvaluate) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( $data = Length(EvaluateExpr("long string" || " very loooong string")); SELECT $data as Name FROM Input; )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); +} - Y_UNIT_TEST(TestInvalidSql) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestInvalidSql) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( Just some invalid SQL; )"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }(), TCompileError, "failed to parse SQL"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }(), TCompileError, "failed to parse SQL"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }(), TCompileError, "failed to parse SQL"); + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }(), TCompileError, "failed to parse SQL"); - UNIT_ASSERT_EXCEPTION_CONTAINS([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }(), TCompileError, "failed to parse SQL"); - } + UNIT_ASSERT_EXCEPTION_CONTAINS([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }(), TCompileError, "failed to parse SQL"); +} - Y_UNIT_TEST(TestUseProcess) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestUseProcess) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( $processor = ($row) -> ($row); PROCESS Input using $processor(TableRow()); )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePushStreamProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); +} - Y_UNIT_TEST(TestUseCodegen) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestUseCodegen) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( $processor = ($row) -> { $lambda = EvaluateCode(LambdaCode(($row) -> ($row))); return $lambda($row); @@ -219,15 +219,15 @@ Y_UNIT_TEST_SUITE(TestSql) { PROCESS Input using $processor(TableRow()); )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); +} - Y_UNIT_TEST(TestUseDefineSubquery) { - auto factory = MakeProgramFactory(); +Y_UNIT_TEST(TestUseDefineSubquery) { + auto factory = MakeProgramFactory(); - auto sql = TString(R"( + auto sql = TString(R"( DEFINE SUBQUERY $source() AS PROCESS Input; END DEFINE; @@ -239,8 +239,8 @@ Y_UNIT_TEST_SUITE(TestSql) { PROCESS $handler($source); )"); - UNIT_ASSERT_NO_EXCEPTION([&](){ - factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); - }()); - } + UNIT_ASSERT_NO_EXCEPTION([&]() { + factory->MakePullListProgram(FakeIS(), FakeOS(), sql, ETranslationMode::SQL); + }()); } +} // Y_UNIT_TEST_SUITE(TestSql) diff --git a/yql/essentials/public/purecalc/ut/test_udf.cpp b/yql/essentials/public/purecalc/ut/test_udf.cpp index 732917739e7..702225c656e 100644 --- a/yql/essentials/public/purecalc/ut/test_udf.cpp +++ b/yql/essentials/public/purecalc/ut/test_udf.cpp @@ -7,18 +7,19 @@ #include <yql/essentials/public/udf/udf_type_builder.h> #include <library/cpp/testing/unittest/registar.h> -class TMyModule : public NKikimr::NUdf::IUdfModule { +class TMyModule: public NKikimr::NUdf::IUdfModule { public: - class TFunc : public NKikimr::NUdf::TBoxedValue { + class TFunc: public NKikimr::NUdf::TBoxedValue { public: TFunc(NKikimr::NUdf::TCounter counter, NKikimr::NUdf::TScopedProbe scopedProbe) : Counter_(counter) , ScopedProbe_(scopedProbe) - {} + { + } NKikimr::NUdf::TUnboxedValue Run(const NKikimr::NUdf::IValueBuilder* valueBuilder, const NKikimr::NUdf::TUnboxedValuePod* args) const override { Y_UNUSED(valueBuilder); - with_lock(ScopedProbe_) { + with_lock (ScopedProbe_) { Counter_.Inc(); return NKikimr::NUdf::TUnboxedValuePod(args[0].Get<i32>()); } @@ -45,9 +46,8 @@ public: if (name == NKikimr::NUdf::TStringRef::Of("Func")) { builder.SimpleSignature<i32(i32)>(); builder.Implementation(new TFunc( - builder.GetCounter("FuncCalls",true), - builder.GetScopedProbe("FuncTime") - )); + builder.GetCounter("FuncCalls", true), + builder.GetScopedProbe("FuncTime"))); } } @@ -55,12 +55,13 @@ public: } }; -class TMyCountersProvider : public NKikimr::NUdf::ICountersProvider, public NKikimr::NUdf::IScopedProbeHost { +class TMyCountersProvider: public NKikimr::NUdf::ICountersProvider, public NKikimr::NUdf::IScopedProbeHost { public: TMyCountersProvider(i64* calls, TString* log) : Calls_(calls) , Log_(log) - {} + { + } NKikimr::NUdf::TCounter GetCounter(const NKikimr::NUdf::TStringRef& module, const NKikimr::NUdf::TStringRef& name, bool deriv) override { UNIT_ASSERT_VALUES_EQUAL(module, "MyModule"); @@ -91,11 +92,11 @@ private: }; namespace NPureCalcProto { - class TUnparsed; - class TParsed; -} +class TUnparsed; +class TParsed; +} // namespace NPureCalcProto -class TDocInput : public NYql::NPureCalc::IStream<NPureCalcProto::TUnparsed*> { +class TDocInput: public NYql::NPureCalc::IStream<NPureCalcProto::TUnparsed*> { public: NPureCalcProto::TUnparsed* Fetch() override { if (Extracted) { @@ -113,83 +114,83 @@ public: }; Y_UNIT_TEST_SUITE(TestUdf) { - Y_UNIT_TEST(TestCounters) { - using namespace NYql::NPureCalc; - - auto factory = MakeProgramFactory(); - - i64 callCounter = 0; - TMyCountersProvider myCountersProvider(&callCounter, nullptr); - factory->AddUdfModule("MyModule", new TMyModule); - factory->SetCountersProvider(&myCountersProvider); - - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TUnparsed>(), - TProtobufOutputSpec<NPureCalcProto::TParsed>(), - "select MyModule::Func(1) as A, 2 as B, 3 as C from Input", - ETranslationMode::SQL); - - auto out = program->Apply(MakeHolder<TDocInput>()); - auto* message = out->Fetch(); - UNIT_ASSERT(message); - UNIT_ASSERT_VALUES_EQUAL(message->GetA(), 1); - UNIT_ASSERT_VALUES_EQUAL(message->GetB(), 2); - UNIT_ASSERT_VALUES_EQUAL(message->GetC(), 3); - UNIT_ASSERT_VALUES_EQUAL(callCounter, 1); - UNIT_ASSERT(!out->Fetch()); - } +Y_UNIT_TEST(TestCounters) { + using namespace NYql::NPureCalc; + + auto factory = MakeProgramFactory(); + + i64 callCounter = 0; + TMyCountersProvider myCountersProvider(&callCounter, nullptr); + factory->AddUdfModule("MyModule", new TMyModule); + factory->SetCountersProvider(&myCountersProvider); + + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TUnparsed>(), + TProtobufOutputSpec<NPureCalcProto::TParsed>(), + "select MyModule::Func(1) as A, 2 as B, 3 as C from Input", + ETranslationMode::SQL); + + auto out = program->Apply(MakeHolder<TDocInput>()); + auto* message = out->Fetch(); + UNIT_ASSERT(message); + UNIT_ASSERT_VALUES_EQUAL(message->GetA(), 1); + UNIT_ASSERT_VALUES_EQUAL(message->GetB(), 2); + UNIT_ASSERT_VALUES_EQUAL(message->GetC(), 3); + UNIT_ASSERT_VALUES_EQUAL(callCounter, 1); + UNIT_ASSERT(!out->Fetch()); +} - Y_UNIT_TEST(TestCountersFilteredColumns) { - using namespace NYql::NPureCalc; - - auto factory = MakeProgramFactory(); - - i64 callCounter = 0; - TMyCountersProvider myCountersProvider(&callCounter, nullptr); - factory->AddUdfModule("MyModule", new TMyModule); - factory->SetCountersProvider(&myCountersProvider); - - auto ospec = TProtobufOutputSpec<NPureCalcProto::TParsed>(); - ospec.SetOutputColumnsFilter(THashSet<TString>({"B", "C"})); - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TUnparsed>(), - ospec, - "select MyModule::Func(1) as A, 2 as B, 3 as C from Input", - ETranslationMode::SQL); - - auto out = program->Apply(MakeHolder<TDocInput>()); - auto* message = out->Fetch(); - UNIT_ASSERT(message); - UNIT_ASSERT_VALUES_EQUAL(message->GetA(), 0); - UNIT_ASSERT_VALUES_EQUAL(message->GetB(), 2); - UNIT_ASSERT_VALUES_EQUAL(message->GetC(), 3); - UNIT_ASSERT_VALUES_EQUAL(callCounter, 0); - UNIT_ASSERT(!out->Fetch()); - } +Y_UNIT_TEST(TestCountersFilteredColumns) { + using namespace NYql::NPureCalc; + + auto factory = MakeProgramFactory(); + + i64 callCounter = 0; + TMyCountersProvider myCountersProvider(&callCounter, nullptr); + factory->AddUdfModule("MyModule", new TMyModule); + factory->SetCountersProvider(&myCountersProvider); + + auto ospec = TProtobufOutputSpec<NPureCalcProto::TParsed>(); + ospec.SetOutputColumnsFilter(THashSet<TString>({"B", "C"})); + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TUnparsed>(), + ospec, + "select MyModule::Func(1) as A, 2 as B, 3 as C from Input", + ETranslationMode::SQL); + + auto out = program->Apply(MakeHolder<TDocInput>()); + auto* message = out->Fetch(); + UNIT_ASSERT(message); + UNIT_ASSERT_VALUES_EQUAL(message->GetA(), 0); + UNIT_ASSERT_VALUES_EQUAL(message->GetB(), 2); + UNIT_ASSERT_VALUES_EQUAL(message->GetC(), 3); + UNIT_ASSERT_VALUES_EQUAL(callCounter, 0); + UNIT_ASSERT(!out->Fetch()); +} - Y_UNIT_TEST(TestScopedProbes) { - using namespace NYql::NPureCalc; - - auto factory = MakeProgramFactory(); - - TString log; - TMyCountersProvider myCountersProvider(nullptr, &log); - factory->AddUdfModule("MyModule", new TMyModule); - factory->SetCountersProvider(&myCountersProvider); - - auto program = factory->MakePullStreamProgram( - TProtobufInputSpec<NPureCalcProto::TUnparsed>(), - TProtobufOutputSpec<NPureCalcProto::TParsed>(), - "select MyModule::Func(1) as A, 2 as B, 3 as C from Input", - ETranslationMode::SQL); - - auto out = program->Apply(MakeHolder<TDocInput>()); - auto* message = out->Fetch(); - UNIT_ASSERT(message); - UNIT_ASSERT_VALUES_EQUAL(message->GetA(), 1); - UNIT_ASSERT_VALUES_EQUAL(message->GetB(), 2); - UNIT_ASSERT_VALUES_EQUAL(message->GetC(), 3); - UNIT_ASSERT_VALUES_EQUAL(log, "Enter\nExit\n"); - UNIT_ASSERT(!out->Fetch()); - } +Y_UNIT_TEST(TestScopedProbes) { + using namespace NYql::NPureCalc; + + auto factory = MakeProgramFactory(); + + TString log; + TMyCountersProvider myCountersProvider(nullptr, &log); + factory->AddUdfModule("MyModule", new TMyModule); + factory->SetCountersProvider(&myCountersProvider); + + auto program = factory->MakePullStreamProgram( + TProtobufInputSpec<NPureCalcProto::TUnparsed>(), + TProtobufOutputSpec<NPureCalcProto::TParsed>(), + "select MyModule::Func(1) as A, 2 as B, 3 as C from Input", + ETranslationMode::SQL); + + auto out = program->Apply(MakeHolder<TDocInput>()); + auto* message = out->Fetch(); + UNIT_ASSERT(message); + UNIT_ASSERT_VALUES_EQUAL(message->GetA(), 1); + UNIT_ASSERT_VALUES_EQUAL(message->GetB(), 2); + UNIT_ASSERT_VALUES_EQUAL(message->GetC(), 3); + UNIT_ASSERT_VALUES_EQUAL(log, "Enter\nExit\n"); + UNIT_ASSERT(!out->Fetch()); } +} // Y_UNIT_TEST_SUITE(TestUdf) diff --git a/yql/essentials/public/purecalc/ut/test_user_data.cpp b/yql/essentials/public/purecalc/ut/test_user_data.cpp index b87940ab6b2..164a3b4de33 100644 --- a/yql/essentials/public/purecalc/ut/test_user_data.cpp +++ b/yql/essentials/public/purecalc/ut/test_user_data.cpp @@ -6,57 +6,55 @@ #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TestUserData) { - Y_UNIT_TEST(TestUserData) { - using namespace NYql::NPureCalc; +Y_UNIT_TEST(TestUserData) { + using namespace NYql::NPureCalc; + auto options = TProgramFactoryOptions() + .AddFile(NYql::NUserData::EDisposition::INLINE, "my_file.txt", "my content!"); + + auto factory = MakeProgramFactory(options); + + auto program = factory->MakePullListProgram( + TProtobufInputSpec<NPureCalcProto::TStringMessage>(), + TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), + "SELECT UNWRAP(CAST(FileContent(\"my_file.txt\") AS Utf8)) AS X", + ETranslationMode::SQL); + + auto stream = program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); + + NPureCalcProto::TStringMessage* message; + + UNIT_ASSERT(message = stream->Fetch()); + UNIT_ASSERT_EQUAL(message->GetX(), "my content!"); + UNIT_ASSERT(!stream->Fetch()); +} + +Y_UNIT_TEST(TestUserDataLibrary) { + using namespace NYql::NPureCalc; + + try { auto options = TProgramFactoryOptions() - .AddFile(NYql::NUserData::EDisposition::INLINE, "my_file.txt", "my content!"); + .AddLibrary(NYql::NUserData::EDisposition::INLINE, "a.sql", "$x = 1; EXPORT $x;") + .AddLibrary(NYql::NUserData::EDisposition::INLINE, "b.sql", "IMPORT a SYMBOLS $x; $y = CAST($x + 1 AS String); EXPORT $y;"); auto factory = MakeProgramFactory(options); auto program = factory->MakePullListProgram( TProtobufInputSpec<NPureCalcProto::TStringMessage>(), TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - "SELECT UNWRAP(CAST(FileContent(\"my_file.txt\") AS Utf8)) AS X", - ETranslationMode::SQL - ); + "IMPORT b SYMBOLS $y; SELECT CAST($y AS Utf8) ?? '' AS X;", + ETranslationMode::SQL); auto stream = program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); NPureCalcProto::TStringMessage* message; UNIT_ASSERT(message = stream->Fetch()); - UNIT_ASSERT_EQUAL(message->GetX(), "my content!"); + UNIT_ASSERT_EQUAL(message->GetX(), "2"); UNIT_ASSERT(!stream->Fetch()); - } - - Y_UNIT_TEST(TestUserDataLibrary) { - using namespace NYql::NPureCalc; - - try { - auto options = TProgramFactoryOptions() - .AddLibrary(NYql::NUserData::EDisposition::INLINE, "a.sql", "$x = 1; EXPORT $x;") - .AddLibrary(NYql::NUserData::EDisposition::INLINE, "b.sql", "IMPORT a SYMBOLS $x; $y = CAST($x + 1 AS String); EXPORT $y;"); - - auto factory = MakeProgramFactory(options); - - auto program = factory->MakePullListProgram( - TProtobufInputSpec<NPureCalcProto::TStringMessage>(), - TProtobufOutputSpec<NPureCalcProto::TStringMessage>(), - "IMPORT b SYMBOLS $y; SELECT CAST($y AS Utf8) ?? '' AS X;", - ETranslationMode::SQL - ); - - auto stream = program->Apply(EmptyStream<NPureCalcProto::TStringMessage*>()); - - NPureCalcProto::TStringMessage* message; - - UNIT_ASSERT(message = stream->Fetch()); - UNIT_ASSERT_EQUAL(message->GetX(), "2"); - UNIT_ASSERT(!stream->Fetch()); - } catch (const TCompileError& e) { - Cerr << e; - throw e; - } + } catch (const TCompileError& e) { + Cerr << e; + throw e; } } +} // Y_UNIT_TEST_SUITE(TestUserData) diff --git a/yql/essentials/public/purecalc/ut/ya.make b/yql/essentials/public/purecalc/ut/ya.make index 2f4882c1291..a9fbe4dd9ab 100644 --- a/yql/essentials/public/purecalc/ut/ya.make +++ b/yql/essentials/public/purecalc/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST() +ENABLE(YQL_STYLE_CPP) + SRCS( empty_stream.h fake_spec.cpp diff --git a/yql/essentials/public/purecalc/ya.make b/yql/essentials/public/purecalc/ya.make index e7f3ff8818f..0d820f08348 100644 --- a/yql/essentials/public/purecalc/ya.make +++ b/yql/essentials/public/purecalc/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( purecalc.cpp ) diff --git a/yql/essentials/public/result_format/ut/ya.make b/yql/essentials/public/result_format/ut/ya.make index 6230085e7dc..22655955c78 100644 --- a/yql/essentials/public/result_format/ut/ya.make +++ b/yql/essentials/public/result_format/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(yql/essentials/public/result_format) +ENABLE(YQL_STYLE_CPP) + SRCS( yql_result_format_response_ut.cpp yql_result_format_type_ut.cpp diff --git a/yql/essentials/public/result_format/ut/yql_restricted_yson_ut.cpp b/yql/essentials/public/result_format/ut/yql_restricted_yson_ut.cpp index 2175ce63f70..fcf3d7211f6 100644 --- a/yql/essentials/public/result_format/ut/yql_restricted_yson_ut.cpp +++ b/yql/essentials/public/result_format/ut/yql_restricted_yson_ut.cpp @@ -22,74 +22,69 @@ TString Normalize(const TString& yson) { return FormatNode(NYT::NodeFromYsonString(yson)); } -} +} // namespace Y_UNIT_TEST_SUITE(TRestrictedYson) { - void RunTest(const NYT::TNode& node, const TString& expectedNodeStr, const TString& expectedEncodedStr) { - UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), expectedNodeStr); - TString encoded = NResult::EncodeRestrictedYson(node, NYson::EYsonFormat::Text); - UNIT_ASSERT_VALUES_EQUAL(Normalize(encoded), expectedEncodedStr); - TString decoded = NResult::DecodeRestrictedYson(TStringBuf(encoded), NYson::EYsonFormat::Text); - UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), Normalize(decoded)); - } - - Y_UNIT_TEST(MapScalars) { - NYT::TNode node = NYT::TNode::CreateMap(); - node["a"] = NYT::TNode("abc"); - node["b"] = NYT::TNode::CreateEntity(); - node["c"] = NYT::TNode(true); - node["d"] = NYT::TNode(1); - node["e"] = NYT::TNode(1u); - node["f"] = NYT::TNode(1.25); - - RunTest(node, +void RunTest(const NYT::TNode& node, const TString& expectedNodeStr, const TString& expectedEncodedStr) { + UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), expectedNodeStr); + TString encoded = NResult::EncodeRestrictedYson(node, NYson::EYsonFormat::Text); + UNIT_ASSERT_VALUES_EQUAL(Normalize(encoded), expectedEncodedStr); + TString decoded = NResult::DecodeRestrictedYson(TStringBuf(encoded), NYson::EYsonFormat::Text); + UNIT_ASSERT_VALUES_EQUAL(FormatNode(node), Normalize(decoded)); +} + +Y_UNIT_TEST(MapScalars) { + NYT::TNode node = NYT::TNode::CreateMap(); + node["a"] = NYT::TNode("abc"); + node["b"] = NYT::TNode::CreateEntity(); + node["c"] = NYT::TNode(true); + node["d"] = NYT::TNode(1); + node["e"] = NYT::TNode(1u); + node["f"] = NYT::TNode(1.25); + + RunTest(node, R"({"a"="abc";"b"=#;"c"=%true;"d"=1;"e"=1u;"f"=1.25})", - R"({"a"={"$type"="string";"$value"="abc"};"b"=#;"c"={"$type"="boolean";"$value"="true"};"d"={"$type"="int64";"$value"="1"};"e"={"$type"="uint64";"$value"="1"};"f"={"$type"="double";"$value"="1.25"}})" - ); - } + R"({"a"={"$type"="string";"$value"="abc"};"b"=#;"c"={"$type"="boolean";"$value"="true"};"d"={"$type"="int64";"$value"="1"};"e"={"$type"="uint64";"$value"="1"};"f"={"$type"="double";"$value"="1.25"}})"); +} - Y_UNIT_TEST(ScalarWithAttributes) { - NYT::TNode node("abc"); - node.Attributes()["d"] = NYT::TNode(true); +Y_UNIT_TEST(ScalarWithAttributes) { + NYT::TNode node("abc"); + node.Attributes()["d"] = NYT::TNode(true); - RunTest(node, + RunTest(node, R"(<"d"=%true>"abc")", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$type"="string";"$value"="abc"})" - ); - } + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$type"="string";"$value"="abc"})"); +} - Y_UNIT_TEST(MapWithAttributes) { - NYT::TNode node = NYT::TNode::CreateMap(); - node["b"] = NYT::TNode::CreateEntity(); - node["c"] = NYT::TNode(false); - node.Attributes()["d"] = NYT::TNode(true); +Y_UNIT_TEST(MapWithAttributes) { + NYT::TNode node = NYT::TNode::CreateMap(); + node["b"] = NYT::TNode::CreateEntity(); + node["c"] = NYT::TNode(false); + node.Attributes()["d"] = NYT::TNode(true); - RunTest(node, + RunTest(node, R"(<"d"=%true>{"b"=#;"c"=%false})", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"={"b"=#;"c"={"$type"="boolean";"$value"="false"}}})" - ); - } + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"={"b"=#;"c"={"$type"="boolean";"$value"="false"}}})"); +} - Y_UNIT_TEST(ListWithAttributes) { - NYT::TNode node = NYT::TNode::CreateList(); - node.Add(NYT::TNode::CreateEntity()); - node.Add(NYT::TNode(false)); - node.Attributes()["d"] = NYT::TNode(true); +Y_UNIT_TEST(ListWithAttributes) { + NYT::TNode node = NYT::TNode::CreateList(); + node.Add(NYT::TNode::CreateEntity()); + node.Add(NYT::TNode(false)); + node.Attributes()["d"] = NYT::TNode(true); - RunTest(node, + RunTest(node, R"(<"d"=%true>[#;%false])", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=[#;{"$type"="boolean";"$value"="false"}]})" - ); - } + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=[#;{"$type"="boolean";"$value"="false"}]})"); +} - Y_UNIT_TEST(EntityWithAttributes) { - NYT::TNode node = NYT::TNode::CreateEntity(); - node.Attributes()["d"] = NYT::TNode(true); +Y_UNIT_TEST(EntityWithAttributes) { + NYT::TNode node = NYT::TNode::CreateEntity(); + node.Attributes()["d"] = NYT::TNode(true); - RunTest(node, + RunTest(node, R"(<"d"=%true>#)", - R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=#})" - ); - } -} + R"({"$attributes"={"d"={"$type"="boolean";"$value"="true"}};"$value"=#})"); } +} // Y_UNIT_TEST_SUITE(TRestrictedYson) +} // namespace NYql diff --git a/yql/essentials/public/result_format/ut/yql_result_format_data_ut.cpp b/yql/essentials/public/result_format/ut/yql_result_format_data_ut.cpp index 642065b3b15..50a6a8ad49e 100644 --- a/yql/essentials/public/result_format/ut/yql_result_format_data_ut.cpp +++ b/yql/essentials/public/result_format/ut/yql_result_format_data_ut.cpp @@ -22,224 +22,224 @@ void Test(const TString& inputType, const TString& inputData, const TMaybe<TStri } Y_UNIT_TEST_SUITE(ParseData) { - Y_UNIT_TEST(EmptyVisitor) { - TEmptyDataVisitor v; - v.OnVoid(); - v.OnNull(); - v.OnEmptyList(); - v.OnEmptyDict(); - v.OnBool(true); - v.OnInt8(1); - v.OnUint8(1); - v.OnInt16(1); - v.OnUint16(1); - v.OnInt32(1); - v.OnUint32(1); - v.OnInt64(1); - v.OnUint64(1); - v.OnFloat(1.2f); - v.OnDouble(1.2f); - v.OnString("foo", true); - v.OnUtf8("foo"); - v.OnYson("foo", true); - v.OnJson("foo"); - v.OnJsonDocument("foo"); - v.OnUuid("foo", true); - v.OnDyNumber("foo", true); - v.OnDate(1); - v.OnDatetime(1); - v.OnTimestamp(1); - v.OnTzDate("foo"); - v.OnTzDatetime("foo"); - v.OnTzTimestamp("foo"); - v.OnInterval(1); - v.OnDate32(1); - v.OnDatetime64(1); - v.OnTimestamp64(1); - v.OnTzDate32("foo"); - v.OnTzDatetime64("foo"); - v.OnTzTimestamp64("foo"); - v.OnInterval64(1); - v.OnDecimal("1.2"); - v.OnBeginOptional(); - v.OnBeforeOptionalItem(); - v.OnAfterOptionalItem(); - v.OnEmptyOptional(); - v.OnEndOptional(); - v.OnBeginList(); - v.OnBeforeListItem(); - v.OnAfterListItem(); - v.OnEndList(); - v.OnBeginTuple(); - v.OnBeforeTupleItem(); - v.OnAfterTupleItem(); - v.OnEndTuple(); - v.OnBeginStruct(); - v.OnBeforeStructItem(); - v.OnAfterStructItem(); - v.OnEndStruct(); - v.OnBeginDict(); - v.OnBeforeDictItem(); - v.OnBeforeDictKey(); - v.OnAfterDictKey(); - v.OnBeforeDictPayload(); - v.OnAfterDictPayload(); - v.OnAfterDictItem(); - v.OnEndDict(); - v.OnBeginVariant(0); - v.OnEndVariant(); - v.OnPg("foo",true); - } +Y_UNIT_TEST(EmptyVisitor) { + TEmptyDataVisitor v; + v.OnVoid(); + v.OnNull(); + v.OnEmptyList(); + v.OnEmptyDict(); + v.OnBool(true); + v.OnInt8(1); + v.OnUint8(1); + v.OnInt16(1); + v.OnUint16(1); + v.OnInt32(1); + v.OnUint32(1); + v.OnInt64(1); + v.OnUint64(1); + v.OnFloat(1.2f); + v.OnDouble(1.2f); + v.OnString("foo", true); + v.OnUtf8("foo"); + v.OnYson("foo", true); + v.OnJson("foo"); + v.OnJsonDocument("foo"); + v.OnUuid("foo", true); + v.OnDyNumber("foo", true); + v.OnDate(1); + v.OnDatetime(1); + v.OnTimestamp(1); + v.OnTzDate("foo"); + v.OnTzDatetime("foo"); + v.OnTzTimestamp("foo"); + v.OnInterval(1); + v.OnDate32(1); + v.OnDatetime64(1); + v.OnTimestamp64(1); + v.OnTzDate32("foo"); + v.OnTzDatetime64("foo"); + v.OnTzTimestamp64("foo"); + v.OnInterval64(1); + v.OnDecimal("1.2"); + v.OnBeginOptional(); + v.OnBeforeOptionalItem(); + v.OnAfterOptionalItem(); + v.OnEmptyOptional(); + v.OnEndOptional(); + v.OnBeginList(); + v.OnBeforeListItem(); + v.OnAfterListItem(); + v.OnEndList(); + v.OnBeginTuple(); + v.OnBeforeTupleItem(); + v.OnAfterTupleItem(); + v.OnEndTuple(); + v.OnBeginStruct(); + v.OnBeforeStructItem(); + v.OnAfterStructItem(); + v.OnEndStruct(); + v.OnBeginDict(); + v.OnBeforeDictItem(); + v.OnBeforeDictKey(); + v.OnAfterDictKey(); + v.OnBeforeDictPayload(); + v.OnAfterDictPayload(); + v.OnAfterDictItem(); + v.OnEndDict(); + v.OnBeginVariant(0); + v.OnEndVariant(); + v.OnPg("foo", true); +} - Y_UNIT_TEST(ThrowingVisitor) { - TThrowingDataVisitor v; - UNIT_ASSERT_EXCEPTION(v.OnVoid(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnNull(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEmptyList(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEmptyDict(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBool(true), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt8(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint8(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt16(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint16(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt32(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint32(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt64(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint64(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnFloat(1.2f), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDouble(1.2f), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnString("foo", true), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUtf8("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnYson("foo", true), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnJson("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnJsonDocument("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUuid("foo", true), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDyNumber("foo", true), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDate(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDatetime(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTimestamp(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDate("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDatetime("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInterval(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDate32(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDatetime64(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTimestamp64(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDate32("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDatetime64("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp64("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInterval64(1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDecimal("1.2"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginOptional(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeOptionalItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterOptionalItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEmptyOptional(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndOptional(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginList(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeListItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterListItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndList(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginTuple(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeTupleItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterTupleItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndTuple(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginStruct(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeStructItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterStructItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndStruct(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginDict(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeDictItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeDictKey(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterDictKey(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeforeDictPayload(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterDictPayload(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnAfterDictItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndDict(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginVariant(0), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndVariant(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnPg("foo",true), TUnsupportedException); - } +Y_UNIT_TEST(ThrowingVisitor) { + TThrowingDataVisitor v; + UNIT_ASSERT_EXCEPTION(v.OnVoid(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnNull(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEmptyList(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEmptyDict(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBool(true), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt8(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint8(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt16(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint16(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt32(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint32(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt64(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint64(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnFloat(1.2f), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDouble(1.2f), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnString("foo", true), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUtf8("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnYson("foo", true), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnJson("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnJsonDocument("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUuid("foo", true), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDyNumber("foo", true), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDate(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDatetime(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTimestamp(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDate("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDatetime("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInterval(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDate32(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDatetime64(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTimestamp64(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDate32("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDatetime64("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp64("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInterval64(1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDecimal("1.2"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginOptional(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeOptionalItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterOptionalItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEmptyOptional(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndOptional(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginList(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeListItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterListItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndList(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginTuple(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeTupleItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterTupleItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndTuple(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginStruct(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeStructItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterStructItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndStruct(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginDict(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeDictItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeDictKey(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterDictKey(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeforeDictPayload(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterDictPayload(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnAfterDictItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndDict(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginVariant(0), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndVariant(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnPg("foo", true), TUnsupportedException); +} - Y_UNIT_TEST(Void) { - Test("[VoidType]","Void"); - } +Y_UNIT_TEST(Void) { + Test("[VoidType]", "Void"); +} - Y_UNIT_TEST(Null) { - Test("[NullType]","#"); - } +Y_UNIT_TEST(Null) { + Test("[NullType]", "#"); +} - Y_UNIT_TEST(EmptyList) { - Test("[EmptyListType]","[]"); - } +Y_UNIT_TEST(EmptyList) { + Test("[EmptyListType]", "[]"); +} - Y_UNIT_TEST(EmptyDict) { - Test("[EmptyDictType]","[]"); - } +Y_UNIT_TEST(EmptyDict) { + Test("[EmptyDictType]", "[]"); +} - Y_UNIT_TEST(Bool) { - Test("[DataType;Bool]","%true"); - Test("[DataType;Bool]","%false"); - } +Y_UNIT_TEST(Bool) { + Test("[DataType;Bool]", "%true"); + Test("[DataType;Bool]", "%false"); +} - Y_UNIT_TEST(Int8) { - Test("[DataType;Int8]","\"1\""); - } +Y_UNIT_TEST(Int8) { + Test("[DataType;Int8]", "\"1\""); +} - Y_UNIT_TEST(Uint8) { - Test("[DataType;Uint8]","\"1\""); - } +Y_UNIT_TEST(Uint8) { + Test("[DataType;Uint8]", "\"1\""); +} - Y_UNIT_TEST(Int16) { - Test("[DataType;Int16]","\"1\""); - } +Y_UNIT_TEST(Int16) { + Test("[DataType;Int16]", "\"1\""); +} - Y_UNIT_TEST(Uint16) { - Test("[DataType;Uint16]","\"1\""); - } +Y_UNIT_TEST(Uint16) { + Test("[DataType;Uint16]", "\"1\""); +} - Y_UNIT_TEST(Int32) { - Test("[DataType;Int32]","\"1\""); - } +Y_UNIT_TEST(Int32) { + Test("[DataType;Int32]", "\"1\""); +} - Y_UNIT_TEST(Uint32) { - Test("[DataType;Uint32]","\"1\""); - } +Y_UNIT_TEST(Uint32) { + Test("[DataType;Uint32]", "\"1\""); +} - Y_UNIT_TEST(Int64) { - Test("[DataType;Int64]","\"1\""); - } +Y_UNIT_TEST(Int64) { + Test("[DataType;Int64]", "\"1\""); +} - Y_UNIT_TEST(Uint64) { - Test("[DataType;Uint64]","\"1\""); - } +Y_UNIT_TEST(Uint64) { + Test("[DataType;Uint64]", "\"1\""); +} - Y_UNIT_TEST(Float) { - Test("[DataType;Float]","\"1.2\""); - Test("[DataType;Float]","\"-inf\""); - Test("[DataType;Float]","\"nan\""); - } +Y_UNIT_TEST(Float) { + Test("[DataType;Float]", "\"1.2\""); + Test("[DataType;Float]", "\"-inf\""); + Test("[DataType;Float]", "\"nan\""); +} - Y_UNIT_TEST(Double) { - Test("[DataType;Double]","\"1.2\""); - Test("[DataType;Double]","\"-inf\""); - Test("[DataType;Double]","\"nan\""); - } +Y_UNIT_TEST(Double) { + Test("[DataType;Double]", "\"1.2\""); + Test("[DataType;Double]", "\"-inf\""); + Test("[DataType;Double]", "\"nan\""); +} - Y_UNIT_TEST(String) { - Test("[DataType;String]","\"foo\""); - Test("[DataType;String]","[\"/w==\"]"); - } +Y_UNIT_TEST(String) { + Test("[DataType;String]", "\"foo\""); + Test("[DataType;String]", "[\"/w==\"]"); +} - Y_UNIT_TEST(Utf8) { - Test("[DataType;Utf8]","\"foo\""); - } +Y_UNIT_TEST(Utf8) { + Test("[DataType;Utf8]", "\"foo\""); +} - Y_UNIT_TEST(Json) { - Test("[DataType;Json]","\"[]\""); - } +Y_UNIT_TEST(Json) { + Test("[DataType;Json]", "\"[]\""); +} - Y_UNIT_TEST(Yson) { - Test("[DataType;Yson]",R"( +Y_UNIT_TEST(Yson) { + Test("[DataType;Yson]", R"( { "$attributes" = { "a" = { @@ -254,143 +254,143 @@ Y_UNIT_TEST_SUITE(ParseData) { } } })"); - } - - Y_UNIT_TEST(JsonDocument) { - Test("[DataType;JsonDocument]","\"[]\""); - } +} - Y_UNIT_TEST(Uuid) { - Test("[DataType;Uuid]","[\"AIQOVZvi1EGnFkRmVUQAAA==\"]"); - } +Y_UNIT_TEST(JsonDocument) { + Test("[DataType;JsonDocument]", "\"[]\""); +} - Y_UNIT_TEST(DyNumber) { - Test("[DataType;DyNumber]","[\"AoIS\"]"); - } +Y_UNIT_TEST(Uuid) { + Test("[DataType;Uuid]", "[\"AIQOVZvi1EGnFkRmVUQAAA==\"]"); +} - Y_UNIT_TEST(Date) { - Test("[DataType;Date]","\"1\""); - } +Y_UNIT_TEST(DyNumber) { + Test("[DataType;DyNumber]", "[\"AoIS\"]"); +} - Y_UNIT_TEST(Datetime) { - Test("[DataType;Datetime]","\"1\""); - } +Y_UNIT_TEST(Date) { + Test("[DataType;Date]", "\"1\""); +} - Y_UNIT_TEST(Timestamp) { - Test("[DataType;Timestamp]","\"1\""); - } +Y_UNIT_TEST(Datetime) { + Test("[DataType;Datetime]", "\"1\""); +} - Y_UNIT_TEST(TzDate) { - Test("[DataType;TzDate]","\"2011-04-27,Europe/Moscow\""); - } +Y_UNIT_TEST(Timestamp) { + Test("[DataType;Timestamp]", "\"1\""); +} - Y_UNIT_TEST(TzDatetime) { - Test("[DataType;TzDatetime]","\"2011-04-27T01:02:03,Europe/Moscow\""); - } +Y_UNIT_TEST(TzDate) { + Test("[DataType;TzDate]", "\"2011-04-27,Europe/Moscow\""); +} - Y_UNIT_TEST(TzTimestamp) { - Test("[DataType;TzTimestamp]","\"2011-04-27T01:02:03.456789,Europe/Moscow\""); - } +Y_UNIT_TEST(TzDatetime) { + Test("[DataType;TzDatetime]", "\"2011-04-27T01:02:03,Europe/Moscow\""); +} - Y_UNIT_TEST(Interval) { - Test("[DataType;Interval]","\"1\""); - } +Y_UNIT_TEST(TzTimestamp) { + Test("[DataType;TzTimestamp]", "\"2011-04-27T01:02:03.456789,Europe/Moscow\""); +} - Y_UNIT_TEST(Date32) { - Test("[DataType;Date32]","\"1\""); - } +Y_UNIT_TEST(Interval) { + Test("[DataType;Interval]", "\"1\""); +} - Y_UNIT_TEST(Datetime64) { - Test("[DataType;Datetime64]","\"1\""); - } +Y_UNIT_TEST(Date32) { + Test("[DataType;Date32]", "\"1\""); +} - Y_UNIT_TEST(Timestamp64) { - Test("[DataType;Timestamp64]","\"1\""); - } +Y_UNIT_TEST(Datetime64) { + Test("[DataType;Datetime64]", "\"1\""); +} - Y_UNIT_TEST(TzDate32) { - Test("[DataType;TzDate32]","\"2011-04-27,Europe/Moscow\""); - } +Y_UNIT_TEST(Timestamp64) { + Test("[DataType;Timestamp64]", "\"1\""); +} - Y_UNIT_TEST(TzDatetime64) { - Test("[DataType;TzDatetime64]","\"2011-04-27T01:02:03,Europe/Moscow\""); - } +Y_UNIT_TEST(TzDate32) { + Test("[DataType;TzDate32]", "\"2011-04-27,Europe/Moscow\""); +} - Y_UNIT_TEST(TzTimestamp64) { - Test("[DataType;TzTimestamp64]","\"2011-04-27T01:02:03.456789,Europe/Moscow\""); - } +Y_UNIT_TEST(TzDatetime64) { + Test("[DataType;TzDatetime64]", "\"2011-04-27T01:02:03,Europe/Moscow\""); +} - Y_UNIT_TEST(Interval64) { - Test("[DataType;Interval64]","\"1\""); - } +Y_UNIT_TEST(TzTimestamp64) { + Test("[DataType;TzTimestamp64]", "\"2011-04-27T01:02:03.456789,Europe/Moscow\""); +} - Y_UNIT_TEST(Decimal) { - Test("[DataType;Decimal;\"10\";\"1\"]","\"1.2\""); - Test("[DataType;Decimal;\"10\";\"1\"]","\"-inf\""); - Test("[DataType;Decimal;\"10\";\"1\"]","\"nan\""); - } +Y_UNIT_TEST(Interval64) { + Test("[DataType;Interval64]", "\"1\""); +} - Y_UNIT_TEST(Optional1) { - Test("[OptionalType;[DataType;Int32]]","[]"); - Test("[OptionalType;[DataType;Int32]]","[\"1\"]"); - } +Y_UNIT_TEST(Decimal) { + Test("[DataType;Decimal;\"10\";\"1\"]", "\"1.2\""); + Test("[DataType;Decimal;\"10\";\"1\"]", "\"-inf\""); + Test("[DataType;Decimal;\"10\";\"1\"]", "\"nan\""); +} - Y_UNIT_TEST(Optional2) { - Test("[OptionalType;[OptionalType;[DataType;Int32]]]","[]", "#"); - Test("[OptionalType;[OptionalType;[DataType;Int32]]]","[[]]", "[#]"); - Test("[OptionalType;[OptionalType;[DataType;Int32]]]","[[\"1\"]]"); - } +Y_UNIT_TEST(Optional1) { + Test("[OptionalType;[DataType;Int32]]", "[]"); + Test("[OptionalType;[DataType;Int32]]", "[\"1\"]"); +} - Y_UNIT_TEST(List1) { - Test("[ListType;[DataType;Int32]]","[]"); - Test("[ListType;[DataType;Int32]]","[\"1\"]"); - Test("[ListType;[DataType;Int32]]","[\"1\";\"2\"]"); - } +Y_UNIT_TEST(Optional2) { + Test("[OptionalType;[OptionalType;[DataType;Int32]]]", "[]", "#"); + Test("[OptionalType;[OptionalType;[DataType;Int32]]]", "[[]]", "[#]"); + Test("[OptionalType;[OptionalType;[DataType;Int32]]]", "[[\"1\"]]"); +} - Y_UNIT_TEST(List2) { - Test("[ListType;[ListType;[DataType;Int32]]]","[]"); - Test("[ListType;[ListType;[DataType;Int32]]]","[[];[\"1\"]]"); - Test("[ListType;[ListType;[DataType;Int32]]]","[[\"1\";\"2\";\"3\"];[\"4\";\"5\";\"6\"]]"); - } +Y_UNIT_TEST(List1) { + Test("[ListType;[DataType;Int32]]", "[]"); + Test("[ListType;[DataType;Int32]]", "[\"1\"]"); + Test("[ListType;[DataType;Int32]]", "[\"1\";\"2\"]"); +} - Y_UNIT_TEST(EmptyTuple) { - Test("[TupleType;[]]","[]"); - } +Y_UNIT_TEST(List2) { + Test("[ListType;[ListType;[DataType;Int32]]]", "[]"); + Test("[ListType;[ListType;[DataType;Int32]]]", "[[];[\"1\"]]"); + Test("[ListType;[ListType;[DataType;Int32]]]", "[[\"1\";\"2\";\"3\"];[\"4\";\"5\";\"6\"]]"); +} - Y_UNIT_TEST(Tuple) { - Test("[TupleType;[[DataType;Int32];[DataType;String]]]","[\"1\";\"foo\"]"); - } +Y_UNIT_TEST(EmptyTuple) { + Test("[TupleType;[]]", "[]"); +} - Y_UNIT_TEST(EmptyStruct) { - Test("[StructType;[]]","[]"); - } +Y_UNIT_TEST(Tuple) { + Test("[TupleType;[[DataType;Int32];[DataType;String]]]", "[\"1\";\"foo\"]"); +} - Y_UNIT_TEST(Struct) { - Test("[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]","[\"1\";\"foo\"]"); - } +Y_UNIT_TEST(EmptyStruct) { + Test("[StructType;[]]", "[]"); +} - Y_UNIT_TEST(Dict) { - Test("[DictType;[DataType;Int32];[DataType;String]]","[]"); - Test("[DictType;[DataType;Int32];[DataType;String]]","[[\"1\";\"foo\"]]"); - Test("[DictType;[DataType;Int32];[DataType;String]]","[[\"1\";\"foo\"];[\"2\";\"bar\"]]"); - } +Y_UNIT_TEST(Struct) { + Test("[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]", "[\"1\";\"foo\"]"); +} - Y_UNIT_TEST(Variant) { - Test("[VariantType;[TupleType;[[DataType;Int32];[DataType;String]]]]","[\"0\";\"7\"]"); - Test("[VariantType;[TupleType;[[DataType;Int32];[DataType;String]]]]","[\"1\";\"foo\"]"); - Test("[VariantType;[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]]","[\"0\";\"7\"]"); - Test("[VariantType;[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]]","[\"1\";\"foo\"]"); - } +Y_UNIT_TEST(Dict) { + Test("[DictType;[DataType;Int32];[DataType;String]]", "[]"); + Test("[DictType;[DataType;Int32];[DataType;String]]", "[[\"1\";\"foo\"]]"); + Test("[DictType;[DataType;Int32];[DataType;String]]", "[[\"1\";\"foo\"];[\"2\";\"bar\"]]"); +} - Y_UNIT_TEST(Tagged) { - Test("[TaggedType;foo;[DataType;Int32]]","\"1\""); - } +Y_UNIT_TEST(Variant) { + Test("[VariantType;[TupleType;[[DataType;Int32];[DataType;String]]]]", "[\"0\";\"7\"]"); + Test("[VariantType;[TupleType;[[DataType;Int32];[DataType;String]]]]", "[\"1\";\"foo\"]"); + Test("[VariantType;[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]]", "[\"0\";\"7\"]"); + Test("[VariantType;[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]]", "[\"1\";\"foo\"]"); +} - Y_UNIT_TEST(Pg) { - Test("[PgType;\"text\";\"S\"]","#"); - Test("[PgType;\"text\";\"S\"]","\"foo\""); - Test("[PgType;\"bytea\";\"U\"]","[\"EjRWeJoAvN4=\"]"); - } +Y_UNIT_TEST(Tagged) { + Test("[TaggedType;foo;[DataType;Int32]]", "\"1\""); } +Y_UNIT_TEST(Pg) { + Test("[PgType;\"text\";\"S\"]", "#"); + Test("[PgType;\"text\";\"S\"]", "\"foo\""); + Test("[PgType;\"bytea\";\"U\"]", "[\"EjRWeJoAvN4=\"]"); } +} // Y_UNIT_TEST_SUITE(ParseData) + +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/ut/yql_result_format_response_ut.cpp b/yql/essentials/public/result_format/ut/yql_result_format_response_ut.cpp index 04d05bbc5a8..1675842bdce 100644 --- a/yql/essentials/public/result_format/ut/yql_result_format_response_ut.cpp +++ b/yql/essentials/public/result_format/ut/yql_result_format_response_ut.cpp @@ -7,26 +7,26 @@ namespace NYql::NResult { Y_UNIT_TEST_SUITE(ParseResponse) { - Y_UNIT_TEST(Empty) { - auto response = NYT::NodeFromYsonString("[]"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 0); - } - - Y_UNIT_TEST(One) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(Empty) { + auto response = NYT::NodeFromYsonString("[]"); + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 0); +} + +Y_UNIT_TEST(One) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ ] }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); +} - Y_UNIT_TEST(Two) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(Two) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ ] @@ -37,12 +37,12 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 2); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 2); +} - Y_UNIT_TEST(Label) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(Label) { + auto response = NYT::NodeFromYsonString(R"([ { Label = foo; Write = [ @@ -50,15 +50,15 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT(res.Label.Defined()); - UNIT_ASSERT_VALUES_EQUAL(*res.Label, "foo"); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT(res.Label.Defined()); + UNIT_ASSERT_VALUES_EQUAL(*res.Label, "foo"); +} - Y_UNIT_TEST(Position) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(Position) { + auto response = NYT::NodeFromYsonString(R"([ { Position = { File = "<main>"; @@ -70,17 +70,17 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT(res.Position.Defined()); - UNIT_ASSERT_VALUES_EQUAL(res.Position->File, "<main>"); - UNIT_ASSERT_VALUES_EQUAL(res.Position->Row, 1); - UNIT_ASSERT_VALUES_EQUAL(res.Position->Column, 7); - } - - Y_UNIT_TEST(Truncated) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT(res.Position.Defined()); + UNIT_ASSERT_VALUES_EQUAL(res.Position->File, "<main>"); + UNIT_ASSERT_VALUES_EQUAL(res.Position->Row, 1); + UNIT_ASSERT_VALUES_EQUAL(res.Position->Column, 7); +} + +Y_UNIT_TEST(Truncated) { + auto response = NYT::NodeFromYsonString(R"([ { Truncated = %true; Write = [ @@ -88,14 +88,14 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT(res.IsTruncated); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT(res.IsTruncated); +} - Y_UNIT_TEST(Unordered) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(Unordered) { + auto response = NYT::NodeFromYsonString(R"([ { Unordered = %true; Write = [ @@ -103,14 +103,14 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT(res.IsUnordered); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT(res.IsUnordered); +} - Y_UNIT_TEST(WriteOne) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(WriteOne) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -121,17 +121,17 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Data), "\"data\""); - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Type), "\"type\""); - } - - Y_UNIT_TEST(WriteTwo) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Data), "\"data\""); + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Type), "\"type\""); +} + +Y_UNIT_TEST(WriteTwo) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -146,20 +146,20 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 2); - const auto& write1 = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write1.Data), "\"data1\""); - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write1.Type), "\"type1\""); - const auto& write2 = res.Writes[1]; - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write2.Data), "\"data2\""); - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write2.Type), "\"type2\""); - } - - Y_UNIT_TEST(WriteTruncated) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 2); + const auto& write1 = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write1.Data), "\"data1\""); + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write1.Type), "\"type1\""); + const auto& write2 = res.Writes[1]; + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write2.Data), "\"data2\""); + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write2.Type), "\"type2\""); +} + +Y_UNIT_TEST(WriteTruncated) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -171,17 +171,17 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT(write.IsTruncated); - UNIT_ASSERT(!res.IsTruncated); - } - - Y_UNIT_TEST(WriteNoType) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT(write.IsTruncated); + UNIT_ASSERT(!res.IsTruncated); +} + +Y_UNIT_TEST(WriteNoType) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -192,17 +192,17 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Data), "\"data\""); - UNIT_ASSERT(!write.Type); - } - - Y_UNIT_TEST(WriteNoData) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Data), "\"data\""); + UNIT_ASSERT(!write.Type); +} + +Y_UNIT_TEST(WriteNoData) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -213,17 +213,17 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Type), "\"type\""); - UNIT_ASSERT(!write.Data); - } - - Y_UNIT_TEST(RefsEmpty) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(NYT::NodeToCanonicalYsonString(*write.Type), "\"type\""); + UNIT_ASSERT(!write.Data); +} + +Y_UNIT_TEST(RefsEmpty) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -235,16 +235,16 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 0); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 0); +} - Y_UNIT_TEST(RefsOne) { - auto response = NYT::NodeFromYsonString(R"([ +Y_UNIT_TEST(RefsOne) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -261,19 +261,19 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 1); - const auto& ref = write.Refs[0]; - UNIT_ASSERT_VALUES_EQUAL(ref.Reference.size(), 0); - UNIT_ASSERT(ref.Remove); - } - - Y_UNIT_TEST(RefsTwo) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 1); + const auto& ref = write.Refs[0]; + UNIT_ASSERT_VALUES_EQUAL(ref.Reference.size(), 0); + UNIT_ASSERT(ref.Remove); +} + +Y_UNIT_TEST(RefsTwo) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -294,22 +294,22 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 2); - const auto& ref1 = write.Refs[0]; - UNIT_ASSERT_VALUES_EQUAL(ref1.Reference.size(), 0); - UNIT_ASSERT(ref1.Remove); - const auto& ref2 = write.Refs[1]; - UNIT_ASSERT_VALUES_EQUAL(ref2.Reference.size(), 0); - UNIT_ASSERT(ref2.Remove); - } - - Y_UNIT_TEST(RefsComponents) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 2); + const auto& ref1 = write.Refs[0]; + UNIT_ASSERT_VALUES_EQUAL(ref1.Reference.size(), 0); + UNIT_ASSERT(ref1.Remove); + const auto& ref2 = write.Refs[1]; + UNIT_ASSERT_VALUES_EQUAL(ref2.Reference.size(), 0); + UNIT_ASSERT(ref2.Remove); +} + +Y_UNIT_TEST(RefsComponents) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -326,21 +326,21 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 1); - const auto& ref = write.Refs[0]; - UNIT_ASSERT_VALUES_EQUAL(ref.Reference.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(ref.Reference[0], "foo"); - UNIT_ASSERT_VALUES_EQUAL(ref.Reference[1], "bar"); - UNIT_ASSERT(!ref.Remove); - } - - Y_UNIT_TEST(RefsColumns) { - auto response = NYT::NodeFromYsonString(R"([ + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 1); + const auto& ref = write.Refs[0]; + UNIT_ASSERT_VALUES_EQUAL(ref.Reference.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(ref.Reference[0], "foo"); + UNIT_ASSERT_VALUES_EQUAL(ref.Reference[1], "bar"); + UNIT_ASSERT(!ref.Remove); +} + +Y_UNIT_TEST(RefsColumns) { + auto response = NYT::NodeFromYsonString(R"([ { Write = [ { @@ -358,20 +358,18 @@ Y_UNIT_TEST_SUITE(ParseResponse) { }; ])"); - auto resList = ParseResponse(response); - UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); - const auto& res = resList[0]; - UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); - const auto& write = res.Writes[0]; - UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 1); - const auto& ref = write.Refs[0]; - UNIT_ASSERT(ref.Columns.Defined()); - UNIT_ASSERT_VALUES_EQUAL(ref.Columns->size(), 2); - UNIT_ASSERT_VALUES_EQUAL((*ref.Columns)[0], "col1"); - UNIT_ASSERT_VALUES_EQUAL((*ref.Columns)[1], "col2"); - } + auto resList = ParseResponse(response); + UNIT_ASSERT_VALUES_EQUAL(resList.size(), 1); + const auto& res = resList[0]; + UNIT_ASSERT_VALUES_EQUAL(res.Writes.size(), 1); + const auto& write = res.Writes[0]; + UNIT_ASSERT_VALUES_EQUAL(write.Refs.size(), 1); + const auto& ref = write.Refs[0]; + UNIT_ASSERT(ref.Columns.Defined()); + UNIT_ASSERT_VALUES_EQUAL(ref.Columns->size(), 2); + UNIT_ASSERT_VALUES_EQUAL((*ref.Columns)[0], "col1"); + UNIT_ASSERT_VALUES_EQUAL((*ref.Columns)[1], "col2"); } +} // Y_UNIT_TEST_SUITE(ParseResponse) -} - - +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/ut/yql_result_format_type_ut.cpp b/yql/essentials/public/result_format/ut/yql_result_format_type_ut.cpp index 70a34fb39d6..394d6a31515 100644 --- a/yql/essentials/public/result_format/ut/yql_result_format_type_ut.cpp +++ b/yql/essentials/public/result_format/ut/yql_result_format_type_ut.cpp @@ -14,339 +14,337 @@ void Test(const TString& input) { } Y_UNIT_TEST_SUITE(ParseType) { - Y_UNIT_TEST(EmptyVisitor) { - TEmptyTypeVisitor v; - v.OnNull(); - v.OnVoid(); - v.OnEmptyList(); - v.OnEmptyDict(); - v.OnBool(); - v.OnInt8(); - v.OnUint8(); - v.OnInt16(); - v.OnUint16(); - v.OnInt32(); - v.OnUint32(); - v.OnInt64(); - v.OnUint64(); - v.OnFloat(); - v.OnDouble(); - v.OnString(); - v.OnUtf8(); - v.OnYson(); - v.OnJson(); - v.OnJsonDocument(); - v.OnUuid(); - v.OnDyNumber(); - v.OnDate(); - v.OnDatetime(); - v.OnTimestamp(); - v.OnTzDate(); - v.OnTzDatetime(); - v.OnTzTimestamp(); - v.OnInterval(); - v.OnDate32(); - v.OnDatetime64(); - v.OnTimestamp64(); - v.OnTzDate32(); - v.OnTzDatetime64(); - v.OnTzTimestamp64(); - v.OnInterval64(); - v.OnDecimal(10, 1); - v.OnBeginOptional(); - v.OnEndOptional(); - v.OnBeginList(); - v.OnEndList(); - v.OnBeginTuple(); - v.OnTupleItem(); - v.OnEndTuple(); - v.OnBeginStruct(); - v.OnStructItem("foo"); - v.OnEndStruct(); - v.OnBeginDict(); - v.OnDictKey(); - v.OnDictPayload(); - v.OnEndDict(); - v.OnBeginVariant(); - v.OnEndVariant(); - v.OnBeginTagged("foo"); - v.OnEndTagged(); - v.OnPg("int4", "N"); - } - - Y_UNIT_TEST(ThrowingVisitor) { - TThrowingTypeVisitor v; - UNIT_ASSERT_EXCEPTION(v.OnNull(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnVoid(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEmptyList(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEmptyDict(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBool(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt8(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint8(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt16(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint16(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt32(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint32(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInt64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUint64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnFloat(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDouble(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnString(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUtf8(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnYson(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnJson(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnJsonDocument(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnUuid(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDyNumber(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDate(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDatetime(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTimestamp(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDate(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDatetime(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInterval(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDate32(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDatetime64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTimestamp64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDate32(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzDatetime64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnInterval64(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDecimal(10, 1), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginOptional(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndOptional(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginList(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndList(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginTuple(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnTupleItem(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndTuple(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginStruct(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnStructItem("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndStruct(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginDict(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDictKey(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnDictPayload(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndDict(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginVariant(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndVariant(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnBeginTagged("foo"), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnEndTagged(), TUnsupportedException); - UNIT_ASSERT_EXCEPTION(v.OnPg("int4", "N"), TUnsupportedException); - } - - Y_UNIT_TEST(BadNotList) { - UNIT_ASSERT_EXCEPTION(Test("foo"), TUnsupportedException); - } - - Y_UNIT_TEST(BadEmptyList) { - UNIT_ASSERT_EXCEPTION(Test("[]"), TUnsupportedException); - } - - Y_UNIT_TEST(BadNotStringName) { - UNIT_ASSERT_EXCEPTION(Test("[#]"), TUnsupportedException); - } - - Y_UNIT_TEST(BadUnknownName) { - UNIT_ASSERT_EXCEPTION(Test("[unknown]"), TUnsupportedException); - } - - Y_UNIT_TEST(Void) { - Test("[VoidType]"); - } - - Y_UNIT_TEST(Null) { - Test("[NullType]"); - } - - Y_UNIT_TEST(EmptyList) { - Test("[EmptyListType]"); - } - - Y_UNIT_TEST(EmptyDict) { - Test("[EmptyDictType]"); - } - - Y_UNIT_TEST(BadUnknownDataName) { - UNIT_ASSERT_EXCEPTION(Test("[DataType;unknown]"), TUnsupportedException); - } - - Y_UNIT_TEST(Bool) { - Test("[DataType;Bool]"); - } - - Y_UNIT_TEST(Int8) { - Test("[DataType;Int8]"); - } - - Y_UNIT_TEST(Uint8) { - Test("[DataType;Uint8]"); - } - - Y_UNIT_TEST(Int16) { - Test("[DataType;Int16]"); - } - - Y_UNIT_TEST(Uint16) { - Test("[DataType;Uint16]"); - } - - Y_UNIT_TEST(Int32) { - Test("[DataType;Int32]"); - } - - Y_UNIT_TEST(Uint32) { - Test("[DataType;Uint32]"); - } - - Y_UNIT_TEST(Int64) { - Test("[DataType;Int64]"); - } - - Y_UNIT_TEST(Uint64) { - Test("[DataType;Uint64]"); - } - - Y_UNIT_TEST(Float) { - Test("[DataType;Float]"); - } - - Y_UNIT_TEST(Double) { - Test("[DataType;Double]"); - } - - Y_UNIT_TEST(String) { - Test("[DataType;String]"); - } - - Y_UNIT_TEST(Utf8) { - Test("[DataType;Utf8]"); - } - - Y_UNIT_TEST(Yson) { - Test("[DataType;Yson]"); - } - - Y_UNIT_TEST(Json) { - Test("[DataType;Json]"); - } +Y_UNIT_TEST(EmptyVisitor) { + TEmptyTypeVisitor v; + v.OnNull(); + v.OnVoid(); + v.OnEmptyList(); + v.OnEmptyDict(); + v.OnBool(); + v.OnInt8(); + v.OnUint8(); + v.OnInt16(); + v.OnUint16(); + v.OnInt32(); + v.OnUint32(); + v.OnInt64(); + v.OnUint64(); + v.OnFloat(); + v.OnDouble(); + v.OnString(); + v.OnUtf8(); + v.OnYson(); + v.OnJson(); + v.OnJsonDocument(); + v.OnUuid(); + v.OnDyNumber(); + v.OnDate(); + v.OnDatetime(); + v.OnTimestamp(); + v.OnTzDate(); + v.OnTzDatetime(); + v.OnTzTimestamp(); + v.OnInterval(); + v.OnDate32(); + v.OnDatetime64(); + v.OnTimestamp64(); + v.OnTzDate32(); + v.OnTzDatetime64(); + v.OnTzTimestamp64(); + v.OnInterval64(); + v.OnDecimal(10, 1); + v.OnBeginOptional(); + v.OnEndOptional(); + v.OnBeginList(); + v.OnEndList(); + v.OnBeginTuple(); + v.OnTupleItem(); + v.OnEndTuple(); + v.OnBeginStruct(); + v.OnStructItem("foo"); + v.OnEndStruct(); + v.OnBeginDict(); + v.OnDictKey(); + v.OnDictPayload(); + v.OnEndDict(); + v.OnBeginVariant(); + v.OnEndVariant(); + v.OnBeginTagged("foo"); + v.OnEndTagged(); + v.OnPg("int4", "N"); +} + +Y_UNIT_TEST(ThrowingVisitor) { + TThrowingTypeVisitor v; + UNIT_ASSERT_EXCEPTION(v.OnNull(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnVoid(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEmptyList(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEmptyDict(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBool(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt8(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint8(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt16(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint16(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt32(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint32(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInt64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUint64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnFloat(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDouble(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnString(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUtf8(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnYson(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnJson(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnJsonDocument(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnUuid(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDyNumber(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDate(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDatetime(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTimestamp(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDate(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDatetime(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInterval(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDate32(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDatetime64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTimestamp64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDate32(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzDatetime64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTzTimestamp64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnInterval64(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDecimal(10, 1), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginOptional(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndOptional(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginList(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndList(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginTuple(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnTupleItem(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndTuple(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginStruct(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnStructItem("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndStruct(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginDict(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDictKey(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnDictPayload(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndDict(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginVariant(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndVariant(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnBeginTagged("foo"), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnEndTagged(), TUnsupportedException); + UNIT_ASSERT_EXCEPTION(v.OnPg("int4", "N"), TUnsupportedException); +} + +Y_UNIT_TEST(BadNotList) { + UNIT_ASSERT_EXCEPTION(Test("foo"), TUnsupportedException); +} + +Y_UNIT_TEST(BadEmptyList) { + UNIT_ASSERT_EXCEPTION(Test("[]"), TUnsupportedException); +} + +Y_UNIT_TEST(BadNotStringName) { + UNIT_ASSERT_EXCEPTION(Test("[#]"), TUnsupportedException); +} + +Y_UNIT_TEST(BadUnknownName) { + UNIT_ASSERT_EXCEPTION(Test("[unknown]"), TUnsupportedException); +} + +Y_UNIT_TEST(Void) { + Test("[VoidType]"); +} + +Y_UNIT_TEST(Null) { + Test("[NullType]"); +} + +Y_UNIT_TEST(EmptyList) { + Test("[EmptyListType]"); +} + +Y_UNIT_TEST(EmptyDict) { + Test("[EmptyDictType]"); +} + +Y_UNIT_TEST(BadUnknownDataName) { + UNIT_ASSERT_EXCEPTION(Test("[DataType;unknown]"), TUnsupportedException); +} + +Y_UNIT_TEST(Bool) { + Test("[DataType;Bool]"); +} + +Y_UNIT_TEST(Int8) { + Test("[DataType;Int8]"); +} + +Y_UNIT_TEST(Uint8) { + Test("[DataType;Uint8]"); +} + +Y_UNIT_TEST(Int16) { + Test("[DataType;Int16]"); +} + +Y_UNIT_TEST(Uint16) { + Test("[DataType;Uint16]"); +} + +Y_UNIT_TEST(Int32) { + Test("[DataType;Int32]"); +} + +Y_UNIT_TEST(Uint32) { + Test("[DataType;Uint32]"); +} + +Y_UNIT_TEST(Int64) { + Test("[DataType;Int64]"); +} + +Y_UNIT_TEST(Uint64) { + Test("[DataType;Uint64]"); +} - Y_UNIT_TEST(JsonDocument) { - Test("[DataType;JsonDocument]"); - } +Y_UNIT_TEST(Float) { + Test("[DataType;Float]"); +} - Y_UNIT_TEST(Uuid) { - Test("[DataType;Uuid]"); - } +Y_UNIT_TEST(Double) { + Test("[DataType;Double]"); +} - Y_UNIT_TEST(DyNumber) { - Test("[DataType;DyNumber]"); - } +Y_UNIT_TEST(String) { + Test("[DataType;String]"); +} - Y_UNIT_TEST(Date) { - Test("[DataType;Date]"); - } +Y_UNIT_TEST(Utf8) { + Test("[DataType;Utf8]"); +} - Y_UNIT_TEST(Datetime) { - Test("[DataType;Datetime]"); - } +Y_UNIT_TEST(Yson) { + Test("[DataType;Yson]"); +} - Y_UNIT_TEST(Timestamp) { - Test("[DataType;Timestamp]"); - } +Y_UNIT_TEST(Json) { + Test("[DataType;Json]"); +} - Y_UNIT_TEST(TzDate) { - Test("[DataType;TzDate]"); - } +Y_UNIT_TEST(JsonDocument) { + Test("[DataType;JsonDocument]"); +} - Y_UNIT_TEST(TzDatetime) { - Test("[DataType;TzDatetime]"); - } +Y_UNIT_TEST(Uuid) { + Test("[DataType;Uuid]"); +} - Y_UNIT_TEST(TzTimestamp) { - Test("[DataType;TzTimestamp]"); - } +Y_UNIT_TEST(DyNumber) { + Test("[DataType;DyNumber]"); +} - Y_UNIT_TEST(Interval) { - Test("[DataType;Interval]"); - } +Y_UNIT_TEST(Date) { + Test("[DataType;Date]"); +} + +Y_UNIT_TEST(Datetime) { + Test("[DataType;Datetime]"); +} + +Y_UNIT_TEST(Timestamp) { + Test("[DataType;Timestamp]"); +} + +Y_UNIT_TEST(TzDate) { + Test("[DataType;TzDate]"); +} + +Y_UNIT_TEST(TzDatetime) { + Test("[DataType;TzDatetime]"); +} - Y_UNIT_TEST(Date32) { - Test("[DataType;Date32]"); - } +Y_UNIT_TEST(TzTimestamp) { + Test("[DataType;TzTimestamp]"); +} - Y_UNIT_TEST(Datetime64) { - Test("[DataType;Datetime64]"); - } +Y_UNIT_TEST(Interval) { + Test("[DataType;Interval]"); +} - Y_UNIT_TEST(Timestamp64) { - Test("[DataType;Timestamp64]"); - } +Y_UNIT_TEST(Date32) { + Test("[DataType;Date32]"); +} - Y_UNIT_TEST(TzDate32) { - Test("[DataType;TzDate32]"); - } +Y_UNIT_TEST(Datetime64) { + Test("[DataType;Datetime64]"); +} - Y_UNIT_TEST(TzDatetime64) { - Test("[DataType;TzDatetime64]"); - } +Y_UNIT_TEST(Timestamp64) { + Test("[DataType;Timestamp64]"); +} - Y_UNIT_TEST(TzTimestamp64) { - Test("[DataType;TzTimestamp64]"); - } +Y_UNIT_TEST(TzDate32) { + Test("[DataType;TzDate32]"); +} - Y_UNIT_TEST(Interval64) { - Test("[DataType;Interval64]"); - } +Y_UNIT_TEST(TzDatetime64) { + Test("[DataType;TzDatetime64]"); +} - Y_UNIT_TEST(Decimal) { - Test("[DataType;Decimal;\"10\";\"1\"]"); - } +Y_UNIT_TEST(TzTimestamp64) { + Test("[DataType;TzTimestamp64]"); +} - Y_UNIT_TEST(OptionalInt32) { - Test("[OptionalType;[DataType;Int32]]"); - } +Y_UNIT_TEST(Interval64) { + Test("[DataType;Interval64]"); +} - Y_UNIT_TEST(ListInt32) { - Test("[ListType;[DataType;Int32]]"); - } +Y_UNIT_TEST(Decimal) { + Test("[DataType;Decimal;\"10\";\"1\"]"); +} - Y_UNIT_TEST(ListListInt32) { - Test("[ListType;[ListType;[DataType;Int32]]]"); - } +Y_UNIT_TEST(OptionalInt32) { + Test("[OptionalType;[DataType;Int32]]"); +} - Y_UNIT_TEST(TupleInt32String) { - Test("[TupleType;[[DataType;Int32];[DataType;String]]]"); - } +Y_UNIT_TEST(ListInt32) { + Test("[ListType;[DataType;Int32]]"); +} - Y_UNIT_TEST(EmptyTuple) { - Test("[TupleType;[]]"); - } +Y_UNIT_TEST(ListListInt32) { + Test("[ListType;[ListType;[DataType;Int32]]]"); +} - Y_UNIT_TEST(StructInt32String) { - Test("[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]"); - } +Y_UNIT_TEST(TupleInt32String) { + Test("[TupleType;[[DataType;Int32];[DataType;String]]]"); +} - Y_UNIT_TEST(EmptyStruct) { - Test("[StructType;[]]"); - } +Y_UNIT_TEST(EmptyTuple) { + Test("[TupleType;[]]"); +} - Y_UNIT_TEST(DictInt32String) { - Test("[DictType;[DataType;Int32];[DataType;String]]"); - } +Y_UNIT_TEST(StructInt32String) { + Test("[StructType;[[foo;[DataType;Int32]];[bar;[DataType;String]]]]"); +} - Y_UNIT_TEST(VariantTupleInt32String) { - Test("[VariantType;[TupleType;[[DataType;Int32];[DataType;String]]]]"); - } +Y_UNIT_TEST(EmptyStruct) { + Test("[StructType;[]]"); +} - Y_UNIT_TEST(TaggedInt32) { - Test("[TaggedType;foo;[DataType;Int32]]"); - } +Y_UNIT_TEST(DictInt32String) { + Test("[DictType;[DataType;Int32];[DataType;String]]"); +} - Y_UNIT_TEST(Pg) { - Test("[PgType;int4;N]"); - } +Y_UNIT_TEST(VariantTupleInt32String) { + Test("[VariantType;[TupleType;[[DataType;Int32];[DataType;String]]]]"); } +Y_UNIT_TEST(TaggedInt32) { + Test("[TaggedType;foo;[DataType;Int32]]"); } +Y_UNIT_TEST(Pg) { + Test("[PgType;int4;N]"); +} +} // Y_UNIT_TEST_SUITE(ParseType) +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/ya.make b/yql/essentials/public/result_format/ya.make index 3181d30365b..1dfc2382739 100644 --- a/yql/essentials/public/result_format/ya.make +++ b/yql/essentials/public/result_format/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( yql_result_format_response.cpp yql_result_format_type.cpp diff --git a/yql/essentials/public/result_format/yql_codec_results.cpp b/yql/essentials/public/result_format/yql_codec_results.cpp index ac961cb3927..ee7723ccbbc 100644 --- a/yql/essentials/public/result_format/yql_codec_results.cpp +++ b/yql/essentials/public/result_format/yql_codec_results.cpp @@ -21,5 +21,5 @@ void TYsonResultWriter::OnStringScalar(TStringBuf value) { } } -} -} +} // namespace NResult +} // namespace NYql diff --git a/yql/essentials/public/result_format/yql_codec_results.h b/yql/essentials/public/result_format/yql_codec_results.h index f4822b5d8c3..b42eb34c4f6 100644 --- a/yql/essentials/public/result_format/yql_codec_results.h +++ b/yql/essentials/public/result_format/yql_codec_results.h @@ -96,5 +96,5 @@ private: private: NYson::TYsonConsumerBase& Writer_; }; -} -} +} // namespace NResult +} // namespace NYql diff --git a/yql/essentials/public/result_format/yql_restricted_yson.cpp b/yql/essentials/public/result_format/yql_restricted_yson.cpp index dec45b2c2b0..893fe812d24 100644 --- a/yql/essentials/public/result_format/yql_restricted_yson.cpp +++ b/yql/essentials/public/result_format/yql_restricted_yson.cpp @@ -16,10 +16,11 @@ namespace NYql { namespace NResult { namespace { -class TRestrictedYsonFormatter : public NYson::TYsonConsumerBase { +class TRestrictedYsonFormatter: public NYson::TYsonConsumerBase { public: TRestrictedYsonFormatter(TYsonResultWriter& writer) - : Writer_(writer) { + : Writer_(writer) + { } void OnStringScalar(TStringBuf value) override { @@ -199,47 +200,47 @@ TString DecodeRestrictedBinaryString(const TString& data) { void DecodeRestrictedYson(const NYT::TNode& node, NYson::TYsonConsumerBase& writer) { switch (node.GetType()) { - case NYT::TNode::String: - writer.OnStringScalar(node.AsString()); - return; - - case NYT::TNode::Int64: - writer.OnInt64Scalar(node.AsInt64()); - return; - - case NYT::TNode::Uint64: - writer.OnUint64Scalar(node.AsUint64()); - return; - - case NYT::TNode::Bool: - writer.OnBooleanScalar(node.AsBool()); - return; - - case NYT::TNode::Double: - writer.OnDoubleScalar(node.AsDouble()); - return; - - case NYT::TNode::Null: - writer.OnEntity(); - return; - - case NYT::TNode::List: - // just a list without attributes - writer.OnBeginList(); - for (const auto& item : node.AsList()) { - writer.OnListItem(); - DecodeRestrictedYson(item, writer); - } + case NYT::TNode::String: + writer.OnStringScalar(node.AsString()); + return; + + case NYT::TNode::Int64: + writer.OnInt64Scalar(node.AsInt64()); + return; + + case NYT::TNode::Uint64: + writer.OnUint64Scalar(node.AsUint64()); + return; + + case NYT::TNode::Bool: + writer.OnBooleanScalar(node.AsBool()); + return; + + case NYT::TNode::Double: + writer.OnDoubleScalar(node.AsDouble()); + return; + + case NYT::TNode::Null: + writer.OnEntity(); + return; + + case NYT::TNode::List: + // just a list without attributes + writer.OnBeginList(); + for (const auto& item : node.AsList()) { + writer.OnListItem(); + DecodeRestrictedYson(item, writer); + } - writer.OnEndList(); - return; + writer.OnEndList(); + return; - case NYT::TNode::Map: - // process below - break; + case NYT::TNode::Map: + // process below + break; - default: - YQL_ENSURE(false, "Unsupported node type: " << static_cast<int>(node.GetType())); + default: + YQL_ENSURE(false, "Unsupported node type: " << static_cast<int>(node.GetType())); } YQL_ENSURE(node.IsMap()); @@ -325,5 +326,5 @@ TString DecodeRestrictedYson(const TStringBuf& yson, NYson::EYsonFormat format) return DecodeRestrictedYson(NYT::NodeFromYsonString(yson), format); } -} -} +} // namespace NResult +} // namespace NYql diff --git a/yql/essentials/public/result_format/yql_restricted_yson.h b/yql/essentials/public/result_format/yql_restricted_yson.h index ba88c4121a7..e5abae6588c 100644 --- a/yql/essentials/public/result_format/yql_restricted_yson.h +++ b/yql/essentials/public/result_format/yql_restricted_yson.h @@ -6,8 +6,8 @@ #include <library/cpp/yson/public.h> namespace NYT { - class TNode; -} +class TNode; +} // namespace NYT namespace NYql { namespace NResult { @@ -28,5 +28,5 @@ TString DecodeRestrictedYson( const NYT::TNode& node, NYson::EYsonFormat format = NYson::EYsonFormat::Binary); -} -} +} // namespace NResult +} // namespace NYql diff --git a/yql/essentials/public/result_format/yql_result_format_common.h b/yql/essentials/public/result_format/yql_result_format_common.h index 875104d93ff..cd82e3cec12 100644 --- a/yql/essentials/public/result_format/yql_result_format_common.h +++ b/yql/essentials/public/result_format/yql_result_format_common.h @@ -3,6 +3,6 @@ namespace NYql::NResult { -class TUnsupportedException : public yexception {}; +class TUnsupportedException: public yexception {}; -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_data.cpp b/yql/essentials/public/result_format/yql_result_format_data.cpp index df6f553750d..3330d5ce3f5 100644 --- a/yql/essentials/public/result_format/yql_result_format_data.cpp +++ b/yql/essentials/public/result_format/yql_result_format_data.cpp @@ -10,7 +10,6 @@ #include <library/cpp/string_utils/base64/base64.h> #include <util/string/cast.h> - namespace NYql::NResult { class IDataProcessor { @@ -19,7 +18,7 @@ public: virtual void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) = 0; }; -class TVoidProcessor : public IDataProcessor { +class TVoidProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsString()); @@ -28,7 +27,7 @@ public: } }; -class TNullProcessor : public IDataProcessor { +class TNullProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsEntity()); @@ -36,7 +35,7 @@ public: } }; -class TEmptyListProcessor : public IDataProcessor { +class TEmptyListProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList() && dataNode.AsList().size() == 0); @@ -44,7 +43,7 @@ public: } }; -class TEmptyDictProcessor : public IDataProcessor { +class TEmptyDictProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList() && dataNode.AsList().size() == 0); @@ -53,7 +52,7 @@ public: }; template <typename T, void (IDataVisitor::*Func)(T)> -class TIntegerProcessor : public IDataProcessor { +class TIntegerProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsString()); @@ -62,7 +61,7 @@ public: }; template <typename T, T (*Conv)(TStringBuf), void (IDataVisitor::*Func)(T)> -class TFloatingProcessor : public IDataProcessor { +class TFloatingProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsString()); @@ -70,7 +69,7 @@ public: } }; -class TBoolProcessor : public IDataProcessor { +class TBoolProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsBool()); @@ -79,11 +78,11 @@ public: }; template <void (IDataVisitor::*Func)(TStringBuf, bool)> -class TStringProcessor : public IDataProcessor { +class TStringProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsString() || - dataNode.IsList() && dataNode.AsList().size() == 1 && dataNode.AsList()[0].IsString()); + dataNode.IsList() && dataNode.AsList().size() == 1 && dataNode.AsList()[0].IsString()); if (dataNode.IsString()) { (visitor.*Func)(dataNode.AsString(), true); } else { @@ -93,7 +92,7 @@ public: }; template <void (IDataVisitor::*Func)(TStringBuf)> -class TUtf8Processor : public IDataProcessor { +class TUtf8Processor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsString()); @@ -101,7 +100,7 @@ public: } }; -class TYsonProcessor : public IDataProcessor { +class TYsonProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { auto str = DecodeRestrictedYson(dataNode); @@ -109,11 +108,12 @@ public: } }; -class TOptionalProcessor : public IDataProcessor { +class TOptionalProcessor: public IDataProcessor { public: TOptionalProcessor(std::unique_ptr<IDataProcessor>&& inner) : Inner_(std::move(inner)) - {} + { + } void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsEntity() || dataNode.IsList() && dataNode.AsList().size() <= 1); @@ -133,11 +133,12 @@ private: const std::unique_ptr<IDataProcessor> Inner_; }; -class TListProcessor : public IDataProcessor { +class TListProcessor: public IDataProcessor { public: TListProcessor(std::unique_ptr<IDataProcessor>&& inner) : Inner_(std::move(inner)) - {} + { + } void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList()); @@ -155,11 +156,12 @@ private: const std::unique_ptr<IDataProcessor> Inner_; }; -class TTupleProcessor : public IDataProcessor { +class TTupleProcessor: public IDataProcessor { public: TTupleProcessor(TVector<std::unique_ptr<IDataProcessor>>&& inners) : Inners_(std::move(inners)) - {} + { + } void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList()); @@ -178,11 +180,12 @@ private: const TVector<std::unique_ptr<IDataProcessor>> Inners_; }; -class TStructProcessor : public IDataProcessor { +class TStructProcessor: public IDataProcessor { public: TStructProcessor(TVector<std::unique_ptr<IDataProcessor>>&& inners) : Inners_(std::move(inners)) - {} + { + } void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList()); @@ -201,12 +204,13 @@ private: const TVector<std::unique_ptr<IDataProcessor>> Inners_; }; -class TDictProcessor : public IDataProcessor { +class TDictProcessor: public IDataProcessor { public: TDictProcessor(std::unique_ptr<IDataProcessor>&& innerKey, std::unique_ptr<IDataProcessor>&& innerPayload) : InnerKey_(std::move(innerKey)) , InnerPayload_(std::move(innerPayload)) - {} + { + } void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList()); @@ -231,11 +235,12 @@ private: const std::unique_ptr<IDataProcessor> InnerPayload_; }; -class TVariantProcessor : public IDataProcessor { +class TVariantProcessor: public IDataProcessor { public: TVariantProcessor(TVector<std::unique_ptr<IDataProcessor>>&& inners) : Inners_(std::move(inners)) - {} + { + } void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsList() && dataNode.AsList().size() == 2); @@ -251,11 +256,11 @@ private: const TVector<std::unique_ptr<IDataProcessor>> Inners_; }; -class TPgProcessor : public IDataProcessor { +class TPgProcessor: public IDataProcessor { public: void Process(const NYT::TNode& dataNode, IDataVisitor& visitor) final { CHECK(dataNode.IsEntity() || dataNode.IsString() || - dataNode.IsList() && dataNode.AsList().size() == 1 && dataNode.AsList()[0].IsString()); + dataNode.IsList() && dataNode.AsList().size() == 1 && dataNode.AsList()[0].IsString()); if (dataNode.IsEntity()) { visitor.OnPg(Nothing(), true); } else if (dataNode.IsString()) { @@ -266,7 +271,7 @@ public: } }; -class TDataProcessorBuilder : public TThrowingTypeVisitor { +class TDataProcessorBuilder: public TThrowingTypeVisitor { public: TDataProcessorBuilder() { IsVariant_.push_back(false); @@ -1161,4 +1166,4 @@ void TThrowingDataVisitor::Do() { void TEmptyDataVisitor::Do() { } -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_data.h b/yql/essentials/public/result_format/yql_result_format_data.h index 5c03716d6e1..4919b7cb8e4 100644 --- a/yql/essentials/public/result_format/yql_result_format_data.h +++ b/yql/essentials/public/result_format/yql_result_format_data.h @@ -77,7 +77,7 @@ public: virtual void OnPg(TMaybe<TStringBuf> value, bool isUtf8) = 0; }; -class TSameActionDataVisitor : public IDataVisitor { +class TSameActionDataVisitor: public IDataVisitor { public: void OnVoid() override; void OnNull() override; @@ -149,19 +149,19 @@ public: virtual void Do() = 0; }; -class TThrowingDataVisitor : public TSameActionDataVisitor { +class TThrowingDataVisitor: public TSameActionDataVisitor { public: void Do() final; }; -class TEmptyDataVisitor : public TSameActionDataVisitor { +class TEmptyDataVisitor: public TSameActionDataVisitor { public: void Do() final; }; void ParseData(const NYT::TNode& typeNode, const NYT::TNode& dataNode, IDataVisitor& visitor); -class TDataBuilder : public IDataVisitor { +class TDataBuilder: public IDataVisitor { public: TDataBuilder(); const NYT::TNode& GetResult() const; @@ -242,4 +242,4 @@ private: TVector<NYT::TNode*> Stack_; }; -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_impl.h b/yql/essentials/public/result_format/yql_result_format_impl.h index 6a56c350033..aa6a04be8b7 100644 --- a/yql/essentials/public/result_format/yql_result_format_impl.h +++ b/yql/essentials/public/result_format/yql_result_format_impl.h @@ -12,4 +12,4 @@ inline void Check(bool value, const TSourceLocation& location) { #define CHECK(value) Check(value, __LOCATION__) #define UNEXPECTED ythrow TUnsupportedException() << "Unhandled case" -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_response.cpp b/yql/essentials/public/result_format/yql_result_format_response.cpp index b70b3d931ef..35623f83a62 100644 --- a/yql/essentials/public/result_format/yql_result_format_response.cpp +++ b/yql/essentials/public/result_format/yql_result_format_response.cpp @@ -108,4 +108,4 @@ TVector<TResult> ParseResponse(const NYT::TNode& responseNode) { return v; } -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_response.h b/yql/essentials/public/result_format/yql_result_format_response.h index 25e144c597c..3d45ffa054a 100644 --- a/yql/essentials/public/result_format/yql_result_format_response.h +++ b/yql/essentials/public/result_format/yql_result_format_response.h @@ -31,4 +31,4 @@ struct TResult { TVector<TResult> ParseResponse(const NYT::TNode& responseNode); -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_type.cpp b/yql/essentials/public/result_format/yql_result_format_type.cpp index 39ce7334cde..29ead21ddd8 100644 --- a/yql/essentials/public/result_format/yql_result_format_type.cpp +++ b/yql/essentials/public/result_format/yql_result_format_type.cpp @@ -4,10 +4,10 @@ namespace NYql::NResult { namespace { - NYT::TNode MakeDataType(const TString& name) { - return NYT::TNode().Add("DataType").Add(name); - } +NYT::TNode MakeDataType(const TString& name) { + return NYT::TNode().Add("DataType").Add(name); } +} // namespace void ParseType(const NYT::TNode& typeNode, ITypeVisitor& visitor) { CHECK(typeNode.IsList()); @@ -196,7 +196,7 @@ void ParseType(const NYT::TNode& typeNode, ITypeVisitor& visitor) { CHECK(typeNode.AsList()[1].IsString()); CHECK(typeNode.AsList()[2].IsString()); visitor.OnPg(typeNode.AsList()[1].AsString(), typeNode.AsList()[2].AsString()); - } else { + } else { ythrow TUnsupportedException() << "Unexpected type name: " << name; } } @@ -355,8 +355,7 @@ void TTypeBuilder::OnInterval64() { } void TTypeBuilder::OnDecimal(ui32 precision, ui32 scale) { - Top() = NYT::TNode().Add("DataType").Add("Decimal") - .Add(ToString(precision)).Add(ToString(scale)); + Top() = NYT::TNode().Add("DataType").Add("Decimal").Add(ToString(precision)).Add(ToString(scale)); } NYT::TNode& TTypeBuilder::Top() { @@ -432,7 +431,6 @@ void TTypeBuilder::OnBeginDict() { Stack_.push_back(&Top()); } - void TTypeBuilder::OnDictKey() { Push(); } @@ -715,4 +713,4 @@ void TThrowingTypeVisitor::Do() { void TEmptyTypeVisitor::Do() { } -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/result_format/yql_result_format_type.h b/yql/essentials/public/result_format/yql_result_format_type.h index 360f88a4063..fb7783e0740 100644 --- a/yql/essentials/public/result_format/yql_result_format_type.h +++ b/yql/essentials/public/result_format/yql_result_format_type.h @@ -68,7 +68,7 @@ public: virtual void OnPg(TStringBuf name, TStringBuf category) = 0; }; -class TSameActionTypeVisitor : public ITypeVisitor { +class TSameActionTypeVisitor: public ITypeVisitor { public: void OnVoid() override; void OnNull() override; @@ -131,19 +131,19 @@ public: virtual void Do() = 0; }; -class TThrowingTypeVisitor : public TSameActionTypeVisitor { +class TThrowingTypeVisitor: public TSameActionTypeVisitor { public: void Do() final; }; -class TEmptyTypeVisitor : public TSameActionTypeVisitor { +class TEmptyTypeVisitor: public TSameActionTypeVisitor { public: void Do() final; }; void ParseType(const NYT::TNode& typeNode, ITypeVisitor& visitor); -class TTypeBuilder : public ITypeVisitor { +class TTypeBuilder: public ITypeVisitor { public: TTypeBuilder(); const NYT::TNode& GetResult() const; @@ -215,4 +215,4 @@ private: TVector<NYT::TNode*> Stack_; }; -} +} // namespace NYql::NResult diff --git a/yql/essentials/public/sql_format/sql_format.cpp b/yql/essentials/public/sql_format/sql_format.cpp index 2fe769e5fc7..230beae42a1 100644 --- a/yql/essentials/public/sql_format/sql_format.cpp +++ b/yql/essentials/public/sql_format/sql_format.cpp @@ -18,4 +18,4 @@ bool SqlFormatSimple(const TString& query, TString& formattedQuery, TString& err return SqlFormatSimple(lexers, parsers, query, formattedQuery, error); } -} +} // namespace NSQLFormat diff --git a/yql/essentials/public/sql_format/sql_format.h b/yql/essentials/public/sql_format/sql_format.h index afde5204e69..ecf00f0ace2 100644 --- a/yql/essentials/public/sql_format/sql_format.h +++ b/yql/essentials/public/sql_format/sql_format.h @@ -5,4 +5,4 @@ namespace NSQLFormat { bool SqlFormatSimple(const TString& query, TString& formattedQuery, TString& error); -} +} // namespace NSQLFormat diff --git a/yql/essentials/public/sql_format/ya.make b/yql/essentials/public/sql_format/ya.make index d775025e54e..fdd164a3c6b 100644 --- a/yql/essentials/public/sql_format/ya.make +++ b/yql/essentials/public/sql_format/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( sql_format.cpp ) diff --git a/yql/essentials/public/udf/arrow/args_dechunker.cpp b/yql/essentials/public/udf/arrow/args_dechunker.cpp index f52e1839836..48be9f21889 100644 --- a/yql/essentials/public/udf/arrow/args_dechunker.cpp +++ b/yql/essentials/public/udf/arrow/args_dechunker.cpp @@ -65,6 +65,5 @@ bool TArgsDechunker::Next(std::vector<arrow::Datum>& chunk, ui64& chunkLen) { return true; } - -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/args_dechunker.h b/yql/essentials/public/udf/arrow/args_dechunker.h index 6aed9491b41..9b3e288692c 100644 --- a/yql/essentials/public/udf/arrow/args_dechunker.h +++ b/yql/essentials/public/udf/arrow/args_dechunker.h @@ -22,5 +22,5 @@ private: bool Finish_ = false; }; -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/bit_util.cpp b/yql/essentials/public/udf/arrow/bit_util.cpp index 569e207167f..c6d8ea9ebdb 100644 --- a/yql/essentials/public/udf/arrow/bit_util.cpp +++ b/yql/essentials/public/udf/arrow/bit_util.cpp @@ -4,4 +4,4 @@ namespace NYql { namespace NUdf { } -} +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/bit_util.h b/yql/essentials/public/udf/arrow/bit_util.h index d8911d4a881..de2507d1c66 100644 --- a/yql/essentials/public/udf/arrow/bit_util.h +++ b/yql/essentials/public/udf/arrow/bit_util.h @@ -48,7 +48,7 @@ inline ui8* DecompressToSparseBitmap(ui8* dstSparse, const ui8* src, size_t srcO return dstSparse; } -template<bool Negate> +template <bool Negate> inline void CompressSparseImpl(ui8* dst, const ui8* srcSparse, size_t len) { while (len >= 8) { ui8 result = 0; @@ -88,7 +88,7 @@ inline void CompressSparseBitmapNegate(ui8* dst, const ui8* srcSparse, size_t le return CompressSparseImpl<true>(dst, srcSparse, len); } -template<typename T> +template <typename T> inline T* CompressArray(const T* src, const ui8* sparseBitmap, T* dst, size_t count) { while (count--) { *dst = *src++; @@ -207,4 +207,4 @@ Y_FORCE_INLINE std::array<ui64, 8> BitToByteExpand(ui8 x) { } } // namespace NUdf -} +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_builder.cpp b/yql/essentials/public/udf/arrow/block_builder.cpp index ebab4dee625..4b1aed2e42f 100644 --- a/yql/essentials/public/udf/arrow/block_builder.cpp +++ b/yql/essentials/public/udf/arrow/block_builder.cpp @@ -4,4 +4,4 @@ namespace NYql { namespace NUdf { } -} +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_builder.h b/yql/essentials/public/udf/arrow/block_builder.h index 8225ba1cd2a..a4724b8121c 100644 --- a/yql/essentials/public/udf/arrow/block_builder.h +++ b/yql/essentials/public/udf/arrow/block_builder.h @@ -39,7 +39,7 @@ public: }; inline const IArrayBuilder::TArrayDataItem* LookupArrayDataItem(const IArrayBuilder::TArrayDataItem* arrays, size_t arrayCount, ui64& idx) { - IArrayBuilder::TArrayDataItem lookup{ nullptr, idx }; + IArrayBuilder::TArrayDataItem lookup{nullptr, idx}; auto it = std::lower_bound(arrays, arrays + arrayCount, lookup, [](const auto& left, const auto& right) { return left.StartOffset < right.StartOffset; @@ -69,8 +69,9 @@ inline std::shared_ptr<arrow::DataType> GetArrowType(const ITypeInfoHelper& type return ARROW_RESULT(arrow::ImportType(&s)); } -class TArrayBuilderBase : public IArrayBuilder { +class TArrayBuilderBase: public IArrayBuilder { using Self = TArrayBuilderBase; + public: using Ptr = std::unique_ptr<TArrayBuilderBase>; @@ -137,12 +138,12 @@ public: } inline void AddMany(const arrow::ArrayData& array, ui64 beginIndex, size_t count) { - TArrayDataItem item = { &array, 0 }; + TArrayDataItem item = {&array, 0}; Self::AddMany(&item, 1, beginIndex, count); } inline void AddMany(const arrow::ArrayData& array, const ui64* indexes, size_t count) { - TArrayDataItem item = { &array, 0 }; + TArrayDataItem item = {&array, 0}; Self::AddMany(&item, 1, indexes, count); } @@ -237,6 +238,7 @@ public: CurrLen_ = 0; return result; } + protected: virtual void DoAdd(NUdf::TUnboxedValuePod value) = 0; virtual void DoAdd(TBlockItem value) = 0; @@ -333,13 +335,14 @@ protected: const size_t MaxLen_; const size_t MaxBlockSizeInBytes_; const TMaybe<ui8> MinFillPercentage_; + private: size_t CurrLen_ = 0; size_t* TotalAllocated_ = nullptr; }; -template<typename TLayout, bool Nullable, typename TDerived> -class TFixedSizeArrayBuilderBase : public TArrayBuilderBase { +template <typename TLayout, bool Nullable, typename TDerived> +class TFixedSizeArrayBuilderBase: public TArrayBuilderBase { public: TFixedSizeArrayBuilderBase(const ITypeInfoHelper& typeInfoHelper, std::shared_ptr<arrow::DataType> arrowType, arrow::MemoryPool& pool, size_t maxLen, const TParams& params) : TArrayBuilderBase(typeInfoHelper, std::move(arrowType), pool, maxLen, params) @@ -407,7 +410,7 @@ public: static_cast<TDerived*>(this)->DoAddNotNull(value, count); } - void DoAdd(TInputBuffer &input) final { + void DoAdd(TInputBuffer& input) final { if constexpr (Nullable) { if (!input.PopChar()) { DoAddNull(); @@ -451,7 +454,7 @@ public: const TLayout* values = array.GetValues<TLayout>(1); for (size_t i = beginIndex; i < beginIndex + count; ++i) { - ::new(DataPtr_ + GetCurrLen() + i - beginIndex) TLayout(values[i]); + ::new (DataPtr_ + GetCurrLen() + i - beginIndex) TLayout(values[i]); } } @@ -465,7 +468,7 @@ public: const TLayout* values = array.GetValues<TLayout>(1); for (size_t i = 0; i < count; ++i) { - ::new(DataPtr_ + GetCurrLen() + i) TLayout(values[indexes[i]]); + ::new (DataPtr_ + GetCurrLen() + i) TLayout(values[indexes[i]]); } } @@ -490,9 +493,10 @@ public: } return result; } + protected: - void PlaceItem(TLayout&& value) { - ::new(DataPtr_ + GetCurrLen()) TLayout(std::move(value)); + void PlaceItem(TLayout&& value) { + ::new (DataPtr_ + GetCurrLen()) TLayout(std::move(value)); } TLayout* DataPtr_ = nullptr; @@ -517,7 +521,7 @@ private: ui8* NullPtr_ = nullptr; }; -template<typename TLayout, bool Nullable> +template <typename TLayout, bool Nullable> class TFixedSizeArrayBuilder final: public TFixedSizeArrayBuilderBase<TLayout, Nullable, TFixedSizeArrayBuilder<TLayout, Nullable>> { using TSelf = TFixedSizeArrayBuilder<TLayout, Nullable>; using TBase = TFixedSizeArrayBuilderBase<TLayout, Nullable, TSelf>; @@ -526,11 +530,13 @@ class TFixedSizeArrayBuilder final: public TFixedSizeArrayBuilderBase<TLayout, N public: TFixedSizeArrayBuilder(const ITypeInfoHelper& typeInfoHelper, std::shared_ptr<arrow::DataType> arrowType, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TBase(typeInfoHelper, std::move(arrowType), pool, maxLen, params) - {} + { + } TFixedSizeArrayBuilder(const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TBase(typeInfoHelper, type, pool, maxLen, params) - {} + { + } void DoAddNotNull(TUnboxedValuePod value) { this->PlaceItem(value.Get<TLayout>()); @@ -549,7 +555,7 @@ public: } }; -template<bool Nullable> +template <bool Nullable> class TFixedSizeArrayBuilder<NYql::NDecimal::TInt128, Nullable> final: public TFixedSizeArrayBuilderBase<NYql::NDecimal::TInt128, Nullable, TFixedSizeArrayBuilder<NYql::NDecimal::TInt128, Nullable>> { using TSelf = TFixedSizeArrayBuilder<NYql::NDecimal::TInt128, Nullable>; using TBase = TFixedSizeArrayBuilderBase<NYql::NDecimal::TInt128, Nullable, TSelf>; @@ -558,11 +564,13 @@ class TFixedSizeArrayBuilder<NYql::NDecimal::TInt128, Nullable> final: public TF public: TFixedSizeArrayBuilder(const ITypeInfoHelper& typeInfoHelper, std::shared_ptr<arrow::DataType> arrowType, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TBase(typeInfoHelper, std::move(arrowType), pool, maxLen, params) - {} + { + } TFixedSizeArrayBuilder(const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TBase(typeInfoHelper, type, pool, maxLen, params) - {} + { + } void DoAddNotNull(TUnboxedValuePod value) { this->PlaceItem(value.GetInt128()); @@ -581,7 +589,7 @@ public: } }; -template<bool Nullable> +template <bool Nullable> class TResourceArrayBuilder final: public TFixedSizeArrayBuilderBase<TUnboxedValue, Nullable, TResourceArrayBuilder<Nullable>> { using TBase = TFixedSizeArrayBuilderBase<TUnboxedValue, Nullable, TResourceArrayBuilder<Nullable>>; using TParams = TArrayBuilderBase::TParams; @@ -589,11 +597,13 @@ class TResourceArrayBuilder final: public TFixedSizeArrayBuilderBase<TUnboxedVal public: TResourceArrayBuilder(const ITypeInfoHelper& typeInfoHelper, std::shared_ptr<arrow::DataType> arrowType, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TBase(typeInfoHelper, std::move(arrowType), pool, maxLen, params) - {} + { + } TResourceArrayBuilder(const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TBase(typeInfoHelper, type, pool, maxLen, params) - {} + { + } void DoAddNotNull(TUnboxedValuePod value) { this->PlaceItem(TUnboxedValue(value)); @@ -616,13 +626,13 @@ public: void DoAddNotNull(TBlockItem item, size_t count) { for (size_t i = 0; i < count; ++i) { - ::new(this->DataPtr_ + this->GetCurrLen() + i) TUnboxedValue(FromBlockItem(item)); + ::new (this->DataPtr_ + this->GetCurrLen() + i) TUnboxedValue(FromBlockItem(item)); } } }; -template<typename TStringType, bool Nullable, EPgStringType PgString = EPgStringType::None> -class TStringArrayBuilder final : public TArrayBuilderBase { +template <typename TStringType, bool Nullable, EPgStringType PgString = EPgStringType::None> +class TStringArrayBuilder final: public TArrayBuilderBase { using TOffset = typename TStringType::offset_type; public: @@ -734,7 +744,6 @@ public: DoAdd(TBlockItem(ref)); } - void DoAddDefault() final { if constexpr (Nullable) { NullBuilder_->UnsafeAppend(1); @@ -764,15 +773,15 @@ public: } const ui8* begin = srcData + srcOffset[i]; - const ui8* end = srcData + srcOffset[i + 1]; + const ui8* end = srcData + srcOffset[i + 1]; const size_t strSize = end - begin; - size_t availBytes = std::max(dataLen, MaxBlockSizeInBytes_) - dataLen; + size_t availBytes = std::max(dataLen, MaxBlockSizeInBytes_) - dataLen; for (;;) { // try to append ith string if (strSize <= availBytes) { - if (begin == chunkEnd) { + if (begin == chunkEnd) { chunkEnd = end; } else { DataBuilder_->UnsafeAppend(chunkStart, chunkEnd - chunkStart); @@ -838,7 +847,7 @@ public: const ui8* chunkEnd = chunkStart; for (size_t i = beginIndex; i < beginIndex + count; ++i) { const ui8* begin = srcData + offsets[i]; - const ui8* end = srcData + offsets[i + 1]; + const ui8* end = srcData + offsets[i + 1]; const size_t strSize = end - begin; size_t availBytes = std::max(dataLen, MaxBlockSizeInBytes_) - dataLen; @@ -905,7 +914,6 @@ public: } } - TBlockArrayTree::Ptr DoBuildTree(bool finish) final { FlushChunk(finish); TBlockArrayTree::Ptr result = std::make_shared<TBlockArrayTree>(); @@ -954,7 +962,7 @@ private: std::shared_ptr<arrow::Buffer> offsets = OffsetsBuilder_->Finish(); std::shared_ptr<arrow::Buffer> data = DataBuilder_->Finish(); - Chunks_.push_back(arrow::ArrayData::Make(ArrowType_, length, { nullBitmap, offsets, data })); + Chunks_.push_back(arrow::ArrayData::Make(ArrowType_, length, {nullBitmap, offsets, data})); if (!finish) { Reserve(); } @@ -970,8 +978,8 @@ private: i32 TypeLen_ = 0; }; -template<bool Nullable, typename TDerived> -class TTupleArrayBuilderBase : public TArrayBuilderBase { +template <bool Nullable, typename TDerived> +class TTupleArrayBuilderBase: public TArrayBuilderBase { public: TTupleArrayBuilderBase(const ITypeInfoHelper& typeInfoHelper, const TType* type, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) : TArrayBuilderBase(typeInfoHelper, type, pool, maxLen, params) @@ -1077,7 +1085,7 @@ public: } Y_ABORT_UNLESS(length); - result->Payload.push_back(arrow::ArrayData::Make(ArrowType_, length, { nullBitmap })); + result->Payload.push_back(arrow::ArrayData::Make(ArrowType_, length, {nullBitmap})); static_cast<TDerived*>(this)->BuildChildrenTree(finish, result->Children); if (!finish) { @@ -1101,8 +1109,8 @@ private: std::unique_ptr<TTypedBufferBuilder<ui8>> NullBuilder_; }; -template<bool Nullable> -class TTupleArrayBuilder final : public TTupleArrayBuilderBase<Nullable, TTupleArrayBuilder<Nullable>> { +template <bool Nullable> +class TTupleArrayBuilder final: public TTupleArrayBuilderBase<Nullable, TTupleArrayBuilder<Nullable>> { using TBase = TTupleArrayBuilderBase<Nullable, TTupleArrayBuilder<Nullable>>; using TParams = TArrayBuilderBase::TParams; @@ -1179,8 +1187,8 @@ private: TVector<std::unique_ptr<TArrayBuilderBase>> Children_; }; -template<typename TDate, bool Nullable> -class TTzDateArrayBuilder final : public TTupleArrayBuilderBase<Nullable, TTzDateArrayBuilder<TDate, Nullable>> { +template <typename TDate, bool Nullable> +class TTzDateArrayBuilder final: public TTupleArrayBuilderBase<Nullable, TTzDateArrayBuilder<TDate, Nullable>> { using TBase = TTupleArrayBuilderBase<Nullable, TTzDateArrayBuilder<TDate, Nullable>>; using TParams = TArrayBuilderBase::TParams; using TDateLayout = typename TDataType<TDate>::TLayout; @@ -1242,8 +1250,7 @@ private: TFixedSizeArrayBuilder<ui16, false> TimezoneBuilder_; }; - -class TExternalOptionalArrayBuilder final : public TArrayBuilderBase { +class TExternalOptionalArrayBuilder final: public TArrayBuilderBase { public: TExternalOptionalArrayBuilder(std::unique_ptr<TArrayBuilderBase>&& inner, const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, size_t maxLen, const TParams& params = {}) @@ -1338,7 +1345,7 @@ public: nullBitmap = MakeDenseBitmap(nullBitmap->data(), length, Pool_); Y_ABORT_UNLESS(length); - result->Payload.push_back(arrow::ArrayData::Make(ArrowType_, length, { nullBitmap })); + result->Payload.push_back(arrow::ArrayData::Make(ArrowType_, length, {nullBitmap})); result->Children.emplace_back(Inner_->BuildTree(finish)); if (!finish) { @@ -1363,8 +1370,9 @@ private: class TSingularBlockBuilder final: public TArrayBuilderBase { public: TSingularBlockBuilder(const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, - size_t maxLen, const TParams& params = {}) - : TArrayBuilderBase(typeInfoHelper, type, pool, maxLen, params) { + size_t maxLen, const TParams& params = {}) + : TArrayBuilderBase(typeInfoHelper, type, pool, maxLen, params) + { Reserve(); } @@ -1380,7 +1388,8 @@ public: Y_UNUSED(input); } - void DoAddDefault() final {} + void DoAddDefault() final { + } void DoAddMany(const arrow::ArrayData& array, const ui8* sparseBitmap, size_t popCount) final { Y_UNUSED(array, sparseBitmap, popCount); @@ -1418,9 +1427,9 @@ struct TBuilderTraits { template <typename TStringType, bool Nullable, NKikimr::NUdf::EDataSlot TOriginal> using TStrings = TStringArrayBuilder<TStringType, Nullable>; using TExtOptional = TExternalOptionalArrayBuilder; - template<bool Nullable> + template <bool Nullable> using TResource = TResourceArrayBuilder<Nullable>; - template<typename TTzDate, bool Nullable> + template <typename TTzDate, bool Nullable> using TTzDateReader = TTzDateArrayBuilder<TTzDate, Nullable>; using TSingular = TSingularBlockBuilder; @@ -1454,7 +1463,7 @@ struct TBuilderTraits { } } - template<typename TTzDate> + template <typename TTzDate> static std::unique_ptr<TResult> MakeTzDate(bool isOptional, const TType* type, const ITypeInfoHelper& typeInfoHelper, arrow::MemoryPool& pool, size_t maxLen, const TArrayBuilderParams& params) { if (isOptional) { return std::make_unique<TTzDateReader<TTzDate, true>>(type, typeInfoHelper, pool, maxLen, params); @@ -1472,14 +1481,14 @@ inline std::unique_ptr<IArrayBuilder> MakeArrayBuilder( const ITypeInfoHelper& typeInfoHelper, const TType* type, arrow::MemoryPool& pool, size_t maxBlockLength, const IPgBuilder* pgBuilder) { - return DispatchByArrowTraits<TBuilderTraits>(typeInfoHelper, type, pgBuilder, typeInfoHelper, pool, maxBlockLength, TArrayBuilderParams {}); + return DispatchByArrowTraits<TBuilderTraits>(typeInfoHelper, type, pgBuilder, typeInfoHelper, pool, maxBlockLength, TArrayBuilderParams{}); } inline std::unique_ptr<IArrayBuilder> MakeArrayBuilder( const ITypeInfoHelper& typeInfoHelper, const TType* type, arrow::MemoryPool& pool, size_t maxBlockLength, const IPgBuilder* pgBuilder, size_t* totalAllocated) { - return DispatchByArrowTraits<TBuilderTraits>(typeInfoHelper, type, pgBuilder, typeInfoHelper, pool, maxBlockLength, TArrayBuilderParams {.TotalAllocated = totalAllocated}); + return DispatchByArrowTraits<TBuilderTraits>(typeInfoHelper, type, pgBuilder, typeInfoHelper, pool, maxBlockLength, TArrayBuilderParams{.TotalAllocated = totalAllocated}); } inline std::unique_ptr<IArrayBuilder> MakeArrayBuilder( diff --git a/yql/essentials/public/udf/arrow/block_io_buffer.h b/yql/essentials/public/udf/arrow/block_io_buffer.h index 2d1b7b79e11..8af658f67c6 100644 --- a/yql/essentials/public/udf/arrow/block_io_buffer.h +++ b/yql/essentials/public/udf/arrow/block_io_buffer.h @@ -11,7 +11,8 @@ class TInputBuffer { public: TInputBuffer(TStringBuf buf) : Buf_(buf) - {} + { + } char PopChar() { Ensure(1); @@ -109,7 +110,5 @@ private: TVector<char> Vec_; }; - - -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_item.cpp b/yql/essentials/public/udf/arrow/block_item.cpp index c5668bd73de..cac53ed4cae 100644 --- a/yql/essentials/public/udf/arrow/block_item.cpp +++ b/yql/essentials/public/udf/arrow/block_item.cpp @@ -4,4 +4,4 @@ namespace NYql { namespace NUdf { } -} +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_item.h b/yql/essentials/public/udf/arrow/block_item.h index 79686b3094f..1bc94f7519a 100644 --- a/yql/essentials/public/udf/arrow/block_item.h +++ b/yql/essentials/public/udf/arrow/block_item.h @@ -27,7 +27,7 @@ public: *reinterpret_cast<NYql::NDecimal::TInt128*>(&Raw) = value; Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); } - + inline explicit TBlockItem(IBoxedValuePtr&& value) { Raw.Resource.Meta = static_cast<ui8>(EMarkers::Boxed); Raw.Resource.Value = value.Release(); @@ -127,26 +127,26 @@ public: Y_DEBUG_ABORT_UNLESS(GetMarkers() == EMarkers::String); return TStringValue(Raw.StringValue.Value); } - + inline TStringRef GetStringRefFromValue() const { Y_DEBUG_ABORT_UNLESS(GetMarkers() == EMarkers::String); - return { Raw.StringValue.Value->Data() + (Raw.StringValue.Offset & 0xFFFFFF), Raw.StringValue.Size }; + return {Raw.StringValue.Value->Data() + (Raw.StringValue.Offset & 0xFFFFFF), Raw.StringValue.Size}; } - inline TBlockItem MakeOptional() const - { - if (Raw.Simple.Meta) + inline TBlockItem MakeOptional() const { + if (Raw.Simple.Meta) { return *this; + } TBlockItem result(*this); ++result.Raw.Simple.Count; return result; } - inline TBlockItem GetOptionalValue() const - { - if (Raw.Simple.Meta) + inline TBlockItem GetOptionalValue() const { + if (Raw.Simple.Meta) { return *this; + } Y_DEBUG_ABORT_UNLESS(Raw.Simple.Count > 0U, "Can't get value from empty."); @@ -155,8 +155,7 @@ public: return result; } - inline IBoxedValuePtr GetBoxed() const - { + inline IBoxedValuePtr GetBoxed() const { Y_DEBUG_ABORT_UNLESS(GetMarkers() == EMarkers::Boxed, "Value is not boxed"); return Raw.Resource.Value; } @@ -178,21 +177,28 @@ public: return v; } - inline const void* GetRawPtr() const - { + inline const void* GetRawPtr() const { return &Raw; } - inline explicit operator bool() const { return bool(Raw); } - + inline explicit operator bool() const { + return bool(Raw); + } + EMarkers GetMarkers() const { return static_cast<EMarkers>(Raw.Simple.Meta); } - bool HasValue() const { return EMarkers::Empty != GetMarkers(); } + bool HasValue() const { + return EMarkers::Empty != GetMarkers(); + } - bool IsBoxed() const { return EMarkers::Boxed == GetMarkers(); } - bool IsEmbedded() const { return EMarkers::Embedded == GetMarkers(); } + bool IsBoxed() const { + return EMarkers::Boxed == GetMarkers(); + } + bool IsEmbedded() const { + return EMarkers::Embedded == GetMarkers(); + } inline void SetTimezoneId(ui16 id) { UDF_VERIFY(GetMarkers() == EMarkers::Embedded, "Value is not a datetime"); Raw.Simple.TimezoneId = id; @@ -208,16 +214,16 @@ private: ui64 Halfs[2] = {0, 0}; TRawEmbeddedValue Embedded; - + TRawBoxedValue Resource; TRawStringValue StringValue; struct { union { - #define FIELD(type) type type##_; +#define FIELD(type) type type##_; PRIMITIVE_VALUE_TYPES(FIELD); - #undef FIELD +#undef FIELD // According to the YQL <-> arrow type mapping convention, // boolean values are processed as 8-bit unsigned integer // with either 0 or 1 as a condition payload. @@ -246,33 +252,32 @@ private: const TBlockItem* Value; } Tuple; - explicit operator bool() const { return Simple.FullMeta | Simple.Count; } + explicit operator bool() const { + return Simple.FullMeta | Simple.Count; + } } Raw; }; UDF_ASSERT_TYPE_SIZE(TBlockItem, 16); -#define VALUE_AS(xType) \ - template <> \ - inline xType TBlockItem::As<xType>() const \ - { \ +#define VALUE_AS(xType) \ + template <> \ + inline xType TBlockItem::As<xType>() const { \ Y_DEBUG_ABORT_UNLESS(GetMarkers() == EMarkers::Embedded); \ - return Raw.Simple.xType##_; \ + return Raw.Simple.xType##_; \ } -#define VALUE_GET(xType) \ - template <> \ - inline xType TBlockItem::Get<xType>() const \ - { \ +#define VALUE_GET(xType) \ + template <> \ + inline xType TBlockItem::Get<xType>() const { \ Y_DEBUG_ABORT_UNLESS(GetMarkers() == EMarkers::Embedded); \ - return Raw.Simple.xType##_; \ + return Raw.Simple.xType##_; \ } -#define VALUE_CONSTR(xType) \ - template <> \ - inline TBlockItem::TBlockItem(xType value) \ - { \ - Raw.Simple.xType##_ = value; \ +#define VALUE_CONSTR(xType) \ + template <> \ + inline TBlockItem::TBlockItem(xType value) { \ + Raw.Simple.xType##_ = value; \ Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); \ } @@ -287,4 +292,4 @@ VALUE_GET(bool) #undef VALUE_GET #undef VALUE_CONSTR -} +} // namespace NYql::NUdf diff --git a/yql/essentials/public/udf/arrow/block_item_comparator.h b/yql/essentials/public/udf/arrow/block_item_comparator.h index ad803799c63..339eb213829 100644 --- a/yql/essentials/public/udf/arrow/block_item_comparator.h +++ b/yql/essentials/public/udf/arrow/block_item_comparator.h @@ -26,7 +26,7 @@ public: UDF_ASSERT_TYPE_SIZE(IBlockItemComparator, 8); template <typename TDerived, bool Nullable> -class TBlockItemComparatorBase : public IBlockItemComparator { +class TBlockItemComparatorBase: public IBlockItemComparator { public: const TDerived* Derived() const { return static_cast<const TDerived*>(this); @@ -78,8 +78,7 @@ public: if (lhs) { if (rhs) { return Derived()->DoLess(lhs, rhs); - } - else { + } else { return false; } } else { @@ -96,7 +95,7 @@ public: }; template <typename T, bool Nullable> -class TFixedSizeBlockItemComparator : public TBlockItemComparatorBase<TFixedSizeBlockItemComparator<T, Nullable>, Nullable> { +class TFixedSizeBlockItemComparator: public TBlockItemComparatorBase<TFixedSizeBlockItemComparator<T, Nullable>, Nullable> { public: i64 DoCompare(TBlockItem lhs, TBlockItem rhs) const { if constexpr (std::is_integral<T>::value && sizeof(T) < sizeof(i64)) { @@ -132,7 +131,7 @@ public: }; template <bool Nullable> -class TFixedSizeBlockItemComparator<NYql::NDecimal::TInt128, Nullable> : public TBlockItemComparatorBase<TFixedSizeBlockItemComparator<NYql::NDecimal::TInt128, Nullable>, Nullable> { +class TFixedSizeBlockItemComparator<NYql::NDecimal::TInt128, Nullable>: public TBlockItemComparatorBase<TFixedSizeBlockItemComparator<NYql::NDecimal::TInt128, Nullable>, Nullable> { public: i64 DoCompare(TBlockItem lhs, TBlockItem rhs) const { auto l = lhs.GetInt128(); @@ -154,7 +153,7 @@ public: }; template <typename TStringType, bool Nullable> -class TStringBlockItemComparator : public TBlockItemComparatorBase<TStringBlockItemComparator<TStringType, Nullable>, Nullable> { +class TStringBlockItemComparator: public TBlockItemComparatorBase<TStringBlockItemComparator<TStringType, Nullable>, Nullable> { public: i64 DoCompare(TBlockItem lhs, TBlockItem rhs) const { return lhs.AsStringRef().Compare(rhs.AsStringRef()); @@ -187,8 +186,8 @@ public: } }; -template<typename TTzType, bool Nullable> -class TTzDateBlockItemComparator : public TBlockItemComparatorBase<TTzDateBlockItemComparator<TTzType, Nullable>, Nullable> { +template <typename TTzType, bool Nullable> +class TTzDateBlockItemComparator: public TBlockItemComparatorBase<TTzDateBlockItemComparator<TTzType, Nullable>, Nullable> { using TLayout = typename TDataType<TTzType>::TLayout; public: @@ -218,13 +217,13 @@ public: } }; - template <bool Nullable> -class TTupleBlockItemComparator : public TBlockItemComparatorBase<TTupleBlockItemComparator<Nullable>, Nullable> { +class TTupleBlockItemComparator: public TBlockItemComparatorBase<TTupleBlockItemComparator<Nullable>, Nullable> { public: TTupleBlockItemComparator(TVector<std::unique_ptr<IBlockItemComparator>>&& children) : Children_(std::move(children)) - {} + { + } public: i64 DoCompare(TBlockItem lhs, TBlockItem rhs) const { @@ -267,11 +266,12 @@ private: const TVector<std::unique_ptr<IBlockItemComparator>> Children_; }; -class TExternalOptionalBlockItemComparator : public TBlockItemComparatorBase<TExternalOptionalBlockItemComparator, true> { +class TExternalOptionalBlockItemComparator: public TBlockItemComparatorBase<TExternalOptionalBlockItemComparator, true> { public: TExternalOptionalBlockItemComparator(std::unique_ptr<IBlockItemComparator> inner) : Inner_(std::move(inner)) - {} + { + } i64 DoCompare(TBlockItem lhs, TBlockItem rhs) const { return Inner_->Compare(lhs.GetOptionalValue(), rhs.GetOptionalValue()); @@ -289,4 +289,4 @@ private: std::unique_ptr<IBlockItemComparator> Inner_; }; -} +} // namespace NYql::NUdf diff --git a/yql/essentials/public/udf/arrow/block_item_hasher.h b/yql/essentials/public/udf/arrow/block_item_hasher.h index 9108d7b06e8..d776e9740ac 100644 --- a/yql/essentials/public/udf/arrow/block_item_hasher.h +++ b/yql/essentials/public/udf/arrow/block_item_hasher.h @@ -21,7 +21,7 @@ public: UDF_ASSERT_TYPE_SIZE(IBlockItemHasher, 8); template <typename TDerived, bool Nullable> -class TBlockItemHasherBase : public IBlockItemHasher { +class TBlockItemHasherBase: public IBlockItemHasher { public: const TDerived* Derived() const { return static_cast<const TDerived*>(this); @@ -42,7 +42,7 @@ public: }; template <typename T, bool Nullable> -class TFixedSizeBlockItemHasher : public TBlockItemHasherBase<TFixedSizeBlockItemHasher<T, Nullable>, Nullable> { +class TFixedSizeBlockItemHasher: public TBlockItemHasherBase<TFixedSizeBlockItemHasher<T, Nullable>, Nullable> { public: ui64 DoHash(TBlockItem value) const { return GetValueHash<TDataType<T>::Slot>(NUdf::TUnboxedValuePod(value.As<T>())); @@ -50,7 +50,7 @@ public: }; template <bool Nullable> -class TFixedSizeBlockItemHasher<NYql::NDecimal::TInt128, Nullable> : public TBlockItemHasherBase<TFixedSizeBlockItemHasher<NYql::NDecimal::TInt128, Nullable>, Nullable> { +class TFixedSizeBlockItemHasher<NYql::NDecimal::TInt128, Nullable>: public TBlockItemHasherBase<TFixedSizeBlockItemHasher<NYql::NDecimal::TInt128, Nullable>, Nullable> { public: ui64 DoHash(TBlockItem value) const { return GetValueHash<TDataType<NUdf::TDecimal>::Slot>(NUdf::TUnboxedValuePod(value.GetInt128())); @@ -58,25 +58,25 @@ public: }; template <typename T, bool Nullable> -class TTzDateBlockItemHasher : public TBlockItemHasherBase<TTzDateBlockItemHasher<T, Nullable>, Nullable> { +class TTzDateBlockItemHasher: public TBlockItemHasherBase<TTzDateBlockItemHasher<T, Nullable>, Nullable> { public: ui64 DoHash(TBlockItem value) const { using TLayout = typename TDataType<T>::TLayout; - TUnboxedValuePod uv {value.Get<TLayout>()}; + TUnboxedValuePod uv{value.Get<TLayout>()}; uv.SetTimezoneId(value.GetTimezoneId()); return GetValueHash<TDataType<T>::Slot>(uv); } }; template <typename TStringType, bool Nullable> -class TStringBlockItemHasher : public TBlockItemHasherBase<TStringBlockItemHasher<TStringType, Nullable>, Nullable> { +class TStringBlockItemHasher: public TBlockItemHasherBase<TStringBlockItemHasher<TStringType, Nullable>, Nullable> { public: ui64 DoHash(TBlockItem value) const { return GetStringHash(value.AsStringRef()); } }; -class TSingularTypeBlockItemHaser : public TBlockItemHasherBase<TSingularTypeBlockItemHaser, /*Nullable=*/false> { +class TSingularTypeBlockItemHaser: public TBlockItemHasherBase<TSingularTypeBlockItemHaser, /*Nullable=*/false> { public: ui64 DoHash(TBlockItem value) const { Y_UNUSED(value); @@ -85,11 +85,12 @@ public: }; template <bool Nullable> -class TTupleBlockItemHasher : public TBlockItemHasherBase<TTupleBlockItemHasher<Nullable>, Nullable> { +class TTupleBlockItemHasher: public TBlockItemHasherBase<TTupleBlockItemHasher<Nullable>, Nullable> { public: TTupleBlockItemHasher(TVector<std::unique_ptr<IBlockItemHasher>>&& children) : Children_(std::move(children)) - {} + { + } ui64 DoHash(TBlockItem value) const { // keep hash computation in sync with @@ -106,11 +107,12 @@ private: const TVector<std::unique_ptr<IBlockItemHasher>> Children_; }; -class TExternalOptionalBlockItemHasher : public TBlockItemHasherBase<TExternalOptionalBlockItemHasher, true> { +class TExternalOptionalBlockItemHasher: public TBlockItemHasherBase<TExternalOptionalBlockItemHasher, true> { public: TExternalOptionalBlockItemHasher(std::unique_ptr<IBlockItemHasher>&& inner) : Inner_(std::move(inner)) - {} + { + } ui64 DoHash(TBlockItem value) const { return Inner_->Hash(value.GetOptionalValue()); @@ -120,4 +122,4 @@ private: const std::unique_ptr<IBlockItemHasher> Inner_; }; -} +} // namespace NYql::NUdf diff --git a/yql/essentials/public/udf/arrow/block_reader.cpp b/yql/essentials/public/udf/arrow/block_reader.cpp index 89d21389af4..76db9fba3a5 100644 --- a/yql/essentials/public/udf/arrow/block_reader.cpp +++ b/yql/essentials/public/udf/arrow/block_reader.cpp @@ -4,4 +4,4 @@ namespace NYql { namespace NUdf { } -} +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_reader.h b/yql/essentials/public/udf/arrow/block_reader.h index 25a3dbe4775..cb81e6b8745 100644 --- a/yql/essentials/public/udf/arrow/block_reader.h +++ b/yql/essentials/public/udf/arrow/block_reader.h @@ -12,7 +12,7 @@ namespace NYql { namespace NUdf { -class IBlockReader : private TNonCopyable { +class IBlockReader: private TNonCopyable { public: virtual ~IBlockReader() = default; // result will reference to Array/Scalar internals and will be valid until next call to GetItem/GetScalarItem @@ -34,7 +34,7 @@ struct TBlockItemSerializeProps { bool IsFixed = true; // true if each block item takes fixed size }; -class TBlockReaderBase : public IBlockReader { +class TBlockReaderBase: public IBlockReader { public: ui64 GetSliceDataWeight(const arrow::ArrayData& data, int64_t offset, int64_t length) const final { Y_ENSURE(0 <= offset && offset < data.length); @@ -54,8 +54,8 @@ protected: } }; -template<typename T, bool Nullable, typename TDerived> -class TFixedSizeBlockReaderBase : public TBlockReaderBase { +template <typename T, bool Nullable, typename TDerived> +class TFixedSizeBlockReaderBase: public TBlockReaderBase { public: TBlockItem GetItem(const arrow::ArrayData& data, size_t index) final { if constexpr (Nullable) { @@ -75,14 +75,14 @@ public: } } - if constexpr(std::is_same_v<T, NYql::NDecimal::TInt128>) { + if constexpr (std::is_same_v<T, NYql::NDecimal::TInt128>) { auto& fixedScalar = checked_cast<const arrow::FixedSizeBinaryScalar&>(scalar); - T value; memcpy((void*)&value, fixedScalar.value->data(), sizeof(T)); + T value; + memcpy((void*)&value, fixedScalar.value->data(), sizeof(T)); return static_cast<TDerived*>(this)->MakeBlockItem(value); } else { return static_cast<TDerived*>(this)->MakeBlockItem( - *static_cast<const T*>(checked_cast<const PrimitiveScalarBase&>(scalar).data()) - ); + *static_cast<const T*>(checked_cast<const PrimitiveScalarBase&>(scalar).data())); } } @@ -126,9 +126,10 @@ public: out.PushChar(1); } - if constexpr(std::is_same_v<T, NYql::NDecimal::TInt128>) { + if constexpr (std::is_same_v<T, NYql::NDecimal::TInt128>) { auto& fixedScalar = arrow::internal::checked_cast<const arrow::FixedSizeBinaryScalar&>(scalar); - T value; memcpy((void*)&value, fixedScalar.value->data(), sizeof(T)); + T value; + memcpy((void*)&value, fixedScalar.value->data(), sizeof(T)); out.PushNumber(value); } else { out.PushNumber(*static_cast<const T*>(arrow::internal::checked_cast<const arrow::internal::PrimitiveScalarBase&>(scalar).data())); @@ -145,16 +146,16 @@ private: } }; -template<typename T, bool Nullable> -class TFixedSizeBlockReader : public TFixedSizeBlockReaderBase<T, Nullable, TFixedSizeBlockReader<T, Nullable>> { +template <typename T, bool Nullable> +class TFixedSizeBlockReader: public TFixedSizeBlockReaderBase<T, Nullable, TFixedSizeBlockReader<T, Nullable>> { public: TBlockItem MakeBlockItem(const T& item) const { return TBlockItem(item); } }; -template<bool Nullable> -class TResourceBlockReader : public TFixedSizeBlockReaderBase<TUnboxedValuePod, Nullable, TResourceBlockReader<Nullable>> { +template <bool Nullable> +class TResourceBlockReader: public TFixedSizeBlockReaderBase<TUnboxedValuePod, Nullable, TResourceBlockReader<Nullable>> { public: TBlockItem MakeBlockItem(const TUnboxedValuePod& pod) const { TBlockItem item; @@ -163,8 +164,8 @@ public: } }; -template<typename TStringType, bool Nullable, NKikimr::NUdf::EDataSlot TOriginal = NKikimr::NUdf::EDataSlot::String> -class TStringBlockReader final : public TBlockReaderBase { +template <typename TStringType, bool Nullable, NKikimr::NUdf::EDataSlot TOriginal = NKikimr::NUdf::EDataSlot::String> +class TStringBlockReader final: public TBlockReaderBase { public: using TOffset = typename TStringType::offset_type; @@ -258,8 +259,8 @@ private: } }; -template<bool Nullable, typename TDerived> -class TTupleBlockReaderBase : public TBlockReaderBase { +template <bool Nullable, typename TDerived> +class TTupleBlockReaderBase: public TBlockReaderBase { public: TBlockItem GetItem(const arrow::ArrayData& data, size_t index) final { if constexpr (Nullable) { @@ -337,13 +338,14 @@ public: } }; -template<bool Nullable> -class TTupleBlockReader final : public TTupleBlockReaderBase<Nullable, TTupleBlockReader<Nullable>> { +template <bool Nullable> +class TTupleBlockReader final: public TTupleBlockReaderBase<Nullable, TTupleBlockReader<Nullable>> { public: TTupleBlockReader(TVector<std::unique_ptr<IBlockReader>>&& children) : Children_(std::move(children)) , Items_(Children_.size()) - {} + { + } TBlockItem GetChildrenItems(const arrow::ArrayData& data, size_t index) { for (ui32 i = 0; i < Children_.size(); ++i) { @@ -423,13 +425,13 @@ private: TVector<TBlockItem> Items_; }; -template<typename TTzDate, bool Nullable> -class TTzDateBlockReader final : public TTupleBlockReaderBase<Nullable, TTzDateBlockReader<TTzDate, Nullable>> { +template <typename TTzDate, bool Nullable> +class TTzDateBlockReader final: public TTupleBlockReaderBase<Nullable, TTzDateBlockReader<TTzDate, Nullable>> { public: TBlockItem GetChildrenItems(const arrow::ArrayData& data, size_t index) { Y_DEBUG_ABORT_UNLESS(data.child_data.size() == 2); - TBlockItem item {DateReader_.GetItem(*data.child_data[0], index)}; + TBlockItem item{DateReader_.GetItem(*data.child_data[0], index)}; item.SetTimezoneId(TimezoneReader_.GetItem(*data.child_data[1], index).Get<ui16>()); return item; } @@ -437,7 +439,7 @@ public: TBlockItem GetChildrenScalarItems(const arrow::StructScalar& structScalar) { Y_DEBUG_ABORT_UNLESS(structScalar.value.size() == 2); - TBlockItem item {DateReader_.GetScalarItem(*structScalar.value[0])}; + TBlockItem item{DateReader_.GetScalarItem(*structScalar.value[0])}; item.SetTimezoneId(TimezoneReader_.GetScalarItem(*structScalar.value[1]).Get<ui16>()); return item; } @@ -487,8 +489,8 @@ public: } private: - TFixedSizeBlockReader<typename TDataType<TTzDate>::TLayout, /* Nullable */false> DateReader_; - TFixedSizeBlockReader<ui16, /* Nullable */false> TimezoneReader_; + TFixedSizeBlockReader<typename TDataType<TTzDate>::TLayout, /* Nullable */ false> DateReader_; + TFixedSizeBlockReader<ui16, /* Nullable */ false> TimezoneReader_; }; // NOTE: For any singular type we use arrow::null() data type. @@ -538,11 +540,12 @@ public: } }; -class TExternalOptionalBlockReader final : public TBlockReaderBase { +class TExternalOptionalBlockReader final: public TBlockReaderBase { public: TExternalOptionalBlockReader(std::unique_ptr<IBlockReader>&& inner) : Inner_(std::move(inner)) - {} + { + } TBlockItem GetItem(const arrow::ArrayData& data, size_t index) final { if (IsNull(data, index)) { @@ -612,9 +615,9 @@ struct TReaderTraits { template <typename TStringType, bool Nullable, NKikimr::NUdf::EDataSlot TOriginal> using TStrings = TStringBlockReader<TStringType, Nullable, TOriginal>; using TExtOptional = TExternalOptionalBlockReader; - template<bool Nullable> + template <bool Nullable> using TResource = TResourceBlockReader<Nullable>; - template<typename TTzDate, bool Nullable> + template <typename TTzDate, bool Nullable> using TTzDateReader = TTzDateBlockReader<TTzDate, Nullable>; using TSingularType = TSingularTypeBlockReader; @@ -641,7 +644,7 @@ struct TReaderTraits { return std::make_unique<TSingularType>(); } - template<typename TTzDate> + template <typename TTzDate> static std::unique_ptr<TResult> MakeTzDate(bool isOptional) { if (isOptional) { return std::make_unique<TTzDateReader<TTzDate, true>>(); @@ -698,8 +701,7 @@ inline void UpdateBlockItemSerializeProps(const ITypeInfoHelper& typeInfoHelper, props.IsFixed = false; } else if (dataTypeInfo.Features & TzDateType) { *props.MaxSize += dataTypeInfo.FixedSize + sizeof(TTimezoneId); - } - else { + } else { *props.MaxSize += dataTypeInfo.FixedSize; } return; @@ -725,5 +727,5 @@ inline void UpdateBlockItemSerializeProps(const ITypeInfoHelper& typeInfoHelper, Y_ENSURE(false, "Unsupported type"); } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_type_helper.cpp b/yql/essentials/public/udf/arrow/block_type_helper.cpp index 362d9faf254..33477afd563 100644 --- a/yql/essentials/public/udf/arrow/block_type_helper.cpp +++ b/yql/essentials/public/udf/arrow/block_type_helper.cpp @@ -3,7 +3,8 @@ namespace NYql { namespace NUdf { -IBlockTypeHelper::IBlockTypeHelper() {} +IBlockTypeHelper::IBlockTypeHelper() { +} } // namespace NUdf } // namespace NYql diff --git a/yql/essentials/public/udf/arrow/block_type_helper.h b/yql/essentials/public/udf/arrow/block_type_helper.h index 55995c5f16d..a3aa995b393 100644 --- a/yql/essentials/public/udf/arrow/block_type_helper.h +++ b/yql/essentials/public/udf/arrow/block_type_helper.h @@ -17,19 +17,19 @@ public: }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 34) -class IBlockTypeHelper2 : public IBlockTypeHelper1 { +class IBlockTypeHelper2: public IBlockTypeHelper1 { public: - virtual IBlockItemHasher::TPtr MakeHasher(TType *type) const = 0; + virtual IBlockItemHasher::TPtr MakeHasher(TType* type) const = 0; }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 34) -class IBlockTypeHelper : public IBlockTypeHelper2 { +class IBlockTypeHelper: public IBlockTypeHelper2 { public: IBlockTypeHelper(); }; #else -class IBlockTypeHelper : public IBlockTypeHelper1 { +class IBlockTypeHelper: public IBlockTypeHelper1 { public: IBlockTypeHelper(); }; @@ -37,11 +37,10 @@ public: UDF_ASSERT_TYPE_SIZE(IBlockTypeHelper, 8); -template<EDataSlot slot> +template <EDataSlot slot> std::shared_ptr<arrow::DataType> MakeTzLayoutArrowType() { - static_assert(slot == EDataSlot::TzDate || slot == EDataSlot::TzDatetime || slot == EDataSlot::TzTimestamp - || slot == EDataSlot::TzDate32 || slot == EDataSlot::TzDatetime64 || slot == EDataSlot::TzTimestamp64, - "Expected tz date type slot"); + static_assert(slot == EDataSlot::TzDate || slot == EDataSlot::TzDatetime || slot == EDataSlot::TzTimestamp || slot == EDataSlot::TzDate32 || slot == EDataSlot::TzDatetime64 || slot == EDataSlot::TzTimestamp64, + "Expected tz date type slot"); if constexpr (slot == EDataSlot::TzDate) { return arrow::uint16(); @@ -63,5 +62,5 @@ std::shared_ptr<arrow::DataType> MakeTzLayoutArrowType() { } } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/defs.h b/yql/essentials/public/udf/arrow/defs.h index f05965396b8..f468802ab86 100644 --- a/yql/essentials/public/udf/arrow/defs.h +++ b/yql/essentials/public/udf/arrow/defs.h @@ -3,30 +3,31 @@ #include <arrow/result.h> #include <util/generic/yexception.h> -#define ARROW_CHECK_STATUS(s, op, ...) \ - if (!s.ok()) { \ - ythrow yexception() << "Operation failed: [" << #op << "]\n" \ - << "" __VA_ARGS__ << ": [" << s.ToString() << "]"; \ +#define ARROW_CHECK_STATUS(s, op, ...) \ + if (!s.ok()) { \ + ythrow yexception() << "Operation failed: [" << #op << "]\n" \ + << "" __VA_ARGS__ << ": [" << s.ToString() << "]"; \ } -#define ARROW_OK_S(op, ...) \ -do { \ - ::arrow::Status _s = (op); \ - ARROW_CHECK_STATUS(_s, op, __VA_ARGS__); \ - } while (false) +#define ARROW_OK_S(op, ...) \ + do { \ + ::arrow::Status _s = (op); \ + ARROW_CHECK_STATUS(_s, op, __VA_ARGS__); \ + } while (false) -#define ARROW_OK(op) ARROW_OK_S(op, "Bad status") +#define ARROW_OK(op) ARROW_OK_S(op, "Bad status") -#define ARROW_RESULT_S(op, ...) \ - [&]() { \ - auto result = (op); \ - ARROW_CHECK_STATUS(result.status(), op, __VA_ARGS__); \ - return std::move(result).ValueOrDie(); \ +#define ARROW_RESULT_S(op, ...) \ + [&]() { \ + auto result = (op); \ + ARROW_CHECK_STATUS(result.status(), op, __VA_ARGS__); \ + return std::move(result).ValueOrDie(); \ }() -#define ARROW_RESULT(op) ARROW_RESULT_S(op, "Bad status") +#define ARROW_RESULT(op) ARROW_RESULT_S(op, "Bad status") -#define ARROW_DEBUG_CHECK_DATUM_TYPES(expected, got) do { \ - Y_DEBUG_ABORT_UNLESS((expected) == (got), "Bad datum type: %s expected, %s got", \ - (expected).ToString().c_str(), (got).ToString().c_str()); \ -} while(false) +#define ARROW_DEBUG_CHECK_DATUM_TYPES(expected, got) \ + do { \ + Y_DEBUG_ABORT_UNLESS((expected) == (got), "Bad datum type: %s expected, %s got", \ + (expected).ToString().c_str(), (got).ToString().c_str()); \ + } while (false) diff --git a/yql/essentials/public/udf/arrow/dispatch_traits.h b/yql/essentials/public/udf/arrow/dispatch_traits.h index 9fb6559f4aa..b4f54608571 100644 --- a/yql/essentials/public/udf/arrow/dispatch_traits.h +++ b/yql/essentials/public/udf/arrow/dispatch_traits.h @@ -69,7 +69,7 @@ std::unique_ptr<typename TTraits::TResult> MakeTzDateArrowTraitsImpl(bool isOpti } } -template<typename TTraits> +template <typename TTraits> concept CanInstantiateArrowTraitsForDecimal = requires { typename TTraits::template TFixedSize<NYql::NDecimal::TInt128, true>; }; @@ -150,67 +150,67 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo if (typeData) { auto typeId = typeData.GetTypeId(); switch (GetDataSlot(typeId)) { - case NUdf::EDataSlot::Int8: - return MakeFixedSizeArrowTraitsImpl<TTraits, i8>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Bool: - case NUdf::EDataSlot::Uint8: - return MakeFixedSizeArrowTraitsImpl<TTraits, ui8>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Int16: - return MakeFixedSizeArrowTraitsImpl<TTraits, i16>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Uint16: - case NUdf::EDataSlot::Date: - return MakeFixedSizeArrowTraitsImpl<TTraits, ui16>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Int32: - case NUdf::EDataSlot::Date32: - return MakeFixedSizeArrowTraitsImpl<TTraits, i32>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Uint32: - case NUdf::EDataSlot::Datetime: - return MakeFixedSizeArrowTraitsImpl<TTraits, ui32>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Int64: - case NUdf::EDataSlot::Interval: - case NUdf::EDataSlot::Interval64: - case NUdf::EDataSlot::Datetime64: - case NUdf::EDataSlot::Timestamp64: - return MakeFixedSizeArrowTraitsImpl<TTraits, i64>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Uint64: - case NUdf::EDataSlot::Timestamp: - return MakeFixedSizeArrowTraitsImpl<TTraits, ui64>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Float: - return MakeFixedSizeArrowTraitsImpl<TTraits, float>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Double: - return MakeFixedSizeArrowTraitsImpl<TTraits, double>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::String: - return MakeStringArrowTraitsImpl<TTraits, arrow::BinaryType, NUdf::EDataSlot::String>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Yson: - return MakeStringArrowTraitsImpl<TTraits, arrow::BinaryType, NUdf::EDataSlot::Yson>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::JsonDocument: - return MakeStringArrowTraitsImpl<TTraits, arrow::BinaryType, NUdf::EDataSlot::JsonDocument>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Utf8: - return MakeStringArrowTraitsImpl<TTraits, arrow::StringType, NUdf::EDataSlot::Utf8>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Json: - return MakeStringArrowTraitsImpl<TTraits, arrow::StringType, NUdf::EDataSlot::Json>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::TzDate: - return MakeTzDateArrowTraitsImpl<TTraits, TTzDate>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::TzDatetime: - return MakeTzDateArrowTraitsImpl<TTraits, TTzDatetime>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::TzTimestamp: - return MakeTzDateArrowTraitsImpl<TTraits, TTzTimestamp>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::TzDate32: - return MakeTzDateArrowTraitsImpl<TTraits, TTzDate32>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::TzDatetime64: - return MakeTzDateArrowTraitsImpl<TTraits, TTzDatetime64>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::TzTimestamp64: - return MakeTzDateArrowTraitsImpl<TTraits, TTzTimestamp64>(isOptional, type, std::forward<TArgs>(args)...); - case NUdf::EDataSlot::Decimal: { - if constexpr (CanInstantiateArrowTraitsForDecimal<TTraits>) { - return MakeFixedSizeArrowTraitsImpl<TTraits, NYql::NDecimal::TInt128>(isOptional, type, std::forward<TArgs>(args)...); - } else { - Y_ENSURE(false, "Unsupported data slot"); + case NUdf::EDataSlot::Int8: + return MakeFixedSizeArrowTraitsImpl<TTraits, i8>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Bool: + case NUdf::EDataSlot::Uint8: + return MakeFixedSizeArrowTraitsImpl<TTraits, ui8>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Int16: + return MakeFixedSizeArrowTraitsImpl<TTraits, i16>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Uint16: + case NUdf::EDataSlot::Date: + return MakeFixedSizeArrowTraitsImpl<TTraits, ui16>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Int32: + case NUdf::EDataSlot::Date32: + return MakeFixedSizeArrowTraitsImpl<TTraits, i32>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Uint32: + case NUdf::EDataSlot::Datetime: + return MakeFixedSizeArrowTraitsImpl<TTraits, ui32>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Int64: + case NUdf::EDataSlot::Interval: + case NUdf::EDataSlot::Interval64: + case NUdf::EDataSlot::Datetime64: + case NUdf::EDataSlot::Timestamp64: + return MakeFixedSizeArrowTraitsImpl<TTraits, i64>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Uint64: + case NUdf::EDataSlot::Timestamp: + return MakeFixedSizeArrowTraitsImpl<TTraits, ui64>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Float: + return MakeFixedSizeArrowTraitsImpl<TTraits, float>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Double: + return MakeFixedSizeArrowTraitsImpl<TTraits, double>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::String: + return MakeStringArrowTraitsImpl<TTraits, arrow::BinaryType, NUdf::EDataSlot::String>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Yson: + return MakeStringArrowTraitsImpl<TTraits, arrow::BinaryType, NUdf::EDataSlot::Yson>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::JsonDocument: + return MakeStringArrowTraitsImpl<TTraits, arrow::BinaryType, NUdf::EDataSlot::JsonDocument>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Utf8: + return MakeStringArrowTraitsImpl<TTraits, arrow::StringType, NUdf::EDataSlot::Utf8>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Json: + return MakeStringArrowTraitsImpl<TTraits, arrow::StringType, NUdf::EDataSlot::Json>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::TzDate: + return MakeTzDateArrowTraitsImpl<TTraits, TTzDate>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::TzDatetime: + return MakeTzDateArrowTraitsImpl<TTraits, TTzDatetime>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::TzTimestamp: + return MakeTzDateArrowTraitsImpl<TTraits, TTzTimestamp>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::TzDate32: + return MakeTzDateArrowTraitsImpl<TTraits, TTzDate32>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::TzDatetime64: + return MakeTzDateArrowTraitsImpl<TTraits, TTzDatetime64>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::TzTimestamp64: + return MakeTzDateArrowTraitsImpl<TTraits, TTzTimestamp64>(isOptional, type, std::forward<TArgs>(args)...); + case NUdf::EDataSlot::Decimal: { + if constexpr (CanInstantiateArrowTraitsForDecimal<TTraits>) { + return MakeFixedSizeArrowTraitsImpl<TTraits, NYql::NDecimal::TInt128>(isOptional, type, std::forward<TArgs>(args)...); + } else { + Y_ENSURE(false, "Unsupported data slot"); + } } - } - case NUdf::EDataSlot::Uuid: - case NUdf::EDataSlot::DyNumber: - Y_ENSURE(false, "Unsupported data slot"); + case NUdf::EDataSlot::Uuid: + case NUdf::EDataSlot::DyNumber: + Y_ENSURE(false, "Unsupported data slot"); } } @@ -245,5 +245,5 @@ std::unique_ptr<typename TTraits::TResult> DispatchByArrowTraits(const ITypeInfo Y_ENSURE(false, "Unsupported type"); } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/memory_pool.cpp b/yql/essentials/public/udf/arrow/memory_pool.cpp index 5a074b60aba..281973e1f08 100644 --- a/yql/essentials/public/udf/arrow/memory_pool.cpp +++ b/yql/essentials/public/udf/arrow/memory_pool.cpp @@ -8,7 +8,7 @@ namespace NYql { namespace NUdf { -class TYqlMemoryPool : public arrow::MemoryPool { +class TYqlMemoryPool: public arrow::MemoryPool { arrow::Status Allocate(int64_t size, uint8_t** out) final { Y_ENSURE(size >= 0 && out); *out = (uint8_t*)UdfArrowAllocate(size); @@ -58,5 +58,5 @@ arrow::MemoryPool* GetYqlMemoryPool() { return Singleton<TYqlMemoryPool>(); } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/memory_pool.h b/yql/essentials/public/udf/arrow/memory_pool.h index bad76f9450e..1bc7874e81d 100644 --- a/yql/essentials/public/udf/arrow/memory_pool.h +++ b/yql/essentials/public/udf/arrow/memory_pool.h @@ -17,5 +17,5 @@ inline arrow::MemoryPool* GetYqlMemoryPool() { } #endif -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/udf_arrow_helpers.cpp b/yql/essentials/public/udf/arrow/udf_arrow_helpers.cpp index dea156b206a..44493835929 100644 --- a/yql/essentials/public/udf/arrow/udf_arrow_helpers.cpp +++ b/yql/essentials/public/udf/arrow/udf_arrow_helpers.cpp @@ -4,4 +4,4 @@ namespace NYql { namespace NUdf { } -} +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/udf_arrow_helpers.h b/yql/essentials/public/udf/arrow/udf_arrow_helpers.h index add584513f9..f2e0de8a68d 100644 --- a/yql/essentials/public/udf/arrow/udf_arrow_helpers.h +++ b/yql/essentials/public/udf/arrow/udf_arrow_helpers.h @@ -23,9 +23,9 @@ namespace NYql { namespace NUdf { -using TExec = arrow::Status(*)(arrow::compute::KernelContext*, const arrow::compute::ExecBatch&, arrow::Datum*); +using TExec = arrow::Status (*)(arrow::compute::KernelContext*, const arrow::compute::ExecBatch&, arrow::Datum*); -class TUdfKernelState : public arrow::compute::KernelState { +class TUdfKernelState: public arrow::compute::KernelState { public: TUdfKernelState(const TVector<const TType*>& argTypes, const TType* outputType, bool onlyScalars, const ITypeInfoHelper* typeInfoHelper, const IValueBuilder* valueBuilder) : ArgTypes_(argTypes) @@ -79,11 +79,11 @@ private: std::unique_ptr<IScalarBuilder> ScalarBuilder_; }; -class TSimpleArrowUdfImpl : public TBoxedValue { +class TSimpleArrowUdfImpl: public TBoxedValue { public: TSimpleArrowUdfImpl(const TVector<const TType*> argBlockTypes, const TType* outputType, bool onlyScalars, - TExec exec, IFunctionTypeInfoBuilder& builder, const TString& name, - arrow::compute::NullHandling::type nullHandling) + TExec exec, IFunctionTypeInfoBuilder& builder, const TString& name, + arrow::compute::NullHandling::type nullHandling) : OnlyScalars_(onlyScalars) , Exec_(exec) , Pos_(GetSourcePosition(builder)) @@ -170,7 +170,7 @@ public: arrow::Datum res; if (OnlyScalars_) { auto executor = arrow::compute::detail::KernelExecutor::MakeScalar(); - ARROW_OK(executor->Init(&kernelContext, { &Kernel_, ArgsValuesDescr_, nullptr })); + ARROW_OK(executor->Init(&kernelContext, {&Kernel_, ArgsValuesDescr_, nullptr})); auto listener = std::make_shared<arrow::compute::detail::DatumAccumulator>(); ARROW_OK(executor->Execute(argDatums, listener.get())); @@ -182,7 +182,7 @@ public: while (dechunker.Next(chunk)) { auto executor = arrow::compute::detail::KernelExecutor::MakeScalar(); - ARROW_OK(executor->Init(&kernelContext, { &Kernel_, ArgsValuesDescr_, nullptr })); + ARROW_OK(executor->Init(&kernelContext, {&Kernel_, ArgsValuesDescr_, nullptr})); arrow::compute::detail::DatumAccumulator listener; ARROW_OK(executor->Execute(chunk, &listener)); @@ -240,7 +240,7 @@ private: }; inline void SetCallableArgumentAttributes(IFunctionArgTypesBuilder& argsBuilder, - const TCallableTypeInspector& callableInspector, const ui32 index) { + const TCallableTypeInspector& callableInspector, const ui32 index) { if (callableInspector.GetArgumentName(index).Size() > 0) { argsBuilder.Name(callableInspector.GetArgumentName(index)); } @@ -250,7 +250,7 @@ inline void SetCallableArgumentAttributes(IFunctionArgTypesBuilder& argsBuilder, } inline void PrepareSimpleArrowUdf(IFunctionTypeInfoBuilder& builder, TType* signature, TType* userType, TExec exec, bool typesOnly, - const TString& name, arrow::compute::NullHandling::type nullHandling = arrow::compute::NullHandling::type::COMPUTED_NO_PREALLOCATE) { + const TString& name, arrow::compute::NullHandling::type nullHandling = arrow::compute::NullHandling::type::COMPUTED_NO_PREALLOCATE) { auto typeInfoHelper = builder.TypeInfoHelper(); TCallableTypeInspector callableInspector(*typeInfoHelper, signature); Y_ENSURE(callableInspector); @@ -322,11 +322,11 @@ inline void PrepareSimpleArrowUdf(IFunctionTypeInfoBuilder& builder, TType* sign if (!typesOnly) { builder.Implementation(new TSimpleArrowUdfImpl(argBlockTypes, callableInspector.GetReturnType(), - onlyScalars, exec, builder, name, nullHandling)); + onlyScalars, exec, builder, name, nullHandling)); } } -template<typename TBuilder> +template <typename TBuilder> TBuilder* CastToArrayBuilderImpl(IArrayBuilder& builder) { static_assert(std::is_base_of_v<IArrayBuilder, TBuilder>); @@ -335,7 +335,7 @@ TBuilder* CastToArrayBuilderImpl(IArrayBuilder& builder) { return builderImpl; } -template<typename TScalarBuilderImpl> +template <typename TScalarBuilderImpl> TScalarBuilderImpl* CastToScalarBuilderImpl(IScalarBuilder& builder) { static_assert(std::is_base_of_v<IScalarBuilder, TScalarBuilderImpl>); @@ -344,7 +344,7 @@ TScalarBuilderImpl* CastToScalarBuilderImpl(IScalarBuilder& builder) { return builderImpl; } -template<typename TReader> +template <typename TReader> TReader* CastToBlockReaderImpl(IBlockReader& reader) { static_assert(std::is_base_of_v<IBlockReader, TReader>); @@ -355,7 +355,6 @@ TReader* CastToBlockReaderImpl(IBlockReader& reader) { template <typename TDerived, typename TReader = IBlockReader, typename TArrayBuilderImpl = IArrayBuilder, typename TScalarBuilderImpl = IScalarBuilder> struct TUnaryKernelExec { - static arrow::Status Do(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { auto& state = dynamic_cast<TUdfKernelState&>(*ctx->state()); auto& reader = state.GetReader(0); @@ -370,8 +369,7 @@ struct TUnaryKernelExec { TDerived::Process(&state.GetValueBuilder(), item, [&](TBlockItem out) { *res = builderImpl->Build(out); }); - } - else { + } else { auto& array = *arg.array(); auto& builder = state.GetArrayBuilder(); auto* builderImpl = CastToArrayBuilderImpl<TArrayBuilderImpl>(builder); @@ -420,8 +418,7 @@ struct TBinaryKernelExec { TDerived::Process(&state.GetValueBuilder(), item1, item2, [&](TBlockItem out) { *res = builderImpl->Build(out); }); - } - else if (arg1.is_scalar() && arg2.is_array()) { + } else if (arg1.is_scalar() && arg2.is_array()) { auto item1 = reader1Impl->GetScalarItem(*arg1.scalar()); auto& array2 = *arg2.array(); auto& builder = state.GetArrayBuilder(); @@ -580,7 +577,7 @@ struct TGenericKernelExec { } }; -template <typename TInput, typename TOutput, TOutput(*Core)(TInput)> +template <typename TInput, typename TOutput, TOutput (*Core)(TInput)> arrow::Status UnaryPreallocatedExecImpl(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { Y_UNUSED(ctx); auto& inArray = batch.values[0].array(); @@ -595,8 +592,7 @@ arrow::Status UnaryPreallocatedExecImpl(arrow::compute::KernelContext* ctx, cons return arrow::Status::OK(); } - -template <typename TReader, typename TOutput, TOutput(*Core)(TBlockItem)> +template <typename TReader, typename TOutput, TOutput (*Core)(TBlockItem)> arrow::Status UnaryPreallocatedReaderExecImpl(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { Y_UNUSED(ctx); static_assert(std::is_base_of_v<IBlockReader, TReader>); @@ -614,7 +610,7 @@ arrow::Status UnaryPreallocatedReaderExecImpl(arrow::compute::KernelContext* ctx return arrow::Status::OK(); } -template<typename TInput, typename TOutput, std::pair<TOutput, bool> Core(TInput)> +template <typename TInput, typename TOutput, std::pair<TOutput, bool> Core(TInput)> struct TUnaryUnsafeFixedSizeFilterKernel { static arrow::Status Do(arrow::compute::KernelContext* ctx, const arrow::compute::ExecBatch& batch, arrow::Datum* res) { static_assert(std::is_arithmetic<TInput>::value); @@ -656,9 +652,8 @@ struct TUnaryUnsafeFixedSizeFilterKernel { } }; - -template <typename TInput, typename TOutput, TOutput(*Core)(TInput)> -class TUnaryOverOptionalImpl : public TBoxedValue { +template <typename TInput, typename TOutput, TOutput (*Core)(TInput)> +class TUnaryOverOptionalImpl: public TBoxedValue { public: TUnboxedValue Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const final { Y_UNUSED(valueBuilder); @@ -670,77 +665,77 @@ public: } }; -} -} - -#define BEGIN_ARROW_UDF_IMPL(udfNameBlocks, signatureFunc, optArgc, isStrict) \ - class udfNameBlocks { \ - public: \ - typedef bool TTypeAwareMarker; \ - static const ::NYql::NUdf::TStringRef& Name() { \ - static auto name = ::NYql::NUdf::TStringRef::Of(#udfNameBlocks).Substring(1, 256); \ - return name; \ - } \ - static bool IsStrict() { \ - return isStrict; \ - } \ +} // namespace NUdf +} // namespace NYql + +#define BEGIN_ARROW_UDF_IMPL(udfNameBlocks, signatureFunc, optArgc, isStrict) \ + class udfNameBlocks { \ + public: \ + typedef bool TTypeAwareMarker; \ + static const ::NYql::NUdf::TStringRef& Name() { \ + static auto name = ::NYql::NUdf::TStringRef::Of(#udfNameBlocks).Substring(1, 256); \ + return name; \ + } \ + static bool IsStrict() { \ + return isStrict; \ + } \ static ::NYql::NUdf::TType* GetSignatureType(::NYql::NUdf::IFunctionTypeInfoBuilder& builder) { \ - return builder.SimpleSignatureType<signatureFunc>(optArgc); \ - } \ - static bool DeclareSignature(\ - const ::NYql::NUdf::TStringRef& name, \ - ::NYql::NUdf::TType* userType, \ - ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ - bool typesOnly); \ + return builder.SimpleSignatureType<signatureFunc>(optArgc); \ + } \ + static bool DeclareSignature( \ + const ::NYql::NUdf::TStringRef& name, \ + ::NYql::NUdf::TType* userType, \ + ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ + bool typesOnly); \ }; -#define BEGIN_SIMPLE_ARROW_UDF(udfName, signatureFunc) \ +#define BEGIN_SIMPLE_ARROW_UDF(udfName, signatureFunc) \ BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, 0, false) \ UDF_IMPL(udfName, builder.SimpleSignature<signatureFunc>().SupportsBlocks();, ;, ;, "", "", udfName##_BlocksImpl) -#define BEGIN_SIMPLE_STRICT_ARROW_UDF(udfName, signatureFunc) \ +#define BEGIN_SIMPLE_STRICT_ARROW_UDF(udfName, signatureFunc) \ BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, 0, true) \ UDF_IMPL(udfName, builder.SimpleSignature<signatureFunc>().SupportsBlocks().IsStrict();, ;, ;, "", "", udfName##_BlocksImpl) #define BEGIN_SIMPLE_STRICT_ARROW_UDF_OPTIONS(udfName, signatureFunc, options) \ - BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, 0, true) \ + BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, 0, true) \ UDF_IMPL(udfName, builder.SimpleSignature<signatureFunc>().SupportsBlocks().IsStrict(); options;, ;, ;, "", "", udfName##_BlocksImpl) #define BEGIN_SIMPLE_ARROW_UDF_WITH_OPTIONAL_ARGS(udfName, signatureFunc, optArgc) \ - BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, optArgc, false) \ + BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, optArgc, false) \ UDF_IMPL(udfName, builder.SimpleSignature<signatureFunc>().SupportsBlocks().OptionalArgs(optArgc);, ;, ;, "", "", udfName##_BlocksImpl) #define BEGIN_SIMPLE_STRICT_ARROW_UDF_WITH_OPTIONAL_ARGS(udfName, signatureFunc, optArgc) \ - BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, optArgc, true) \ + BEGIN_ARROW_UDF_IMPL(udfName##_BlocksImpl, signatureFunc, optArgc, true) \ UDF_IMPL(udfName, builder.SimpleSignature<signatureFunc>().SupportsBlocks().IsStrict().OptionalArgs(optArgc);, ;, ;, "", "", udfName##_BlocksImpl) -#define END_ARROW_UDF(udfNameBlocks, exec) \ - inline bool udfNameBlocks::DeclareSignature(\ - const ::NYql::NUdf::TStringRef& name, \ - ::NYql::NUdf::TType* userType, \ - ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ - bool typesOnly) { \ - if (Name() == name) { \ - if (IsStrict()) { \ - builder.IsStrict(); \ - } \ - PrepareSimpleArrowUdf(builder, GetSignatureType(builder), userType, exec, typesOnly, TString(name)); \ - return true; \ - } \ - return false; \ +#define END_ARROW_UDF(udfNameBlocks, exec) \ + inline bool udfNameBlocks::DeclareSignature( \ + const ::NYql::NUdf::TStringRef& name, \ + ::NYql::NUdf::TType* userType, \ + ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ + bool typesOnly) { \ + if (Name() == name) { \ + if (IsStrict()) { \ + builder.IsStrict(); \ + } \ + PrepareSimpleArrowUdf(builder, GetSignatureType(builder), userType, exec, typesOnly, TString(name)); \ + return true; \ + } \ + return false; \ } -#define END_ARROW_UDF_WITH_NULL_HANDLING(udfNameBlocks, exec, nullHandling) \ - inline bool udfNameBlocks::DeclareSignature(\ - const ::NYql::NUdf::TStringRef& name, \ - ::NYql::NUdf::TType* userType, \ - ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ - bool typesOnly) { \ - if (Name() == name) { \ - PrepareSimpleArrowUdf(builder, GetSignatureType(builder), userType, exec, typesOnly, TString(name), nullHandling); \ - return true; \ - } \ - return false; \ +#define END_ARROW_UDF_WITH_NULL_HANDLING(udfNameBlocks, exec, nullHandling) \ + inline bool udfNameBlocks::DeclareSignature( \ + const ::NYql::NUdf::TStringRef& name, \ + ::NYql::NUdf::TType* userType, \ + ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ + bool typesOnly) { \ + if (Name() == name) { \ + PrepareSimpleArrowUdf(builder, GetSignatureType(builder), userType, exec, typesOnly, TString(name), nullHandling); \ + return true; \ + } \ + return false; \ } #define END_SIMPLE_ARROW_UDF(udfName, exec) \ diff --git a/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp b/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp index cc389d3d94c..7c173be3651 100644 --- a/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp +++ b/yql/essentials/public/udf/arrow/ut/array_builder_ut.cpp @@ -34,352 +34,353 @@ struct TArrayBuilderTestData { std::unique_ptr<IArrayBuilder> MakeResourceArrayBuilder(TType* resourceType, TArrayBuilderTestData& data) { auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), resourceType, - *data.ArrowPool, MAX_BLOCK_SIZE, /* pgBuilder */nullptr); + *data.ArrowPool, MAX_BLOCK_SIZE, /* pgBuilder */ nullptr); UNIT_ASSERT_C(arrayBuilder, "Failed to make resource arrow array builder"); return arrayBuilder; } Y_UNIT_TEST_SUITE(TArrayBuilderTest) { - Y_UNIT_TEST(TestEmbeddedResourceBuilder) { - TArrayBuilderTestData data; - const auto resourceType = data.PgmBuilder.NewResourceType("Test.Resource"); - const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - auto resource = TUnboxedValuePod::Embedded("testtest"); - arrayBuilder->Add(resource); - auto datum = arrayBuilder->Build(true); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 1); +Y_UNIT_TEST(TestEmbeddedResourceBuilder) { + TArrayBuilderTestData data; + const auto resourceType = data.PgmBuilder.NewResourceType("Test.Resource"); + const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); + auto resource = TUnboxedValuePod::Embedded("testtest"); + arrayBuilder->Add(resource); + auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 1); + + auto value = datum.array()->GetValues<TUnboxedValue>(1)[0]; + UNIT_ASSERT(value.IsEmbedded()); + UNIT_ASSERT_VALUES_EQUAL_C(TStringRef(value.AsStringRef()), TStringRef(resource.AsStringRef()), + "Expected equal values after building array"); +} - auto value = datum.array()->GetValues<TUnboxedValue>(1)[0]; - UNIT_ASSERT(value.IsEmbedded()); - UNIT_ASSERT_VALUES_EQUAL_C(TStringRef(value.AsStringRef()), TStringRef(resource.AsStringRef()), - "Expected equal values after building array"); - } +Y_UNIT_TEST(TestTaggedTypeBuilder) { + TArrayBuilderTestData data; + const auto intType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::Int32, false); + const auto taggedType = data.PgmBuilder.NewTaggedType(intType, "tag"); - Y_UNIT_TEST(TestTaggedTypeBuilder) { - TArrayBuilderTestData data; - const auto intType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::Int32, false); - const auto taggedType = data.PgmBuilder.NewTaggedType(intType, "tag"); + const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), taggedType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); - const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), taggedType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); + TUnboxedValue testData = TUnboxedValuePod(123); - TUnboxedValue testData = TUnboxedValuePod(123); + arrayBuilder->Add(testData); - arrayBuilder->Add(testData); + auto datum = arrayBuilder->Build(true); - auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 1); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 1); + auto value = datum.array()->buffers[1]; - auto value = datum.array()->buffers[1]; + UNIT_ASSERT_VALUES_EQUAL(*reinterpret_cast<int32_t*>(value->address()), 123); +} - UNIT_ASSERT_VALUES_EQUAL(*reinterpret_cast<int32_t*>(value->address()), 123); - } +Y_UNIT_TEST(TestTaggedTypeReader) { + TArrayBuilderTestData data; + const auto intType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::Int32, false); + const auto taggedType = data.PgmBuilder.NewTaggedType(intType, "tag"); - Y_UNIT_TEST(TestTaggedTypeReader) { - TArrayBuilderTestData data; - const auto intType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::Int32, false); - const auto taggedType = data.PgmBuilder.NewTaggedType(intType, "tag"); + const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), taggedType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); - const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), taggedType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); + TUnboxedValue first = TUnboxedValuePod(123); + TUnboxedValue second = TUnboxedValuePod(456); - TUnboxedValue first = TUnboxedValuePod(123); - TUnboxedValue second = TUnboxedValuePod(456); + arrayBuilder->Add(first); + arrayBuilder->Add(second); - arrayBuilder->Add(first); - arrayBuilder->Add(second); + auto datum = arrayBuilder->Build(true); - auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), taggedType); - const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), taggedType); + const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); + const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); - const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); - const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); + UNIT_ASSERT_C(item1AfterRead.HasValue(), "Expected not null"); + UNIT_ASSERT_C(item2AfterRead.HasValue(), "Expected not null"); - UNIT_ASSERT_C(item1AfterRead.HasValue(), "Expected not null"); - UNIT_ASSERT_C(item2AfterRead.HasValue(), "Expected not null"); + UNIT_ASSERT_VALUES_EQUAL(item1AfterRead.Get<int>(), 123); + UNIT_ASSERT_VALUES_EQUAL(item2AfterRead.Get<int>(), 456); +} - UNIT_ASSERT_VALUES_EQUAL(item1AfterRead.Get<int>(), 123); - UNIT_ASSERT_VALUES_EQUAL(item2AfterRead.Get<int>(), 456); - } +extern const char ResourceName[] = "Resource.Name"; +Y_UNIT_TEST(TestDtorCall) { + TArrayBuilderTestData data; + const auto resourceType = data.PgmBuilder.NewResourceType("Test.Resource"); + const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - extern const char ResourceName[] = "Resource.Name"; - Y_UNIT_TEST(TestDtorCall) { - TArrayBuilderTestData data; - const auto resourceType = data.PgmBuilder.NewResourceType("Test.Resource"); - const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - - auto destructorCallsCnt = std::make_shared<int>(0); - - struct TWithDtor { - int Payload; - std::shared_ptr<int> DestructorCallsCnt; - TWithDtor(int payload, std::shared_ptr<int> destructorCallsCnt): - Payload(payload), DestructorCallsCnt(std::move(destructorCallsCnt)) { - } - ~TWithDtor() { - *DestructorCallsCnt = *DestructorCallsCnt + 1; - } - }; - int payload = 123; - - using TTestResource = TBoxedResource<std::shared_ptr<TWithDtor>, ResourceName>; - auto resourcePtr = std::make_shared<TWithDtor>(payload, destructorCallsCnt); - TUnboxedValuePod resource(new TTestResource(std::move(resourcePtr))); + auto destructorCallsCnt = std::make_shared<int>(0); + struct TWithDtor { + int Payload; + std::shared_ptr<int> DestructorCallsCnt; + TWithDtor(int payload, std::shared_ptr<int> destructorCallsCnt) + : Payload(payload) + , DestructorCallsCnt(std::move(destructorCallsCnt)) { - arrayBuilder->Add(resource); - auto datum = arrayBuilder->Build(true); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 1); - - const auto value = datum.array()->GetValues<TUnboxedValuePod>(1)[0]; - auto boxed = value.AsBoxed().Get(); - const auto resource = reinterpret_cast<TTestResource*>(boxed); - UNIT_ASSERT_VALUES_EQUAL(resource->Get()->get()->Payload, payload); } + ~TWithDtor() { + *DestructorCallsCnt = *DestructorCallsCnt + 1; + } + }; + int payload = 123; + + using TTestResource = TBoxedResource<std::shared_ptr<TWithDtor>, ResourceName>; + auto resourcePtr = std::make_shared<TWithDtor>(payload, destructorCallsCnt); + TUnboxedValuePod resource(new TTestResource(std::move(resourcePtr))); + + { + arrayBuilder->Add(resource); + auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 1); - UNIT_ASSERT_VALUES_EQUAL_C(*destructorCallsCnt, 1, "Expected 1 call to resource destructor"); + const auto value = datum.array()->GetValues<TUnboxedValuePod>(1)[0]; + auto boxed = value.AsBoxed().Get(); + const auto resource = reinterpret_cast<TTestResource*>(boxed); + UNIT_ASSERT_VALUES_EQUAL(resource->Get()->get()->Payload, payload); } - Y_UNIT_TEST(TestBoxedResourceNullable) { - TArrayBuilderTestData data; - const auto resourceType = data.PgmBuilder.NewOptionalType(data.PgmBuilder.NewResourceType("Test.Resource")); - const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - - struct TResourceItem { - int Payload; - }; - using TTestResource = TBoxedResource<TResourceItem, ResourceName>; - for (int i = 0; i < 4; i++) { - if ((i % 2) == 0) { - TUnboxedValuePod resource(new TTestResource(TResourceItem{i})); - arrayBuilder->Add(resource); - } else { - arrayBuilder->Add(TUnboxedValuePod{}); - } + UNIT_ASSERT_VALUES_EQUAL_C(*destructorCallsCnt, 1, "Expected 1 call to resource destructor"); +} + +Y_UNIT_TEST(TestBoxedResourceNullable) { + TArrayBuilderTestData data; + const auto resourceType = data.PgmBuilder.NewOptionalType(data.PgmBuilder.NewResourceType("Test.Resource")); + const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); + + struct TResourceItem { + int Payload; + }; + using TTestResource = TBoxedResource<TResourceItem, ResourceName>; + for (int i = 0; i < 4; i++) { + if ((i % 2) == 0) { + TUnboxedValuePod resource(new TTestResource(TResourceItem{i})); + arrayBuilder->Add(resource); + } else { + arrayBuilder->Add(TUnboxedValuePod{}); } - auto datum = arrayBuilder->Build(true); - const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); - for (int i = 0; i < 4; i++) { - if ((i % 2) == 0) { - auto item = blockReader->GetItem(*datum.array(), i); - UNIT_ASSERT_C(item.HasValue(), "Expected not null"); - auto* resourcePtr = reinterpret_cast<TTestResource*>(item.GetBoxed().Get()); - UNIT_ASSERT_EQUAL(i, resourcePtr->Get()->Payload); - } else { - auto item = blockReader->GetItem(*datum.array(), i); - UNIT_ASSERT(!item.HasValue()); - } + } + auto datum = arrayBuilder->Build(true); + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); + for (int i = 0; i < 4; i++) { + if ((i % 2) == 0) { + auto item = blockReader->GetItem(*datum.array(), i); + UNIT_ASSERT_C(item.HasValue(), "Expected not null"); + auto* resourcePtr = reinterpret_cast<TTestResource*>(item.GetBoxed().Get()); + UNIT_ASSERT_EQUAL(i, resourcePtr->Get()->Payload); + } else { + auto item = blockReader->GetItem(*datum.array(), i); + UNIT_ASSERT(!item.HasValue()); } } +} - Y_UNIT_TEST(TestBuilderWithReader) { - TArrayBuilderTestData data; - const auto resourceType = data.PgmBuilder.NewResourceType("Test.Resource"); - const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); +Y_UNIT_TEST(TestBuilderWithReader) { + TArrayBuilderTestData data; + const auto resourceType = data.PgmBuilder.NewResourceType("Test.Resource"); + const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - const auto item1 = TUnboxedValuePod::Embedded("1"); - arrayBuilder->Add(item1); - const auto item2 = TUnboxedValuePod::Embedded("22"); - arrayBuilder->Add(item2); + const auto item1 = TUnboxedValuePod::Embedded("1"); + arrayBuilder->Add(item1); + const auto item2 = TUnboxedValuePod::Embedded("22"); + arrayBuilder->Add(item2); - auto datum = arrayBuilder->Build(true); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); + auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); - const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); - const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); - const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); + const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); + const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); - UNIT_ASSERT_C(std::memcmp(item1.GetRawPtr(), item1AfterRead.GetRawPtr(), sizeof(TBlockItem)) == 0, "Expected UnboxedValue to equal to BlockItem"); - UNIT_ASSERT_C(std::memcmp(item2.GetRawPtr(), item2AfterRead.GetRawPtr(), sizeof(TBlockItem)) == 0, "Expected UnboxedValue to equal to BlockItem"); - } - - Y_UNIT_TEST(TestBoxedResourceReader) { - TArrayBuilderTestData data; - const auto resourceType = data.PgmBuilder.NewResourceType(ResourceName); - const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); + UNIT_ASSERT_C(std::memcmp(item1.GetRawPtr(), item1AfterRead.GetRawPtr(), sizeof(TBlockItem)) == 0, "Expected UnboxedValue to equal to BlockItem"); + UNIT_ASSERT_C(std::memcmp(item2.GetRawPtr(), item2AfterRead.GetRawPtr(), sizeof(TBlockItem)) == 0, "Expected UnboxedValue to equal to BlockItem"); +} - using TTestResource = TBoxedResource<int, ResourceName>; +Y_UNIT_TEST(TestBoxedResourceReader) { + TArrayBuilderTestData data; + const auto resourceType = data.PgmBuilder.NewResourceType(ResourceName); + const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - arrayBuilder->Add(TUnboxedValuePod(new TTestResource(11111111))); - arrayBuilder->Add(TUnboxedValuePod(new TTestResource(22222222))); - const auto datum = arrayBuilder->Build(true); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); + using TTestResource = TBoxedResource<int, ResourceName>; - const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); - const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); - const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); + arrayBuilder->Add(TUnboxedValuePod(new TTestResource(11111111))); + arrayBuilder->Add(TUnboxedValuePod(new TTestResource(22222222))); + const auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); - auto boxed1 = item1AfterRead.GetBoxed().Get(); - const auto resource1 = reinterpret_cast<TTestResource*>(boxed1); - UNIT_ASSERT_VALUES_EQUAL(*resource1->Get(), 11111111); - UNIT_ASSERT_VALUES_EQUAL(resource1->GetResourceTag(), ResourceName); + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); + const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); + const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); - auto boxed2 = item2AfterRead.GetBoxed().Get(); - const auto resource2 = reinterpret_cast<TTestResource*>(boxed2); - UNIT_ASSERT_VALUES_EQUAL(*resource2->Get(), 22222222); - UNIT_ASSERT_VALUES_EQUAL(resource2->GetResourceTag(), ResourceName); - } + auto boxed1 = item1AfterRead.GetBoxed().Get(); + const auto resource1 = reinterpret_cast<TTestResource*>(boxed1); + UNIT_ASSERT_VALUES_EQUAL(*resource1->Get(), 11111111); + UNIT_ASSERT_VALUES_EQUAL(resource1->GetResourceTag(), ResourceName); - Y_UNIT_TEST(TestTzDateBuilder_Layout) { - TArrayBuilderTestData data; - const auto tzDateType = data.PgmBuilder.NewDataType(EDataSlot::TzDate); - const NMiniKQL::TTypeInfoHelper typeInfoHelper; - const auto arrayBuilder = MakeArrayBuilder(typeInfoHelper, tzDateType, - *data.ArrowPool, MAX_BLOCK_SIZE, /* pgBuilder */ nullptr); - - auto makeTzDate = [] (ui16 val, ui16 tz) { - TUnboxedValuePod tzDate {val}; - tzDate.SetTimezoneId(tz); - return tzDate; - }; - - TVector<TUnboxedValuePod> dates{makeTzDate(1234, 1), makeTzDate(1234, 2), makeTzDate(45678, 333)}; - for (auto date: dates) { - arrayBuilder->Add(date); - } + auto boxed2 = item2AfterRead.GetBoxed().Get(); + const auto resource2 = reinterpret_cast<TTestResource*>(boxed2); + UNIT_ASSERT_VALUES_EQUAL(*resource2->Get(), 22222222); + UNIT_ASSERT_VALUES_EQUAL(resource2->GetResourceTag(), ResourceName); +} - const auto datum = arrayBuilder->Build(true); - UNIT_ASSERT(datum.is_array()); - const auto array = datum.array(); - const auto expectedType = GetArrowType(typeInfoHelper, tzDateType); - UNIT_ASSERT(array->type->Equals(expectedType)); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), dates.size()); - const auto childData = array->child_data; - UNIT_ASSERT_VALUES_EQUAL_C(childData.size(), 2, "Expected date and timezone children"); - UNIT_ASSERT(childData[0]->type->Equals(expectedType->field(0)->type())); - UNIT_ASSERT(childData[1]->type->Equals(expectedType->field(1)->type())); +Y_UNIT_TEST(TestTzDateBuilder_Layout) { + TArrayBuilderTestData data; + const auto tzDateType = data.PgmBuilder.NewDataType(EDataSlot::TzDate); + const NMiniKQL::TTypeInfoHelper typeInfoHelper; + const auto arrayBuilder = MakeArrayBuilder(typeInfoHelper, tzDateType, + *data.ArrowPool, MAX_BLOCK_SIZE, /* pgBuilder */ nullptr); + + auto makeTzDate = [](ui16 val, ui16 tz) { + TUnboxedValuePod tzDate{val}; + tzDate.SetTimezoneId(tz); + return tzDate; + }; + + TVector<TUnboxedValuePod> dates{makeTzDate(1234, 1), makeTzDate(1234, 2), makeTzDate(45678, 333)}; + for (auto date : dates) { + arrayBuilder->Add(date); } - Y_UNIT_TEST(TestResourceStringValueBuilderReader) { - TArrayBuilderTestData data; - const auto resourceType = data.PgmBuilder.NewResourceType(ResourceName); - const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); + const auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + const auto array = datum.array(); + const auto expectedType = GetArrowType(typeInfoHelper, tzDateType); + UNIT_ASSERT(array->type->Equals(expectedType)); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), dates.size()); + const auto childData = array->child_data; + UNIT_ASSERT_VALUES_EQUAL_C(childData.size(), 2, "Expected date and timezone children"); + UNIT_ASSERT(childData[0]->type->Equals(expectedType->field(0)->type())); + UNIT_ASSERT(childData[1]->type->Equals(expectedType->field(1)->type())); +} - arrayBuilder->Add(TUnboxedValuePod(TStringValue("test"))); - arrayBuilder->Add(TUnboxedValuePod(TStringValue("1234"), /* size */ 3, /* offset */ 1)); - const auto datum = arrayBuilder->Build(true); - UNIT_ASSERT(datum.is_array()); - UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); +Y_UNIT_TEST(TestResourceStringValueBuilderReader) { + TArrayBuilderTestData data; + const auto resourceType = data.PgmBuilder.NewResourceType(ResourceName); + const auto arrayBuilder = MakeResourceArrayBuilder(resourceType, data); - const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); - const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); - const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); + arrayBuilder->Add(TUnboxedValuePod(TStringValue("test"))); + arrayBuilder->Add(TUnboxedValuePod(TStringValue("1234"), /* size */ 3, /* offset */ 1)); + const auto datum = arrayBuilder->Build(true); + UNIT_ASSERT(datum.is_array()); + UNIT_ASSERT_VALUES_EQUAL(datum.length(), 2); - UNIT_ASSERT_VALUES_EQUAL(item1AfterRead.GetStringRefFromValue(), "test"); - UNIT_ASSERT_VALUES_EQUAL(item2AfterRead.GetStringRefFromValue(), "234"); - } + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), resourceType); + const auto item1AfterRead = blockReader->GetItem(*datum.array(), 0); + const auto item2AfterRead = blockReader->GetItem(*datum.array(), 1); - Y_UNIT_TEST(TestSingularTypeValueBuilderReader) { - TArrayBuilderTestData data; - const auto nullType = data.PgmBuilder.NewNullType(); + UNIT_ASSERT_VALUES_EQUAL(item1AfterRead.GetStringRefFromValue(), "test"); + UNIT_ASSERT_VALUES_EQUAL(item2AfterRead.GetStringRefFromValue(), "234"); +} - std::shared_ptr<arrow::ArrayData> arrayData = arrow::NullArray{42}.data(); - IArrayBuilder::TArrayDataItem arrayDataItem = {.Data = arrayData.get(), .StartOffset = 0}; - { - const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), nullType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); - // Check builder. - arrayBuilder->Add(TUnboxedValuePod::Zero()); - arrayBuilder->Add(TBlockItem::Zero()); - arrayBuilder->Add(TBlockItem::Zero(), 4); - TInputBuffer inputBuffer("Just arbitrary string"); - arrayBuilder->Add(inputBuffer); - UNIT_ASSERT_VALUES_EQUAL_C(inputBuffer.PopChar(), 'J', "The input buffer must not be consumed."); - arrayBuilder->AddMany(*arrayData, /*popCount=*/3u, /*sparseBitmat=*/nullptr, /*bitmapSize=*/arrayData->length); - arrayBuilder->AddMany(&arrayDataItem, /*arrayCount=*/1, /*beginIndex=*/1, /*count=*/3u); - std::vector<ui64> indexes = {1, 5, 7, 10}; - arrayBuilder->AddMany(&arrayDataItem, /*arrayCount=*/1, /*beginIndex=*/indexes.data(), /*count=*/4u); - UNIT_ASSERT_VALUES_EQUAL(arrayBuilder->Build(true).array()->length, 1 + 1 + 4 + 1 + 3 + 3 + 4); - } +Y_UNIT_TEST(TestSingularTypeValueBuilderReader) { + TArrayBuilderTestData data; + const auto nullType = data.PgmBuilder.NewNullType(); - { - // Check reader. - const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), nullType); - - UNIT_ASSERT(blockReader->GetItem(*arrayData, 0)); - UNIT_ASSERT(blockReader->GetScalarItem(arrow::Scalar(arrow::null()))); - UNIT_ASSERT_EQUAL(blockReader->GetDataWeight(*arrayData), 0); - UNIT_ASSERT_EQUAL(blockReader->GetDataWeight(TBlockItem::Zero()), 0); - UNIT_ASSERT_EQUAL(blockReader->GetDefaultValueWeight(), 0); - UNIT_ASSERT_EQUAL(blockReader->GetDefaultValueWeight(), 0); - - TOutputBuffer outputBuffer; - blockReader->SaveItem(*arrayData, 1, outputBuffer); - UNIT_ASSERT(outputBuffer.Finish().empty()); - blockReader->SaveScalarItem(arrow::Scalar(arrow::null()), outputBuffer); - UNIT_ASSERT(outputBuffer.Finish().empty()); - } + std::shared_ptr<arrow::ArrayData> arrayData = arrow::NullArray{42}.data(); + IArrayBuilder::TArrayDataItem arrayDataItem = {.Data = arrayData.get(), .StartOffset = 0}; + { + const auto arrayBuilder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), nullType, *data.ArrowPool, MAX_BLOCK_SIZE, /*pgBuilder=*/nullptr); + // Check builder. + arrayBuilder->Add(TUnboxedValuePod::Zero()); + arrayBuilder->Add(TBlockItem::Zero()); + arrayBuilder->Add(TBlockItem::Zero(), 4); + TInputBuffer inputBuffer("Just arbitrary string"); + arrayBuilder->Add(inputBuffer); + UNIT_ASSERT_VALUES_EQUAL_C(inputBuffer.PopChar(), 'J', "The input buffer must not be consumed."); + arrayBuilder->AddMany(*arrayData, /*popCount=*/3u, /*sparseBitmat=*/nullptr, /*bitmapSize=*/arrayData->length); + arrayBuilder->AddMany(&arrayDataItem, /*arrayCount=*/1, /*beginIndex=*/1, /*count=*/3u); + std::vector<ui64> indexes = {1, 5, 7, 10}; + arrayBuilder->AddMany(&arrayDataItem, /*arrayCount=*/1, /*beginIndex=*/indexes.data(), /*count=*/4u); + UNIT_ASSERT_VALUES_EQUAL(arrayBuilder->Build(true).array()->length, 1 + 1 + 4 + 1 + 3 + 3 + 4); } - Y_UNIT_TEST(TestBuilderAllocatedSize) { - TArrayBuilderTestData data; - const auto optStringType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::String, true); - const auto int64Type = data.PgmBuilder.NewDataType(NUdf::EDataSlot::Int64, false); - const auto structType = data.PgmBuilder.NewStructType({{ "a", optStringType }, { "b", int64Type }}); - const auto optStructType = data.PgmBuilder.NewOptionalType(structType); - const auto doubleOptStructType = data.PgmBuilder.NewOptionalType(optStructType); - - size_t itemSize = NMiniKQL::CalcMaxBlockItemSize(doubleOptStructType); - size_t blockLen = NMiniKQL::CalcBlockLen(itemSize); - Y_ENSURE(blockLen > 8); - - size_t bigStringSize = NMiniKQL::MaxBlockSizeInBytes / 8; - size_t hugeStringSize = NMiniKQL::MaxBlockSizeInBytes * 2; - - const TString bString(bigStringSize, 'a'); - TBlockItem strItem1(bString); - TBlockItem intItem1(1); - TBlockItem sItems1[] = { strItem1, intItem1 }; - TBlockItem sItem1(sItems1); - - const TBlockItem bigItem = sItem1.MakeOptional(); - - const TString hString(hugeStringSize, 'b'); - TBlockItem strItem2(hString); - TBlockItem intItem2(2); - TBlockItem sItems2[] = { strItem2, intItem2 }; - TBlockItem sItem2(sItems2); - - const TBlockItem hugeItem = sItem2.MakeOptional(); - - const size_t stringAllocStep = - arrow::BitUtil::RoundUpToMultipleOf64(blockLen + 1) + // String NullMask - arrow::BitUtil::RoundUpToMultipleOf64((blockLen + 1) * 4) + // String Offsets - NMiniKQL::MaxBlockSizeInBytes; // String Data - const size_t initialAllocated = - stringAllocStep + - arrow::BitUtil::RoundUpToMultipleOf64((blockLen + 1) * 8) + // Int64 Data - 2 * arrow::BitUtil::RoundUpToMultipleOf64(blockLen + 1); // Double Optional - - - size_t totalAllocated = 0; - auto builder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), doubleOptStructType, *data.ArrowPool, blockLen, nullptr, &totalAllocated); - UNIT_ASSERT_VALUES_EQUAL(totalAllocated, initialAllocated); - - for (ui32 i = 0; i < 8; ++i) { - builder->Add(bigItem); - } - UNIT_ASSERT_VALUES_EQUAL(totalAllocated, initialAllocated); - // string data block is fully used here + { + // Check reader. + const auto blockReader = MakeBlockReader(NMiniKQL::TTypeInfoHelper(), nullType); + + UNIT_ASSERT(blockReader->GetItem(*arrayData, 0)); + UNIT_ASSERT(blockReader->GetScalarItem(arrow::Scalar(arrow::null()))); + UNIT_ASSERT_EQUAL(blockReader->GetDataWeight(*arrayData), 0); + UNIT_ASSERT_EQUAL(blockReader->GetDataWeight(TBlockItem::Zero()), 0); + UNIT_ASSERT_EQUAL(blockReader->GetDefaultValueWeight(), 0); + UNIT_ASSERT_EQUAL(blockReader->GetDefaultValueWeight(), 0); + + TOutputBuffer outputBuffer; + blockReader->SaveItem(*arrayData, 1, outputBuffer); + UNIT_ASSERT(outputBuffer.Finish().empty()); + blockReader->SaveScalarItem(arrow::Scalar(arrow::null()), outputBuffer); + UNIT_ASSERT(outputBuffer.Finish().empty()); + } +} - size_t beforeBlockBoundary = totalAllocated; +Y_UNIT_TEST(TestBuilderAllocatedSize) { + TArrayBuilderTestData data; + const auto optStringType = data.PgmBuilder.NewDataType(NUdf::EDataSlot::String, true); + const auto int64Type = data.PgmBuilder.NewDataType(NUdf::EDataSlot::Int64, false); + const auto structType = data.PgmBuilder.NewStructType({{"a", optStringType}, {"b", int64Type}}); + const auto optStructType = data.PgmBuilder.NewOptionalType(structType); + const auto doubleOptStructType = data.PgmBuilder.NewOptionalType(optStructType); + + size_t itemSize = NMiniKQL::CalcMaxBlockItemSize(doubleOptStructType); + size_t blockLen = NMiniKQL::CalcBlockLen(itemSize); + Y_ENSURE(blockLen > 8); + + size_t bigStringSize = NMiniKQL::MaxBlockSizeInBytes / 8; + size_t hugeStringSize = NMiniKQL::MaxBlockSizeInBytes * 2; + + const TString bString(bigStringSize, 'a'); + TBlockItem strItem1(bString); + TBlockItem intItem1(1); + TBlockItem sItems1[] = {strItem1, intItem1}; + TBlockItem sItem1(sItems1); + + const TBlockItem bigItem = sItem1.MakeOptional(); + + const TString hString(hugeStringSize, 'b'); + TBlockItem strItem2(hString); + TBlockItem intItem2(2); + TBlockItem sItems2[] = {strItem2, intItem2}; + TBlockItem sItem2(sItems2); + + const TBlockItem hugeItem = sItem2.MakeOptional(); + + const size_t stringAllocStep = + arrow::BitUtil::RoundUpToMultipleOf64(blockLen + 1) + // String NullMask + arrow::BitUtil::RoundUpToMultipleOf64((blockLen + 1) * 4) + // String Offsets + NMiniKQL::MaxBlockSizeInBytes; // String Data + const size_t initialAllocated = + stringAllocStep + + arrow::BitUtil::RoundUpToMultipleOf64((blockLen + 1) * 8) + // Int64 Data + 2 * arrow::BitUtil::RoundUpToMultipleOf64(blockLen + 1); // Double Optional + + size_t totalAllocated = 0; + auto builder = MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), doubleOptStructType, *data.ArrowPool, blockLen, nullptr, &totalAllocated); + UNIT_ASSERT_VALUES_EQUAL(totalAllocated, initialAllocated); + + for (ui32 i = 0; i < 8; ++i) { builder->Add(bigItem); - UNIT_ASSERT_VALUES_EQUAL(totalAllocated, beforeBlockBoundary + stringAllocStep); + } + UNIT_ASSERT_VALUES_EQUAL(totalAllocated, initialAllocated); + // string data block is fully used here - // string data block is partially used - size_t beforeHugeString = totalAllocated; - builder->Add(hugeItem); - UNIT_ASSERT_VALUES_EQUAL(totalAllocated, beforeHugeString + stringAllocStep + hugeStringSize - NMiniKQL::MaxBlockSizeInBytes); + size_t beforeBlockBoundary = totalAllocated; + builder->Add(bigItem); + UNIT_ASSERT_VALUES_EQUAL(totalAllocated, beforeBlockBoundary + stringAllocStep); - totalAllocated = 0; - builder->Build(false); - UNIT_ASSERT_VALUES_EQUAL(totalAllocated, initialAllocated); - } + // string data block is partially used + size_t beforeHugeString = totalAllocated; + builder->Add(hugeItem); + UNIT_ASSERT_VALUES_EQUAL(totalAllocated, beforeHugeString + stringAllocStep + hugeStringSize - NMiniKQL::MaxBlockSizeInBytes); + + totalAllocated = 0; + builder->Build(false); + UNIT_ASSERT_VALUES_EQUAL(totalAllocated, initialAllocated); } +} // Y_UNIT_TEST_SUITE(TArrayBuilderTest) diff --git a/yql/essentials/public/udf/arrow/ut/bit_util_ut.cpp b/yql/essentials/public/udf/arrow/ut/bit_util_ut.cpp index dd95030c1b3..579c90d8b6a 100644 --- a/yql/essentials/public/udf/arrow/ut/bit_util_ut.cpp +++ b/yql/essentials/public/udf/arrow/ut/bit_util_ut.cpp @@ -8,37 +8,37 @@ using namespace NYql::NUdf; namespace { - void PerformTest(const ui8* testData, size_t offset, size_t length) { - std::vector<ui8> copied(arrow::BitUtil::BytesForBits(length) + 1, 0); - CopyDenseBitmap(copied.data(), testData, offset, length); - - std::vector<ui8> origSparse(length), copiedSparse(length); - DecompressToSparseBitmap(origSparse.data(), testData, offset, length); - DecompressToSparseBitmap(copiedSparse.data(), copied.data(), 0, length); - for (size_t i = 0; i < length; i++) { - UNIT_ASSERT_EQUAL_C(origSparse[i], copiedSparse[i], "Expected the same data"); - } +void PerformTest(const ui8* testData, size_t offset, size_t length) { + std::vector<ui8> copied(arrow::BitUtil::BytesForBits(length) + 1, 0); + CopyDenseBitmap(copied.data(), testData, offset, length); + + std::vector<ui8> origSparse(length), copiedSparse(length); + DecompressToSparseBitmap(origSparse.data(), testData, offset, length); + DecompressToSparseBitmap(copiedSparse.data(), copied.data(), 0, length); + for (size_t i = 0; i < length; i++) { + UNIT_ASSERT_EQUAL_C(origSparse[i], copiedSparse[i], "Expected the same data"); } } +} // namespace Y_UNIT_TEST_SUITE(CopyDenseBitmapTest) { - constexpr size_t testSize = 32; +constexpr size_t testSize = 32; - Y_UNIT_TEST(Test) { - SetRandomSeed(0); +Y_UNIT_TEST(Test) { + SetRandomSeed(0); - std::vector<ui8> testData(testSize); - for (size_t i = 0; i < testSize; i++) { - testData[i] = RandomNumber<ui8>(); - } + std::vector<ui8> testData(testSize); + for (size_t i = 0; i < testSize; i++) { + testData[i] = RandomNumber<ui8>(); + } - for (size_t offset = 0; offset < testSize * 8; offset++) { - for (size_t length = 0; length <= testSize * 8 - offset; length++) { - PerformTest(testData.data(), offset, length); - } + for (size_t offset = 0; offset < testSize * 8; offset++) { + for (size_t length = 0; length <= testSize * 8 - offset; length++) { + PerformTest(testData.data(), offset, length); } } } +} // Y_UNIT_TEST_SUITE(CopyDenseBitmapTest) Y_UNIT_TEST_SUITE(BitExpanding) { diff --git a/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp b/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp index 1ac4c88d65e..6d1fb8e21b9 100644 --- a/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp +++ b/yql/essentials/public/udf/arrow/ut/block_reader_ut.cpp @@ -13,15 +13,16 @@ namespace { using namespace NKikimr; -class TBlockReaderFixture : public NUnitTest::TBaseFixture { - class TArrayHelpers : public TThrRefBase { +class TBlockReaderFixture: public NUnitTest::TBaseFixture { + class TArrayHelpers: public TThrRefBase { public: using TPtr = TIntrusivePtr<TArrayHelpers>; explicit TArrayHelpers(const NMiniKQL::TType* type, arrow::MemoryPool* const arrowPool) : Builder(MakeArrayBuilder(NMiniKQL::TTypeInfoHelper(), type, *arrowPool, NMiniKQL::CalcBlockLen(CalcMaxBlockItemSize(type)), nullptr)) , Reader(MakeBlockReader(NMiniKQL::TTypeInfoHelper(), type)) - {} + { + } public: const std::unique_ptr<IArrayBuilder> Builder; @@ -35,7 +36,8 @@ public: , Env(Alloc) , PgmBuilder(Env, *FunctionRegistry) , ArrowPool(GetYqlMemoryPool()) - {} + { + } NMiniKQL::TType* OptionaType(NMiniKQL::TType* type) const { return PgmBuilder.NewOptionalType(type); @@ -67,67 +69,65 @@ public: arrow::MemoryPool* const ArrowPool; }; -} // anonymous namespace +} // anonymous namespace Y_UNIT_TEST_SUITE(BlockReaderTest) { - Y_UNIT_TEST_F(TestLogicalDataSize, TBlockReaderFixture) { - const std::vector arrayHelpers = { - GetArrayHelpers(DataType<ui32>()), - GetArrayHelpers(OptionaType(DataType<char*>())), - GetArrayHelpers(OptionaType(TupleType(OptionaType(DataType<ui32>()), DataType<char*>()))), - GetArrayHelpers(DataType(NUdf::EDataSlot::TzDate)), - GetArrayHelpers(PgmBuilder.NewNullType()) - }; - - constexpr ui32 size = 1000; - constexpr ui32 stringSize = 37; - for (ui32 i = 0; i < size; ++i) { - arrayHelpers[0]->Builder->Add(TBlockItem(i)); - - const auto str = NUnitTest::RandomString(stringSize, i); - arrayHelpers[1]->Builder->Add((i % 2) ? TBlockItem(str) : TBlockItem()); - - TBlockItem tuple[] = { ((i / 2) % 2) ? TBlockItem(i) : TBlockItem(), TBlockItem(str) }; - arrayHelpers[2]->Builder->Add((i % 2) ? TBlockItem(tuple) : TBlockItem()); - - TBlockItem tzDate(i); - tzDate.SetTimezoneId(i % 100); - arrayHelpers[3]->Builder->Add(tzDate); - arrayHelpers[4]->Builder->Add(TBlockItem::Zero()); - } +Y_UNIT_TEST_F(TestLogicalDataSize, TBlockReaderFixture) { + const std::vector arrayHelpers = { + GetArrayHelpers(DataType<ui32>()), + GetArrayHelpers(OptionaType(DataType<char*>())), + GetArrayHelpers(OptionaType(TupleType(OptionaType(DataType<ui32>()), DataType<char*>()))), + GetArrayHelpers(DataType(NUdf::EDataSlot::TzDate)), + GetArrayHelpers(PgmBuilder.NewNullType())}; + + constexpr ui32 size = 1000; + constexpr ui32 stringSize = 37; + for (ui32 i = 0; i < size; ++i) { + arrayHelpers[0]->Builder->Add(TBlockItem(i)); + + const auto str = NUnitTest::RandomString(stringSize, i); + arrayHelpers[1]->Builder->Add((i % 2) ? TBlockItem(str) : TBlockItem()); + + TBlockItem tuple[] = {((i / 2) % 2) ? TBlockItem(i) : TBlockItem(), TBlockItem(str)}; + arrayHelpers[2]->Builder->Add((i % 2) ? TBlockItem(tuple) : TBlockItem()); + + TBlockItem tzDate(i); + tzDate.SetTimezoneId(i % 100); + arrayHelpers[3]->Builder->Add(tzDate); + arrayHelpers[4]->Builder->Add(TBlockItem::Zero()); + } - std::vector<std::shared_ptr<arrow::ArrayData>> arrays; - arrays.reserve(arrayHelpers.size()); - for (const auto& helper : arrayHelpers) { - arrays.emplace_back(helper->Builder->Build(true).array()); - } + std::vector<std::shared_ptr<arrow::ArrayData>> arrays; + arrays.reserve(arrayHelpers.size()); + for (const auto& helper : arrayHelpers) { + arrays.emplace_back(helper->Builder->Build(true).array()); + } - constexpr ui32 offset = 133; - constexpr ui32 len = 533; - static_assert(offset + len < size); - - constexpr ui64 offsetSize = sizeof(arrow::BinaryType::offset_type) * len; - constexpr ui64 bitmaskSize = (len - 1) / 8 + 1; - constexpr ui64 nonEmptyStrings = (len - offset % 2) / 2 + offset % 2; - const std::vector<ui64> expectedLogicalSize = { - sizeof(ui32) * len, - bitmaskSize + offsetSize + stringSize * nonEmptyStrings, - 2 * bitmaskSize + offsetSize + sizeof(ui32) * len + stringSize * nonEmptyStrings, - (sizeof(ui16) + sizeof(ui16)) * len, - 0 - }; - - // Test GetDataWeight with offset and length - for (ui32 i = 0; i < arrayHelpers.size(); ++i) { - UNIT_ASSERT_VALUES_EQUAL_C(arrayHelpers[i]->Reader->GetSliceDataWeight(*arrays[i], offset, len), expectedLogicalSize[i], "array: " << i); - } + constexpr ui32 offset = 133; + constexpr ui32 len = 533; + static_assert(offset + len < size); + + constexpr ui64 offsetSize = sizeof(arrow::BinaryType::offset_type) * len; + constexpr ui64 bitmaskSize = (len - 1) / 8 + 1; + constexpr ui64 nonEmptyStrings = (len - offset % 2) / 2 + offset % 2; + const std::vector<ui64> expectedLogicalSize = { + sizeof(ui32) * len, + bitmaskSize + offsetSize + stringSize * nonEmptyStrings, + 2 * bitmaskSize + offsetSize + sizeof(ui32) * len + stringSize * nonEmptyStrings, + (sizeof(ui16) + sizeof(ui16)) * len, + 0}; + + // Test GetDataWeight with offset and length + for (ui32 i = 0; i < arrayHelpers.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(arrayHelpers[i]->Reader->GetSliceDataWeight(*arrays[i], offset, len), expectedLogicalSize[i], "array: " << i); + } - // Test GetDataWeight after slize - for (ui32 i = 0; i < arrayHelpers.size(); ++i) { - const auto slice = DeepSlice(arrays[i], offset, len); - UNIT_ASSERT_VALUES_EQUAL_C(arrayHelpers[i]->Reader->GetDataWeight(*slice), expectedLogicalSize[i], "sliced array: " << i); - } + // Test GetDataWeight after slize + for (ui32 i = 0; i < arrayHelpers.size(); ++i) { + const auto slice = DeepSlice(arrays[i], offset, len); + UNIT_ASSERT_VALUES_EQUAL_C(arrayHelpers[i]->Reader->GetDataWeight(*slice), expectedLogicalSize[i], "sliced array: " << i); } } +} // Y_UNIT_TEST_SUITE(BlockReaderTest) -} // namespace NYql::NUdf +} // namespace NYql::NUdf diff --git a/yql/essentials/public/udf/arrow/ut/ya.make b/yql/essentials/public/udf/arrow/ut/ya.make index 037deb20fc8..4f3e9e00b77 100644 --- a/yql/essentials/public/udf/arrow/ut/ya.make +++ b/yql/essentials/public/udf/arrow/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST() +ENABLE(YQL_STYLE_CPP) + SRCS( array_builder_ut.cpp bit_util_ut.cpp diff --git a/yql/essentials/public/udf/arrow/util.cpp b/yql/essentials/public/udf/arrow/util.cpp index 1fe4df77b31..60c3317a3ad 100644 --- a/yql/essentials/public/udf/arrow/util.cpp +++ b/yql/essentials/public/udf/arrow/util.cpp @@ -189,5 +189,5 @@ const TType* SkipTaggedType(const ITypeInfoHelper& typeInfoHelper, const TType* return type; } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/arrow/util.h b/yql/essentials/public/udf/arrow/util.h index bc3f79b4174..3f52ff0d209 100644 --- a/yql/essentials/public/udf/arrow/util.h +++ b/yql/essentials/public/udf/arrow/util.h @@ -55,7 +55,7 @@ ui64 GetSizeOfArrayDataInBytes(const arrow::ArrayData& data); ui64 GetSizeOfArrowBatchInBytes(const arrow::RecordBatch& batch); ui64 GetSizeOfArrowExecBatchInBytes(const arrow::compute::ExecBatch& batch); -class TResizeableBuffer : public arrow::ResizableBuffer { +class TResizeableBuffer: public arrow::ResizableBuffer { public: explicit TResizeableBuffer(arrow::MemoryPool* pool) : ResizableBuffer(nullptr, 0, arrow::CPUDevice::memory_manager(pool)) @@ -111,7 +111,7 @@ private: arrow::MemoryPool* Pool_; }; -template<typename TBuffer = TResizeableBuffer> +template <typename TBuffer = TResizeableBuffer> std::unique_ptr<arrow::ResizableBuffer> AllocateResizableBuffer(size_t capacity, arrow::MemoryPool* pool, bool zeroPad = false) { std::unique_ptr<TBuffer> result = std::make_unique<TBuffer>(pool); ARROW_OK(result->Reserve(capacity)); @@ -122,12 +122,15 @@ std::unique_ptr<arrow::ResizableBuffer> AllocateResizableBuffer(size_t capacity, } /// \brief owning buffer that calls destructors -template<typename T> -class TResizableManagedBuffer final : public TResizeableBuffer { +template <typename T> +class TResizableManagedBuffer final: public TResizeableBuffer { static_assert(!std::is_trivially_destructible_v<T>); + public: explicit TResizableManagedBuffer(arrow::MemoryPool* pool) - : TResizeableBuffer(pool) {} + : TResizeableBuffer(pool) + { + } ~TResizableManagedBuffer() override { for (int64_t i = 0; i < size_; i += sizeof(T)) { @@ -141,11 +144,12 @@ public: // 1) with UnsafeAdvance() method // 2) shrinkToFit = false // 3) doesn't zero pad buffer -template<typename T> +template <typename T> class TTypedBufferBuilder { static_assert(!std::is_same_v<T, bool>); using TArrowBuffer = std::conditional_t<std::is_trivially_destructible_v<T>, TResizeableBuffer, TResizableManagedBuffer<T>>; + public: explicit TTypedBufferBuilder(arrow::MemoryPool* pool, TMaybe<ui8> minFillPercentage = {}) : MinFillPercentage_(minFillPercentage) @@ -215,14 +219,15 @@ public: inline std::shared_ptr<arrow::Buffer> Finish() { int64_t newSize = Len_ * sizeof(T); bool shrinkToFit = MinFillPercentage_ - ? newSize <= Buffer_->capacity() * *MinFillPercentage_ / 100 - : false; + ? newSize <= Buffer_->capacity() * *MinFillPercentage_ / 100 + : false; ARROW_OK(Buffer_->Resize(newSize, shrinkToFit)); std::shared_ptr<arrow::ResizableBuffer> result; std::swap(result, Buffer_); Len_ = 0; return result; } + private: const TMaybe<ui8> MinFillPercentage_; arrow::MemoryPool* const Pool_; @@ -262,7 +267,7 @@ inline bool NeedWrapWithExternalOptional(const ITypeInfoHelper& typeInfoHelper, TOptionalTypeInspector typeOptOpt(typeInfoHelper, type); if (typeOptOpt) { return true; - } else if (TPgTypeInspector(typeInfoHelper, type) || IsSingularType(typeInfoHelper, type)) { + } else if (TPgTypeInspector(typeInfoHelper, type) || IsSingularType(typeInfoHelper, type)) { return true; } return false; diff --git a/yql/essentials/public/udf/arrow/ya.make b/yql/essentials/public/udf/arrow/ya.make index 3928b638522..88554de462d 100644 --- a/yql/essentials/public/udf/arrow/ya.make +++ b/yql/essentials/public/udf/arrow/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( args_dechunker.cpp udf_arrow_helpers.cpp diff --git a/yql/essentials/public/udf/sanitizer_utils.h b/yql/essentials/public/udf/sanitizer_utils.h index 60c7b71e3e6..22fa0965e2a 100644 --- a/yql/essentials/public/udf/sanitizer_utils.h +++ b/yql/essentials/public/udf/sanitizer_utils.h @@ -91,7 +91,7 @@ constexpr size_t GetSizeToAlloc(size_t size) { return 0; } return size + 2 * ALLOCATION_REDZONE_SIZE; -#else // defined(_asan_enabled_) || defined(_msan_enabled_) +#else // defined(_asan_enabled_) || defined(_msan_enabled_) return size; #endif // defined(_asan_enabled_) || defined(_msan_enabled_) } @@ -102,7 +102,7 @@ constexpr const void* GetOriginalAllocatedObject(const void* ptr, bool isZeroSiz return ptr; } return static_cast<const char*>(ptr) - ALLOCATION_REDZONE_SIZE; -#else // defined(_asan_enabled_) || defined(_msan_enabled_) +#else // defined(_asan_enabled_) || defined(_msan_enabled_) Y_UNUSED(isZeroSize); return ptr; #endif // defined(_asan_enabled_) || defined(_msan_enabled_) @@ -116,7 +116,7 @@ constexpr void* WrapPointerWithRedZones(void* ptr, size_t extendedSizeWithRedzon SanitizerMakeRegionInaccessible(ptr, extendedSizeWithRedzone); SanitizerMakeRegionAccessible(static_cast<char*>(ptr) + ALLOCATION_REDZONE_SIZE, extendedSizeWithRedzone - 2 * ALLOCATION_REDZONE_SIZE); return static_cast<char*>(ptr) + ALLOCATION_REDZONE_SIZE; -#else // defined(_asan_enabled_) || defined(_msan_enabled_) +#else // defined(_asan_enabled_) || defined(_msan_enabled_) Y_UNUSED(extendedSizeWithRedzone); return ptr; #endif // defined(_asan_enabled_) || defined(_msan_enabled_) @@ -130,7 +130,7 @@ constexpr const void* UnwrapPointerWithRedZones(const void* ptr, size_t size) { SanitizerMakeRegionInaccessible(static_cast<char*>(const_cast<void*>(ptr)) - ALLOCATION_REDZONE_SIZE, 2 * ALLOCATION_REDZONE_SIZE + size); return static_cast<const char*>(ptr) - ALLOCATION_REDZONE_SIZE; -#else // defined(_asan_enabled_) || defined(_msan_enabled_) +#else // defined(_asan_enabled_) || defined(_msan_enabled_) Y_UNUSED(size); return ptr; #endif // defined(_asan_enabled_) || defined(_msan_enabled_) diff --git a/yql/essentials/public/udf/service/exception_policy/udf_service.cpp b/yql/essentials/public/udf/service/exception_policy/udf_service.cpp index 56b954e9b82..2723529b890 100644 --- a/yql/essentials/public/udf/service/exception_policy/udf_service.cpp +++ b/yql/essentials/public/udf/service/exception_policy/udf_service.cpp @@ -30,11 +30,11 @@ extern "C" void UdfFreeWithSize(const void* mem, ui64 size) { return ::NKikimr::NMiniKQL::TWithDefaultMiniKQLAlloc::FreeWithSize(mem, size); } -extern "C" void* UdfArrowAllocate(ui64 size) { +extern "C" void* UdfArrowAllocate(ui64 size) { return ::NKikimr::NMiniKQL::MKQLArrowAllocate(size); } -extern "C" void* UdfArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { +extern "C" void* UdfArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { return ::NKikimr::NMiniKQL::MKQLArrowReallocate(mem, prevSize, size); } diff --git a/yql/essentials/public/udf/service/exception_policy/ya.make b/yql/essentials/public/udf/service/exception_policy/ya.make index b985d2d0f13..720fc9bd574 100644 --- a/yql/essentials/public/udf/service/exception_policy/ya.make +++ b/yql/essentials/public/udf/service/exception_policy/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + PROVIDES(YqlServicePolicy) SRCS( diff --git a/yql/essentials/public/udf/service/stub/udf_service.cpp b/yql/essentials/public/udf/service/stub/udf_service.cpp index 1cfb5fbe9ff..848119194b1 100644 --- a/yql/essentials/public/udf/service/stub/udf_service.cpp +++ b/yql/essentials/public/udf/service/stub/udf_service.cpp @@ -1,13 +1,33 @@ #include <util/system/yassert.h> #include <yql/essentials/public/udf/udf_value.h> -extern "C" void* UdfAllocate(ui64) { Y_ABORT("Called UdfAllocate"); } -extern "C" void UdfFree(const void*) { Y_ABORT("Called UdfFree"); } -extern "C" void UdfTerminate(const char*) { Y_ABORT("Called UdfTerminate."); } -extern "C" void UdfRegisterObject(::NYql::NUdf::TBoxedValue*) { Y_ABORT("Called UdfRegisterObject"); } -extern "C" void UdfUnregisterObject(::NYql::NUdf::TBoxedValue*) { Y_ABORT("Called UdfUnregisterObject"); } -extern "C" void* UdfAllocateWithSize(ui64) { Y_ABORT("Called UdfAllocateWithSize"); } -extern "C" void UdfFreeWithSize(const void*, ui64) { Y_ABORT("Called UdfFreeWithSize"); } -extern "C" void* UdfArrowAllocate(ui64) { Y_ABORT("Called UdfArrowAllocate"); } -extern "C" void* UdfArrowReallocate(const void*, ui64, ui64) { Y_ABORT("Called UdfArrowReallocate"); } -extern "C" void UdfArrowFree(const void*, ui64) { Y_ABORT("Called UdfArrowFree"); } +extern "C" void* UdfAllocate(ui64) { + Y_ABORT("Called UdfAllocate"); +} +extern "C" void UdfFree(const void*) { + Y_ABORT("Called UdfFree"); +} +extern "C" void UdfTerminate(const char*) { + Y_ABORT("Called UdfTerminate."); +} +extern "C" void UdfRegisterObject(::NYql::NUdf::TBoxedValue*) { + Y_ABORT("Called UdfRegisterObject"); +} +extern "C" void UdfUnregisterObject(::NYql::NUdf::TBoxedValue*) { + Y_ABORT("Called UdfUnregisterObject"); +} +extern "C" void* UdfAllocateWithSize(ui64) { + Y_ABORT("Called UdfAllocateWithSize"); +} +extern "C" void UdfFreeWithSize(const void*, ui64) { + Y_ABORT("Called UdfFreeWithSize"); +} +extern "C" void* UdfArrowAllocate(ui64) { + Y_ABORT("Called UdfArrowAllocate"); +} +extern "C" void* UdfArrowReallocate(const void*, ui64, ui64) { + Y_ABORT("Called UdfArrowReallocate"); +} +extern "C" void UdfArrowFree(const void*, ui64) { + Y_ABORT("Called UdfArrowFree"); +} diff --git a/yql/essentials/public/udf/service/stub/ya.make b/yql/essentials/public/udf/service/stub/ya.make index 94ebed75673..2c3248ff47d 100644 --- a/yql/essentials/public/udf/service/stub/ya.make +++ b/yql/essentials/public/udf/service/stub/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + PROVIDES(YqlServicePolicy) SRCS( diff --git a/yql/essentials/public/udf/service/terminate_policy/udf_service.cpp b/yql/essentials/public/udf/service/terminate_policy/udf_service.cpp index 69da98b1d85..9f801a9a4df 100644 --- a/yql/essentials/public/udf/service/terminate_policy/udf_service.cpp +++ b/yql/essentials/public/udf/service/terminate_policy/udf_service.cpp @@ -14,9 +14,11 @@ extern "C" [[noreturn]] void UdfTerminate(const char* message) { ::NKikimr::NMiniKQL::MKQLTerminate(message); } -extern "C" void UdfRegisterObject(::NYql::NUdf::TBoxedValue*) {} +extern "C" void UdfRegisterObject(::NYql::NUdf::TBoxedValue*) { +} -extern "C" void UdfUnregisterObject(::NYql::NUdf::TBoxedValue*) {} +extern "C" void UdfUnregisterObject(::NYql::NUdf::TBoxedValue*) { +} extern "C" void* UdfAllocateWithSize(ui64 size) { return ::NKikimr::NMiniKQL::TWithDefaultMiniKQLAlloc::AllocWithSize(size); @@ -26,11 +28,11 @@ extern "C" void UdfFreeWithSize(const void* mem, ui64 size) { return ::NKikimr::NMiniKQL::TWithDefaultMiniKQLAlloc::FreeWithSize(mem, size); } -extern "C" void* UdfArrowAllocate(ui64 size) { +extern "C" void* UdfArrowAllocate(ui64 size) { return ::NKikimr::NMiniKQL::MKQLArrowAllocate(size); } -extern "C" void* UdfArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { +extern "C" void* UdfArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { return ::NKikimr::NMiniKQL::MKQLArrowReallocate(mem, prevSize, size); } diff --git a/yql/essentials/public/udf/service/terminate_policy/ya.make b/yql/essentials/public/udf/service/terminate_policy/ya.make index b985d2d0f13..720fc9bd574 100644 --- a/yql/essentials/public/udf/service/terminate_policy/ya.make +++ b/yql/essentials/public/udf/service/terminate_policy/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + PROVIDES(YqlServicePolicy) SRCS( diff --git a/yql/essentials/public/udf/support/udf_support.cpp b/yql/essentials/public/udf/support/udf_support.cpp index 2d933ed755f..c357abdede1 100644 --- a/yql/essentials/public/udf/support/udf_support.cpp +++ b/yql/essentials/public/udf/support/udf_support.cpp @@ -2,9 +2,9 @@ #include <util/system/backtrace.h> #if defined(_win_) || defined(_darwin_) -#include <yql/essentials/public/udf/udf_registrator.h> + #include <yql/essentials/public/udf/udf_registrator.h> -#include <exception> + #include <exception> static NYql::NUdf::TStaticSymbols Symbols; @@ -29,7 +29,7 @@ extern "C" void UdfUnregisterObject(::NYql::NUdf::TBoxedValue* object) { return Symbols.UdfUnregisterObject(object); } -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) extern "C" void* UdfAllocateWithSize(ui64 size) { return Symbols.UdfAllocateWithSizeFunc(size); } @@ -37,21 +37,21 @@ extern "C" void* UdfAllocateWithSize(ui64 size) { extern "C" void UdfFreeWithSize(const void* mem, ui64 size) { return Symbols.UdfFreeWithSizeFunc(mem, size); } -#endif + #endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 37) -extern "C" void* UdfArrowAllocate(ui64 size) { + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 37) +extern "C" void* UdfArrowAllocate(ui64 size) { return Symbols.UdfArrowAllocateFunc(size); } -extern "C" void* UdfArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { +extern "C" void* UdfArrowReallocate(const void* mem, ui64 prevSize, ui64 size) { return Symbols.UdfArrowReallocateFunc(mem, prevSize, size); } extern "C" void UdfArrowFree(const void* mem, ui64 size) { return Symbols.UdfArrowFreeFunc(mem, size); } -#endif + #endif extern "C" void BindSymbols(const NYql::NUdf::TStaticSymbols& symbols) { Symbols = symbols; @@ -61,7 +61,7 @@ extern "C" void BindSymbols(const NYql::NUdf::TStaticSymbols& symbols) { namespace NYql { namespace NUdf { -typedef void(*TBackTraceCallback)(); +typedef void (*TBackTraceCallback)(); static TBackTraceCallback BackTraceCallback; static void UdfBackTraceFn(IOutputStream*, void* const*, size_t) { @@ -73,5 +73,5 @@ void SetBackTraceCallbackImpl(TBackTraceCallback callback) { SetFormatBackTraceFn(UdfBackTraceFn); } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/support/ya.make b/yql/essentials/public/udf/support/ya.make index 02c4197666d..6a1c75234d9 100644 --- a/yql/essentials/public/udf/support/ya.make +++ b/yql/essentials/public/udf/support/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( udf_support.cpp ) diff --git a/yql/essentials/public/udf/udf_allocator.h b/yql/essentials/public/udf/udf_allocator.h index 7818b49f10f..a0e08698225 100644 --- a/yql/essentials/public/udf/udf_allocator.h +++ b/yql/essentials/public/udf/udf_allocator.h @@ -25,8 +25,7 @@ namespace NYql { namespace NUdf { template <typename Type> -struct TStdAllocatorForUdf -{ +struct TStdAllocatorForUdf { typedef Type value_type; typedef Type* pointer; typedef const Type* const_pointer; @@ -38,10 +37,20 @@ struct TStdAllocatorForUdf TStdAllocatorForUdf() noexcept = default; ~TStdAllocatorForUdf() noexcept = default; - template<typename U> TStdAllocatorForUdf(const TStdAllocatorForUdf<U>&) noexcept {}; - template<typename U> struct rebind { typedef TStdAllocatorForUdf<U> other; }; // NOLINT(readability-identifier-naming) - template<typename U> bool operator==(const TStdAllocatorForUdf<U>&) const { return true; }; - template<typename U> bool operator!=(const TStdAllocatorForUdf<U>&) const { return false; } + template <typename U> + TStdAllocatorForUdf(const TStdAllocatorForUdf<U>&) noexcept {}; + template <typename U> + struct rebind { // NOLINT(readability-identifier-naming) + typedef TStdAllocatorForUdf<U> other; + }; + template <typename U> + bool operator==(const TStdAllocatorForUdf<U>&) const { + return true; + }; + template <typename U> + bool operator!=(const TStdAllocatorForUdf<U>&) const { + return false; + } static pointer allocate(size_type n, const void* = nullptr) // NOLINT(readability-identifier-naming) { @@ -82,19 +91,19 @@ struct TWithUdfAllocator { } #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) - void operator delete(void *mem, std::size_t sz) noexcept { + void operator delete(void* mem, std::size_t sz) noexcept { return UdfFreeWithSize(mem, sz); } - void operator delete[](void *mem, std::size_t sz) noexcept { + void operator delete[](void* mem, std::size_t sz) noexcept { return UdfFreeWithSize(mem, sz); } #else - void operator delete(void *mem) noexcept { + void operator delete(void* mem) noexcept { return UdfFree(mem); } - void operator delete[](void *mem) noexcept { + void operator delete[](void* mem) noexcept { return UdfFree(mem); } #endif @@ -102,4 +111,3 @@ struct TWithUdfAllocator { } // namespace NUdf } // namespace NYql - diff --git a/yql/essentials/public/udf/udf_counter.h b/yql/essentials/public/udf/udf_counter.h index 2350b597785..f924d582e5f 100644 --- a/yql/essentials/public/udf/udf_counter.h +++ b/yql/essentials/public/udf/udf_counter.h @@ -10,7 +10,8 @@ class TCounter { public: TCounter(i64* ptr = nullptr) : Ptr_(ptr) - {} + { + } void Inc() { if (Ptr_) { @@ -69,7 +70,8 @@ public: TScopedProbe(IScopedProbeHost* host = nullptr, void* cookie = nullptr) : Host_(host ? host : &NullHost_) , Cookie_(cookie) - {} + { + } void Acquire() { Host_->Acquire(Cookie_); @@ -80,7 +82,7 @@ public: } private: - class TNullHost : public IScopedProbeHost { + class TNullHost: public IScopedProbeHost { public: void Acquire(void* cookie) override { Y_UNUSED(cookie); @@ -109,5 +111,5 @@ public: UDF_ASSERT_TYPE_SIZE(ICountersProvider, 8); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_counter_ut.cpp b/yql/essentials/public/udf/udf_counter_ut.cpp index 22fd91d5f54..6e6b4430015 100644 --- a/yql/essentials/public/udf/udf_counter_ut.cpp +++ b/yql/essentials/public/udf/udf_counter_ut.cpp @@ -5,46 +5,46 @@ using namespace NYql::NUdf; Y_UNIT_TEST_SUITE(TUdfCounter) { - Y_UNIT_TEST(NullCounter) { - TCounter c; - c.Inc(); - c.Dec(); - c.Add(1); - c.Sub(2); - c.Set(3); - } +Y_UNIT_TEST(NullCounter) { + TCounter c; + c.Inc(); + c.Dec(); + c.Add(1); + c.Sub(2); + c.Set(3); +} - Y_UNIT_TEST(RealCounter) { - i64 value = 10; - TCounter c(&value); - c.Inc(); - UNIT_ASSERT_VALUES_EQUAL(value, 11); - c.Dec(); - UNIT_ASSERT_VALUES_EQUAL(value, 10); - c.Add(1); - UNIT_ASSERT_VALUES_EQUAL(value, 11); - c.Sub(2); - UNIT_ASSERT_VALUES_EQUAL(value, 9); - c.Set(3); - UNIT_ASSERT_VALUES_EQUAL(value, 3); - } +Y_UNIT_TEST(RealCounter) { + i64 value = 10; + TCounter c(&value); + c.Inc(); + UNIT_ASSERT_VALUES_EQUAL(value, 11); + c.Dec(); + UNIT_ASSERT_VALUES_EQUAL(value, 10); + c.Add(1); + UNIT_ASSERT_VALUES_EQUAL(value, 11); + c.Sub(2); + UNIT_ASSERT_VALUES_EQUAL(value, 9); + c.Set(3); + UNIT_ASSERT_VALUES_EQUAL(value, 3); +} - Y_UNIT_TEST(Copy) { - i64 value = 1; - TCounter c1(&value); - TCounter c2(c1); // copy ctor - c2.Inc(); - UNIT_ASSERT_VALUES_EQUAL(value, 2); - TCounter c3; - c3 = c1; // assign - c3.Inc(); - UNIT_ASSERT_VALUES_EQUAL(value, 3); - TCounter c4(std::move(c1)); // move ctor - c4.Inc(); - UNIT_ASSERT_VALUES_EQUAL(value, 4); - TCounter c5; - c5 = std::move(c3); // move assign - c5.Inc(); - UNIT_ASSERT_VALUES_EQUAL(value, 5); - } +Y_UNIT_TEST(Copy) { + i64 value = 1; + TCounter c1(&value); + TCounter c2(c1); // copy ctor + c2.Inc(); + UNIT_ASSERT_VALUES_EQUAL(value, 2); + TCounter c3; + c3 = c1; // assign + c3.Inc(); + UNIT_ASSERT_VALUES_EQUAL(value, 3); + TCounter c4(std::move(c1)); // move ctor + c4.Inc(); + UNIT_ASSERT_VALUES_EQUAL(value, 4); + TCounter c5; + c5 = std::move(c3); // move assign + c5.Inc(); + UNIT_ASSERT_VALUES_EQUAL(value, 5); } +} // Y_UNIT_TEST_SUITE(TUdfCounter) diff --git a/yql/essentials/public/udf/udf_data_type.cpp b/yql/essentials/public/udf/udf_data_type.cpp index 67f31354ce3..837f4b80c14 100644 --- a/yql/essentials/public/udf/udf_data_type.cpp +++ b/yql/essentials/public/udf/udf_data_type.cpp @@ -9,10 +9,14 @@ namespace { ui8 GetDecimalWidth(EDataTypeFeatures features, size_t size) { if (features & NUdf::IntegralType) { switch (size) { - case 1U: return 3U; - case 2U: return 5U; - case 4U: return 10U; - case 8U: return 20U; + case 1U: + return 3U; + case 2U: + return 5U; + case 4U: + return 10U; + case 8U: + return 20U; } } return 0U; @@ -27,54 +31,54 @@ ui8 GetDecimalWidth(EDataTypeFeatures features, size_t size) { static const std::array<std::array<std::optional<TCastResultOptions>, DataSlotCount>, DataSlotCount> CastResultsTable = {{ // Bool, Int8 ----integrals---- Uint64 Floats, Strings, YJsons, Uuid, DateTimes, Interval, TzDateTimes, Decimal, DyNumber, JsonDocument - {{ OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Bool + {{OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO}}, // Bool - {{ LD, OK, MF, OK, MF, OK, MF, OK, MF, OK, OK, OK, OK, NO, NO, NO, MF, MF, MF, OK, MF, MF, MF, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO }}, // Int8 - {{ LD, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, OK, OK, OK, OK, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO }}, // Uint8 - {{ LD, MF, MF, OK, MF, OK, MF, OK, MF, OK, OK, OK, OK, NO, NO, NO, MF, MF, MF, OK, MF, MF, MF, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO }}, // Int16 - {{ LD, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, MF, OK, OK, OK, MF, OK, OK, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO }}, // Uint16 - {{ LD, MF, MF, MF, MF, OK, MF, OK, MF, OK, LD, OK, OK, NO, NO, NO, MF, MF, MF, OK, MF, MF, MF, UN, NO, NO, MF, OK, OK, OK, NO, NO, NO }}, // Int32 - {{ LD, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, MF, MF, OK, OK, MF, MF, OK, UN, NO, NO, MF, OK, OK, OK, NO, NO, NO }}, // Uint32 - {{ LD, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, MF, MF, MF, MF, MF, MF, MF, UN, NO, NO, MF, MF, MF, MF, NO, NO, NO }}, // Int64 - {{ LD, MF, MF, MF, MF, MF, MF, MF, OK, LD, LD, OK, OK, NO, NO, NO, MF, MF, MF, MF, MF, MF, MF, UN, NO, NO, MF, MF, MF, MF, NO, NO, NO }}, // Uint64 + {{LD, OK, MF, OK, MF, OK, MF, OK, MF, OK, OK, OK, OK, NO, NO, NO, MF, MF, MF, OK, MF, MF, MF, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO}}, // Int8 + {{LD, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, OK, OK, OK, OK, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO}}, // Uint8 + {{LD, MF, MF, OK, MF, OK, MF, OK, MF, OK, OK, OK, OK, NO, NO, NO, MF, MF, MF, OK, MF, MF, MF, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO}}, // Int16 + {{LD, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, MF, OK, OK, OK, MF, OK, OK, UN, NO, NO, OK, OK, OK, OK, NO, NO, NO}}, // Uint16 + {{LD, MF, MF, MF, MF, OK, MF, OK, MF, OK, LD, OK, OK, NO, NO, NO, MF, MF, MF, OK, MF, MF, MF, UN, NO, NO, MF, OK, OK, OK, NO, NO, NO}}, // Int32 + {{LD, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, MF, MF, OK, OK, MF, MF, OK, UN, NO, NO, MF, OK, OK, OK, NO, NO, NO}}, // Uint32 + {{LD, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, MF, MF, MF, MF, MF, MF, MF, UN, NO, NO, MF, MF, MF, MF, NO, NO, NO}}, // Int64 + {{LD, MF, MF, MF, MF, MF, MF, MF, OK, LD, LD, OK, OK, NO, NO, NO, MF, MF, MF, MF, MF, MF, MF, UN, NO, NO, MF, MF, MF, MF, NO, NO, NO}}, // Uint64 - {{ FL, FL, FL, FL, FL, FL, FL, FL, FL, OK, LD, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Double - {{ FL, FL, FL, FL, FL, FL, FL, FL, FL, OK, OK, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Float + {{FL, FL, FL, FL, FL, FL, FL, FL, FL, OK, LD, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO}}, // Double + {{FL, FL, FL, FL, FL, FL, FL, FL, FL, OK, OK, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO}}, // Float - {{ MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, OK, MF, MF, MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, MF, MF, MF, MF, MF, MF, MF, MF }}, // String - {{ MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, OK, OK, MF, MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, MF, MF, MF, MF, MF, MF, MF, MF }}, // Utf8 + {{MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, OK, MF, MF, MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, MF, MF, MF, MF, MF, MF, MF, MF}}, // String + {{MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, OK, OK, MF, MF, MF, MF, MF, MF, MF, MF, MF, MF, FL, FL, MF, MF, MF, MF, MF, MF, MF, MF}}, // Utf8 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Yson - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO }}, // Json + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO}}, // Yson + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO}}, // Json - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Uuid + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO}}, // Uuid - {{ NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK }}, // Date - {{ NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK }}, // Datetime - {{ NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK }}, // Timestamp + {{NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK}}, // Date + {{NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK}}, // Datetime + {{NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK}}, // Timestamp - {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO }}, // Interval + {{NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO}}, // Interval - {{ NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK }}, // TzDate - {{ NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK }}, // TzDatetime - {{ NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK }}, // TzTimestamp + {{NO, MF, MF, MF, OK, OK, OK, OK, OK, OK, OK, OK, OK, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK}}, // TzDate + {{NO, MF, MF, MF, MF, MF, OK, OK, OK, OK, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK}}, // TzDatetime + {{NO, MF, MF, MF, MF, MF, MF, OK, OK, LD, LD, OK, OK, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK}}, // TzTimestamp - {{ NO, UN, UN, UN, UN, UN, UN, UN, UN, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, UN, NO, NO, NO, NO, NO, NO, NO, NO, NO }}, // Decimal - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO }}, // DyNumber - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO }}, // JsonDocument + {{NO, UN, UN, UN, UN, UN, UN, UN, UN, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, UN, NO, NO, NO, NO, NO, NO, NO, NO, NO}}, // Decimal + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO}}, // DyNumber + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, OK, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO, NO, NO, NO, NO}}, // JsonDocument - {{ NO, MF, MF, MF, MF, OK, MF, OK, MF, LD, OK, OK, OK, NO, NO, NO, MF, MF, MF, NO, MF, MF, MF, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK }}, // Date32 - {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, MF, MF, NO, FL, MF, MF, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK }}, // Datetime64 - {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, FL, MF, NO, FL, FL, MF, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK }}, // Timestamp64 + {{NO, MF, MF, MF, MF, OK, MF, OK, MF, LD, OK, OK, OK, NO, NO, NO, MF, MF, MF, NO, MF, MF, MF, NO, NO, NO, OK, OK, OK, NO, OK, OK, OK}}, // Date32 + {{NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, MF, MF, NO, FL, MF, MF, NO, NO, NO, LD, OK, OK, NO, LD, OK, OK}}, // Datetime64 + {{NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, FL, FL, MF, NO, FL, FL, MF, NO, NO, NO, LD, LD, OK, NO, LD, LD, OK}}, // Timestamp64 - {{ NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, MF, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO }}, // Interval64 + {{NO, MF, MF, MF, MF, MF, MF, OK, MF, LD, LD, OK, OK, NO, NO, NO, NO, NO, NO, MF, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, NO, NO, NO}}, // Interval64 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, MF, MF, MF, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK }}, // TzDate32 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, FL, MF, MF, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK }}, // TzDatetime64 - {{ NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, FL, FL, MF, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK }}, // TzTimestamp64 + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, MF, MF, MF, NO, NO, NO, LD, LD, LD, NO, OK, OK, OK}}, // TzDate32 + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, FL, MF, MF, NO, NO, NO, LD, LD, LD, NO, LD, OK, OK}}, // TzDatetime64 + {{NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, NO, OK, OK, NO, NO, NO, FL, FL, FL, NO, FL, FL, MF, NO, NO, NO, LD, LD, LD, NO, LD, LD, OK}}, // TzTimestamp64 }}; -} +} // namespace TMaybe<TCastResultOptions> GetCastResult(EDataSlot source, EDataSlot target) { if (const auto& r = CastResultsTable[static_cast<size_t>(source)][static_cast<size_t>(target)]) { @@ -84,16 +88,17 @@ TMaybe<TCastResultOptions> GetCastResult(EDataSlot source, EDataSlot target) { } #define UDF_TYPE_ID_CHECK(xName, xId, xUnused2, xUnused3, xUnused4, xUnused5) \ - case xId: return EDataSlot::xName; + case xId: \ + return EDataSlot::xName; #define UDF_TYPE_PARSE(xName, xUnused1, xUnused2, xUnused3, xUnused4, xUnused5) \ - if (TStringBuf(#xName) == str) { \ - return EDataSlot::xName; \ + if (TStringBuf(#xName) == str) { \ + return EDataSlot::xName; \ } #define UDF_TYPE_INFO(xName, xId, xType, xFeatures, xLayoutType, xParamsCount) \ - { TStringBuf(#xName), xId, static_cast<EDataTypeFeatures>(xFeatures), \ - TPlainDataType<xType>::Result || TTzDataType<xType>::Result ? sizeof(xLayoutType) : 0, xParamsCount, GetDecimalWidth(static_cast<EDataTypeFeatures>(xFeatures), sizeof(xLayoutType)) }, + {TStringBuf(#xName), xId, static_cast<EDataTypeFeatures>(xFeatures), \ + TPlainDataType<xType>::Result || TTzDataType<xType>::Result ? sizeof(xLayoutType) : 0, xParamsCount, GetDecimalWidth(static_cast<EDataTypeFeatures>(xFeatures), sizeof(xLayoutType))}, TMaybe<EDataSlot> FindDataSlot(TDataTypeId id) { switch (id) { @@ -122,8 +127,7 @@ EDataSlot GetDataSlot(TStringBuf str) { } const TDataTypeInfo DataTypeInfos[DataSlotCount] = { - UDF_TYPE_ID_MAP(UDF_TYPE_INFO) -}; + UDF_TYPE_ID_MAP(UDF_TYPE_INFO)}; bool IsComparable(EDataSlot left, EDataSlot right) { const auto lf = GetDataTypeInfo(left).Features; diff --git a/yql/essentials/public/udf/udf_data_type.h b/yql/essentials/public/udf/udf_data_type.h index 3c5fd264a9d..4dda1c0ae89 100644 --- a/yql/essentials/public/udf/udf_data_type.h +++ b/yql/essentials/public/udf/udf_data_type.h @@ -15,7 +15,7 @@ using TDataTypeId = ui16; using TTimezoneId = ui16; -enum EDataTypeFeatures : ui32 { +enum EDataTypeFeatures: ui32 { CanCompare = 1u << 0, HasDeterministicCompare = 1u << 1, CanEquate = 1u << 2, @@ -30,9 +30,9 @@ enum EDataTypeFeatures : ui32 { AllowFromString = 1u << 11, PayloadType = AllowToBytes | AllowFromBytes | AllowToString | AllowFromString | - HasDeterministicToString | HasDeterministicToBytes, + HasDeterministicToString | HasDeterministicToBytes, CommonType = PayloadType | CanCompare | HasDeterministicCompare | CanEquate | HasDeterministicEquals | - CanHash | HasDeterministicHash, + CanHash | HasDeterministicHash, UnknownType = 0, StringType = 1u << 20, @@ -52,82 +52,85 @@ template <typename T> struct TDataType; template <typename T> -struct TKnownDataType -{ +struct TKnownDataType { static constexpr bool Result = (TDataType<T>::Id != 0); }; template <typename T> -struct TPlainDataType -{ +struct TPlainDataType { static constexpr bool Result = (TDataType<T>::Features & (NumericType | DateType | TimeIntervalType)) != 0 || std::is_same<T, bool>::value; }; template <typename T> -struct TTzDataType -{ +struct TTzDataType { static constexpr bool Result = (TDataType<T>::Features & (TzDateType)) != 0; }; -#define INTEGRAL_VALUE_TYPES(xx) \ - xx(i8) \ - xx(ui8) \ - xx(i16) \ - xx(ui16) \ - xx(i32) \ - xx(ui32) \ - xx(i64) \ +// clang-format off +#define INTEGRAL_VALUE_TYPES(xx) \ + xx(i8) \ + xx(ui8) \ + xx(i16) \ + xx(ui16) \ + xx(i32) \ + xx(ui32) \ + xx(i64) \ xx(ui64) +// clang-format on +// clang-format off #define PRIMITIVE_VALUE_TYPES(xx) \ - INTEGRAL_VALUE_TYPES(xx) \ - xx(float) \ + INTEGRAL_VALUE_TYPES(xx) \ + xx(float) \ xx(double) - -#define KNOWN_PRIMITIVE_VALUE_TYPES(xx) \ - xx(bool) \ - xx(i8) \ - xx(ui8) \ - xx(i16) \ - xx(ui16) \ - xx(i32) \ - xx(ui32) \ - xx(i64) \ - xx(ui64) \ - xx(float) \ +// clang-format on + +// clang-format off +#define KNOWN_PRIMITIVE_VALUE_TYPES(xx) \ + xx(bool) \ + xx(i8) \ + xx(ui8) \ + xx(i16) \ + xx(ui16) \ + xx(i32) \ + xx(ui32) \ + xx(i64) \ + xx(ui64) \ + xx(float) \ xx(double) - -#define KNOWN_FIXED_VALUE_TYPES(xx) \ - xx(bool, bool) \ - xx(i8, i8) \ - xx(ui8, ui8) \ - xx(i16, i16) \ - xx(ui16, ui16) \ - xx(i32, i32) \ - xx(ui32, ui32) \ - xx(i64, i64) \ - xx(ui64, ui64) \ - xx(float, float) \ - xx(double, double) \ - xx(NUdf::TDate, ui16) \ - xx(NUdf::TDatetime, ui32) \ - xx(NUdf::TTimestamp, ui64) \ - xx(NUdf::TInterval, i64) \ - xx(NUdf::TDate32, i32) \ - xx(NUdf::TDatetime64, i64) \ - xx(NUdf::TTimestamp64, i64) \ +// clang-format on + +// clang-format off +#define KNOWN_FIXED_VALUE_TYPES(xx) \ + xx(bool, bool) \ + xx(i8, i8) \ + xx(ui8, ui8) \ + xx(i16, i16) \ + xx(ui16, ui16) \ + xx(i32, i32) \ + xx(ui32, ui32) \ + xx(i64, i64) \ + xx(ui64, ui64) \ + xx(float, float) \ + xx(double, double) \ + xx(NUdf::TDate, ui16) \ + xx(NUdf::TDatetime, ui32) \ + xx(NUdf::TTimestamp, ui64) \ + xx(NUdf::TInterval, i64) \ + xx(NUdf::TDate32, i32) \ + xx(NUdf::TDatetime64, i64) \ + xx(NUdf::TTimestamp64, i64) \ xx(NUdf::TInterval64, i64) +// clang-format on template <typename T> -struct TPrimitiveDataType -{ +struct TPrimitiveDataType { static constexpr bool Result = false; }; -#define UDF_PRIMITIVE_TYPE_IMPL(type) \ - template <> \ - struct TPrimitiveDataType<type> \ - { \ +#define UDF_PRIMITIVE_TYPE_IMPL(type) \ + template <> \ + struct TPrimitiveDataType<type> { \ static constexpr bool Result = true; \ }; @@ -171,43 +174,43 @@ constexpr i64 MAX_DATETIME64 = 4611669811199ll; constexpr i64 MAX_TIMESTAMP64 = 4611669811199999999ll; constexpr i64 MAX_INTERVAL64 = MAX_TIMESTAMP64 - MIN_TIMESTAMP64; constexpr i32 MIN_YEAR32 = -144169; // inclusive -constexpr i32 MAX_YEAR32 = 148108; // non-inclusive +constexpr i32 MAX_YEAR32 = 148108; // non-inclusive constexpr size_t UUID_SIZE = 16; -#define UDF_TYPE_ID_MAP(XX) \ - XX(Bool, NYql::NProto::Bool, bool, CommonType, bool, 0) \ - XX(Int8, NYql::NProto::Int8, i8, CommonType | NumericType | IntegralType | SignedIntegralType, i8, 0) \ - XX(Uint8, NYql::NProto::Uint8, ui8, CommonType | NumericType | IntegralType | UnsignedIntegralType, ui8, 0) \ - XX(Int16, NYql::NProto::Int16, i16, CommonType | NumericType | IntegralType | SignedIntegralType, i16, 0) \ +#define UDF_TYPE_ID_MAP(XX) \ + XX(Bool, NYql::NProto::Bool, bool, CommonType, bool, 0) \ + XX(Int8, NYql::NProto::Int8, i8, CommonType | NumericType | IntegralType | SignedIntegralType, i8, 0) \ + XX(Uint8, NYql::NProto::Uint8, ui8, CommonType | NumericType | IntegralType | UnsignedIntegralType, ui8, 0) \ + XX(Int16, NYql::NProto::Int16, i16, CommonType | NumericType | IntegralType | SignedIntegralType, i16, 0) \ XX(Uint16, NYql::NProto::Uint16, ui16, CommonType | NumericType | IntegralType | UnsignedIntegralType, ui16, 0) \ - XX(Int32, NYql::NProto::Int32, i32, CommonType | NumericType | IntegralType | SignedIntegralType, i32, 0) \ + XX(Int32, NYql::NProto::Int32, i32, CommonType | NumericType | IntegralType | SignedIntegralType, i32, 0) \ XX(Uint32, NYql::NProto::Uint32, ui32, CommonType | NumericType | IntegralType | UnsignedIntegralType, ui32, 0) \ - XX(Int64, NYql::NProto::Int64, i64, CommonType | NumericType | IntegralType | SignedIntegralType, i64, 0) \ + XX(Int64, NYql::NProto::Int64, i64, CommonType | NumericType | IntegralType | SignedIntegralType, i64, 0) \ XX(Uint64, NYql::NProto::Uint64, ui64, CommonType | NumericType | IntegralType | UnsignedIntegralType, ui64, 0) \ - XX(Double, NYql::NProto::Double, double, CommonType | NumericType | FloatType, double, 0) \ - XX(Float, NYql::NProto::Float, float, CommonType | NumericType | FloatType, float, 0) \ - XX(String, NYql::NProto::String, char*, CommonType | StringType, char*, 0) \ - XX(Utf8, NYql::NProto::Utf8, TUtf8, CommonType | StringType, TUtf8, 0) \ - XX(Yson, NYql::NProto::Yson, TYson, PayloadType | StringType, TYson, 0) \ - XX(Json, NYql::NProto::Json, TJson, PayloadType | StringType, TJson, 0) \ - XX(Uuid, NYql::NProto::Uuid, TUuid, CommonType, TUuid, 0) \ - XX(Date, NYql::NProto::Date, TDate, CommonType | DateType, ui16, 0) \ - XX(Datetime, NYql::NProto::Datetime, TDatetime, CommonType | DateType, ui32, 0) \ - XX(Timestamp, NYql::NProto::Timestamp, TTimestamp, CommonType | DateType, ui64, 0) \ - XX(Interval, NYql::NProto::Interval, TInterval, CommonType | TimeIntervalType, i64, 0) \ - XX(TzDate, NYql::NProto::TzDate, TTzDate, CommonType | TzDateType, ui16, 0) \ - XX(TzDatetime, NYql::NProto::TzDatetime, TTzDatetime, CommonType | TzDateType, ui32, 0) \ - XX(TzTimestamp, NYql::NProto::TzTimestamp, TTzTimestamp, CommonType | TzDateType, ui64, 0) \ - XX(Decimal, NYql::NProto::Decimal, TDecimal, CommonType | DecimalType, TDecimal, 2) \ - XX(DyNumber, NYql::NProto::DyNumber, TDyNumber, CommonType, TDyNumber, 0) \ - XX(JsonDocument, NYql::NProto::JsonDocument, TJsonDocument, PayloadType, TJsonDocument, 0) \ - XX(Date32, NYql::NProto::Date32, TDate32, CommonType | DateType | ExtDateType, i32, 0) \ - XX(Datetime64, NYql::NProto::Datetime64, TDatetime64, CommonType | DateType | ExtDateType, i64, 0) \ - XX(Timestamp64, NYql::NProto::Timestamp64, TTimestamp64, CommonType | DateType | ExtDateType, i64, 0) \ - XX(Interval64, NYql::NProto::Interval64, TInterval64, CommonType | TimeIntervalType | ExtDateType, i64, 0) \ - XX(TzDate32, NYql::NProto::TzDate32, TTzDate32, CommonType | TzDateType | ExtDateType, i32, 0) \ - XX(TzDatetime64, NYql::NProto::TzDatetime64, TTzDatetime64, CommonType | TzDateType | ExtDateType, i64, 0) \ - XX(TzTimestamp64, NYql::NProto::TzTimestamp64, TTzTimestamp64, CommonType | TzDateType | ExtDateType, i64, 0) \ + XX(Double, NYql::NProto::Double, double, CommonType | NumericType | FloatType, double, 0) \ + XX(Float, NYql::NProto::Float, float, CommonType | NumericType | FloatType, float, 0) \ + XX(String, NYql::NProto::String, char*, CommonType | StringType, char*, 0) \ + XX(Utf8, NYql::NProto::Utf8, TUtf8, CommonType | StringType, TUtf8, 0) \ + XX(Yson, NYql::NProto::Yson, TYson, PayloadType | StringType, TYson, 0) \ + XX(Json, NYql::NProto::Json, TJson, PayloadType | StringType, TJson, 0) \ + XX(Uuid, NYql::NProto::Uuid, TUuid, CommonType, TUuid, 0) \ + XX(Date, NYql::NProto::Date, TDate, CommonType | DateType, ui16, 0) \ + XX(Datetime, NYql::NProto::Datetime, TDatetime, CommonType | DateType, ui32, 0) \ + XX(Timestamp, NYql::NProto::Timestamp, TTimestamp, CommonType | DateType, ui64, 0) \ + XX(Interval, NYql::NProto::Interval, TInterval, CommonType | TimeIntervalType, i64, 0) \ + XX(TzDate, NYql::NProto::TzDate, TTzDate, CommonType | TzDateType, ui16, 0) \ + XX(TzDatetime, NYql::NProto::TzDatetime, TTzDatetime, CommonType | TzDateType, ui32, 0) \ + XX(TzTimestamp, NYql::NProto::TzTimestamp, TTzTimestamp, CommonType | TzDateType, ui64, 0) \ + XX(Decimal, NYql::NProto::Decimal, TDecimal, CommonType | DecimalType, TDecimal, 2) \ + XX(DyNumber, NYql::NProto::DyNumber, TDyNumber, CommonType, TDyNumber, 0) \ + XX(JsonDocument, NYql::NProto::JsonDocument, TJsonDocument, PayloadType, TJsonDocument, 0) \ + XX(Date32, NYql::NProto::Date32, TDate32, CommonType | DateType | ExtDateType, i32, 0) \ + XX(Datetime64, NYql::NProto::Datetime64, TDatetime64, CommonType | DateType | ExtDateType, i64, 0) \ + XX(Timestamp64, NYql::NProto::Timestamp64, TTimestamp64, CommonType | DateType | ExtDateType, i64, 0) \ + XX(Interval64, NYql::NProto::Interval64, TInterval64, CommonType | TimeIntervalType | ExtDateType, i64, 0) \ + XX(TzDate32, NYql::NProto::TzDate32, TTzDate32, CommonType | TzDateType | ExtDateType, i32, 0) \ + XX(TzDatetime64, NYql::NProto::TzDatetime64, TTzDatetime64, CommonType | TzDateType | ExtDateType, i64, 0) \ + XX(TzTimestamp64, NYql::NProto::TzTimestamp64, TTzTimestamp64, CommonType | TzDateType | ExtDateType, i64, 0) #define UDF_TYPE_ID(xName, xTypeId, xType, xFeatures, xLayoutType, xParamsCount) \ template <> \ @@ -226,7 +229,9 @@ enum class EDataSlot { enum class ECountDataSlot { UDF_TYPE_ID_MAP(ENUM_VALUE_GEN_NO_VALUE) + // clang-format off Count + // clang-format on }; constexpr ui32 DataSlotCount = static_cast<ui32>(ECountDataSlot::Count); @@ -236,7 +241,7 @@ UDF_TYPE_ID(String, 0x1001, const char*, CommonType | StringType, const char*, 0 UDF_TYPE_ID(String, 0x1001, char* const, CommonType | StringType, char* const, 0) template <ui8 Precision, ui8 Scale> -struct TDecimalDataType : public TDataType<TDecimal> {}; +struct TDecimalDataType: public TDataType<TDecimal> {}; TMaybe<EDataSlot> FindDataSlot(TDataTypeId id); EDataSlot GetDataSlot(TDataTypeId id); // throws if data type is unknown @@ -261,7 +266,7 @@ inline const TDataTypeInfo& GetDataTypeInfo(EDataSlot slot) { using TCastResultOptions = ui8; -enum ECastOptions : TCastResultOptions { +enum ECastOptions: TCastResultOptions { Complete = 0U, MayFail = 1U << 0U, @@ -281,5 +286,5 @@ inline IOutputStream& operator<<(IOutputStream& os, EDataSlot slot) { return os; } -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_data_type_ut.cpp b/yql/essentials/public/udf/udf_data_type_ut.cpp index 77316f47b78..551df0054dd 100644 --- a/yql/essentials/public/udf/udf_data_type_ut.cpp +++ b/yql/essentials/public/udf/udf_data_type_ut.cpp @@ -5,12 +5,12 @@ using namespace NYql::NUdf; Y_UNIT_TEST_SUITE(TCastsTests) { - Y_UNIT_TEST(HasAllCasts) { - for (ui32 source = 0; source < DataSlotCount; ++source) { - for (ui32 target = 0; target < DataSlotCount; ++target) { - UNIT_ASSERT_C(GetCastResult((EDataSlot)source, (EDataSlot)target), - "Missing cast from " << (EDataSlot)source << " to " << (EDataSlot)target); - } +Y_UNIT_TEST(HasAllCasts) { + for (ui32 source = 0; source < DataSlotCount; ++source) { + for (ui32 target = 0; target < DataSlotCount; ++target) { + UNIT_ASSERT_C(GetCastResult((EDataSlot)source, (EDataSlot)target), + "Missing cast from " << (EDataSlot)source << " to " << (EDataSlot)target); } } } +} // Y_UNIT_TEST_SUITE(TCastsTests) diff --git a/yql/essentials/public/udf/udf_helpers.cpp b/yql/essentials/public/udf/udf_helpers.cpp index 91e2c1d779f..d1da39a7622 100644 --- a/yql/essentials/public/udf/udf_helpers.cpp +++ b/yql/essentials/public/udf/udf_helpers.cpp @@ -15,8 +15,9 @@ struct TLoadedResources { TString Get(TStringBuf resourceId) { const std::unique_lock lock(Sync); const auto ins = Resources.emplace(resourceId, ""); - if (ins.second) + if (ins.second) { ins.first->second = NResource::Find(resourceId); + } return ins.first->second; } @@ -24,11 +25,11 @@ struct TLoadedResources { std::unordered_map<TString, TString> Resources; }; -} +} // namespace TString LoadResourceOnce(TStringBuf resourceId) { return Singleton<TLoadedResources>()->Get(resourceId); } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_helpers.h b/yql/essentials/public/udf/udf_helpers.h index 84b0bd35662..23f25c16b96 100644 --- a/yql/essentials/public/udf/udf_helpers.h +++ b/yql/essentials/public/udf/udf_helpers.h @@ -14,77 +14,76 @@ #include <util/generic/strbuf.h> #include <util/string/builder.h> - namespace NYql { namespace NUdf { - template <class T> - concept CUDF = requires(const TStringRef& name, TType* userType, IFunctionTypeInfoBuilder& builder, bool typesOnly) { - { T::Name() } -> std::convertible_to<const TStringRef&>; - { T::DeclareSignature(name, userType, builder, typesOnly) } -> std::convertible_to<bool>; - }; +template <class T> +concept CUDF = requires(const TStringRef& name, TType* userType, IFunctionTypeInfoBuilder& builder, bool typesOnly) { + { T::Name() } -> std::convertible_to<const TStringRef&>; + { T::DeclareSignature(name, userType, builder, typesOnly) } -> std::convertible_to<bool>; +}; - template <ui32 V, CUDF TLegacyUDF, CUDF TActualUDF> - class TLangVerForked { - private: - Y_HAS_SUBTYPE(TBlockType); - - public: - using TTypeAwareMarker = bool; - - using TBlockType = decltype([] { - static_assert(THasTBlockType<TLegacyUDF>::value == THasTBlockType<TActualUDF>::value); - if constexpr (THasTBlockType<TActualUDF>::value) { - return TLangVerForked< - V, - typename TLegacyUDF::TBlockType, - typename TActualUDF::TBlockType>(); - } else { - return; - } - }()); +template <ui32 V, CUDF TLegacyUDF, CUDF TActualUDF> +class TLangVerForked { +private: + Y_HAS_SUBTYPE(TBlockType); - static const TStringRef& Name() { - Y_ENSURE(TLegacyUDF::Name() == TActualUDF::Name()); - return TActualUDF::Name(); +public: + using TTypeAwareMarker = bool; + + using TBlockType = decltype([] { + static_assert(THasTBlockType<TLegacyUDF>::value == THasTBlockType<TActualUDF>::value); + if constexpr (THasTBlockType<TActualUDF>::value) { + return TLangVerForked< + V, + typename TLegacyUDF::TBlockType, + typename TActualUDF::TBlockType>(); + } else { + return; } + }()); - static bool DeclareSignature( - const TStringRef& name, TType* userType, - IFunctionTypeInfoBuilder& builder, bool typesOnly) { + static const TStringRef& Name() { + Y_ENSURE(TLegacyUDF::Name() == TActualUDF::Name()); + return TActualUDF::Name(); + } + + static bool DeclareSignature( + const TStringRef& name, TType* userType, + IFunctionTypeInfoBuilder& builder, bool typesOnly) { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 43) - if (V <= builder.GetCurrentLangVer()) { - return TActualUDF::DeclareSignature(name, userType, builder, typesOnly); - } -#endif - return TLegacyUDF::DeclareSignature(name, userType, builder, typesOnly); + if (V <= builder.GetCurrentLangVer()) { + return TActualUDF::DeclareSignature(name, userType, builder, typesOnly); } - }; +#endif + return TLegacyUDF::DeclareSignature(name, userType, builder, typesOnly); + } +}; - inline TSourcePosition GetSourcePosition(IFunctionTypeInfoBuilder& builder) { +inline TSourcePosition GetSourcePosition(IFunctionTypeInfoBuilder& builder) { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 9) - return builder.GetSourcePosition(); + return builder.GetSourcePosition(); #else - Y_UNUSED(builder); - return {}; + Y_UNUSED(builder); + return {}; #endif - } +} - TString LoadResourceOnce(TStringBuf resourceId); +TString LoadResourceOnce(TStringBuf resourceId); - inline void SetIRImplementation(IFunctionTypeInfoBuilder& builder, TStringBuf resourceId, TStringBuf functionName) { +inline void SetIRImplementation(IFunctionTypeInfoBuilder& builder, TStringBuf resourceId, TStringBuf functionName) { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 16) - if (functionName) { - builder.IRImplementation(LoadResourceOnce(resourceId), resourceId, functionName); - } + if (functionName) { + builder.IRImplementation(LoadResourceOnce(resourceId), resourceId, functionName); + } #else - Y_UNUSED(builder); - Y_UNUSED(resourceId); - Y_UNUSED(functionName); + Y_UNUSED(builder); + Y_UNUSED(resourceId); + Y_UNUSED(functionName); #endif - } -} } +} // namespace NUdf +} // namespace NYql #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 17) #define APPEND_SOURCE_LOCATION(sb, valueBuilder, Pos_) sb << valueBuilder->WithCalleePosition(Pos_) << " "; @@ -93,131 +92,136 @@ namespace NUdf { #endif #define UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, blockType, create_impl) \ - class udfName: public ::NYql::NUdf::TBoxedValue { \ - public: \ - using TBlockType = blockType; \ - explicit udfName(::NYql::NUdf::IFunctionTypeInfoBuilder& builder) \ - : Pos_(GetSourcePosition(builder)) \ - { \ - init \ - } \ - static const ::NYql::NUdf::TStringRef& Name() { \ - static auto name = ::NYql::NUdf::TStringRef::Of(#udfName).Substring(1, 256); \ - return name; \ - } \ - inline ::NYql::NUdf::TUnboxedValue RunImpl( \ - const ::NYql::NUdf::IValueBuilder* valueBuilder, \ - const ::NYql::NUdf::TUnboxedValuePod* args) const; \ - ::NYql::NUdf::TUnboxedValue Run( \ - const ::NYql::NUdf::IValueBuilder* valueBuilder, \ - const ::NYql::NUdf::TUnboxedValuePod* args) const override { \ - try { \ - return RunImpl(valueBuilder, args); \ - } catch (const std::exception& ex) { \ - TStringBuilder sb; \ - APPEND_SOURCE_LOCATION(sb, valueBuilder, Pos_) \ - sb << ex.what(); \ - UdfTerminate(sb.c_str()); \ - } \ - } \ - static bool DeclareSignature( \ - const ::NYql::NUdf::TStringRef& name, \ - ::NYql::NUdf::TType* userType, \ - ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ - bool typesOnly) { \ - Y_UNUSED(userType); \ - if (Name() == name) { \ - typeBody if (!typesOnly) { \ - create_impl \ - SetIRImplementation(builder, irResourceId, irFunctionName); \ - } \ - return true; \ - } \ - return false; \ - } \ - ::NYql::NUdf::TSourcePosition GetPos() const { return Pos_; } \ - private: \ - ::NYql::NUdf::TSourcePosition Pos_; \ - members \ - }; \ - ::NYql::NUdf::TUnboxedValue udfName::RunImpl( \ - const ::NYql::NUdf::IValueBuilder* valueBuilder, \ + class udfName: public ::NYql::NUdf::TBoxedValue { \ + public: \ + using TBlockType = blockType; \ + explicit udfName(::NYql::NUdf::IFunctionTypeInfoBuilder& builder) \ + : Pos_(GetSourcePosition(builder)) \ + { \ + init \ + } \ + static const ::NYql::NUdf::TStringRef& Name() { \ + static auto name = ::NYql::NUdf::TStringRef::Of(#udfName).Substring(1, 256); \ + return name; \ + } \ + inline ::NYql::NUdf::TUnboxedValue RunImpl( \ + const ::NYql::NUdf::IValueBuilder* valueBuilder, \ + const ::NYql::NUdf::TUnboxedValuePod* args) const; \ + ::NYql::NUdf::TUnboxedValue Run( \ + const ::NYql::NUdf::IValueBuilder* valueBuilder, \ + const ::NYql::NUdf::TUnboxedValuePod* args) const override { \ + try { \ + return RunImpl(valueBuilder, args); \ + } catch (const std::exception& ex) { \ + TStringBuilder sb; \ + APPEND_SOURCE_LOCATION(sb, valueBuilder, Pos_) \ + sb << ex.what(); \ + UdfTerminate(sb.c_str()); \ + } \ + } \ + static bool DeclareSignature( \ + const ::NYql::NUdf::TStringRef& name, \ + ::NYql::NUdf::TType* userType, \ + ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ + bool typesOnly) { \ + Y_UNUSED(userType); \ + if (Name() == name) { \ + typeBody if (!typesOnly) { \ + create_impl \ + SetIRImplementation(builder, irResourceId, irFunctionName); \ + } \ + return true; \ + } \ + return false; \ + } \ + ::NYql::NUdf::TSourcePosition GetPos() const { \ + return Pos_; \ + } \ + \ + private: \ + ::NYql::NUdf::TSourcePosition Pos_; \ + members \ + }; \ + ::NYql::NUdf::TUnboxedValue udfName::RunImpl( \ + const ::NYql::NUdf::IValueBuilder* valueBuilder, \ const ::NYql::NUdf::TUnboxedValuePod* args) const #define UDF_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName, blockType) \ - UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, blockType, builder.Implementation(new udfName(builder));) + UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, blockType, builder.Implementation(new udfName(builder));) #define UDF(udfName, typeBody) UDF_IMPL(udfName, typeBody, ;, ;, "", "", void) -#define UDF_RUN_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \ - struct udfName##Members { \ - members \ - }; \ - \ +#define UDF_RUN_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \ + struct udfName##Members { \ + members \ + }; \ + \ class udfName: public ::NYql::NUdf::TBoxedValue, public udfName##Members { \ - public: \ + public: \ explicit udfName(::NYql::NUdf::IFunctionTypeInfoBuilder& builder) \ - : Pos_(GetSourcePosition(builder)) \ - { \ - init \ - } \ + : Pos_(GetSourcePosition(builder)) \ + { \ + init \ + } \ static const ::NYql::NUdf::TStringRef& Name() { \ static auto name = ::NYql::NUdf::TStringRef::Of(#udfName).Substring(1, 256); \ - return name; \ - } \ - class TImpl: public TBoxedValue, public udfName##Members { \ - public: \ - TImpl(const udfName##Members& parent, \ - const ::NYql::NUdf::TUnboxedValuePod& runConfig, \ - ::NYql::NUdf::TSourcePosition pos) \ - : udfName##Members(parent) \ + return name; \ + } \ + class TImpl: public TBoxedValue, public udfName##Members { \ + public: \ + TImpl(const udfName##Members& parent, \ + const ::NYql::NUdf::TUnboxedValuePod& runConfig, \ + ::NYql::NUdf::TSourcePosition pos) \ + : udfName##Members(parent) \ , RunConfig(::NYql::NUdf::TUnboxedValuePod(runConfig)) \ - , Pos_(pos) \ - {} \ + , Pos_(pos) \ + { \ + } \ inline ::NYql::NUdf::TUnboxedValue RunImpl( \ const ::NYql::NUdf::IValueBuilder* valueBuilder, \ const ::NYql::NUdf::TUnboxedValuePod* args) const; \ ::NYql::NUdf::TUnboxedValue Run( \ const ::NYql::NUdf::IValueBuilder* valueBuilder, \ const ::NYql::NUdf::TUnboxedValuePod* args) const override { \ - try { \ - return RunImpl(valueBuilder, args); \ - } catch (const std::exception& ex) { \ - TStringBuilder sb; \ - APPEND_SOURCE_LOCATION(sb, valueBuilder, Pos_) \ - sb << ex.what(); \ - UdfTerminate(sb.c_str()); \ - } \ - } \ - \ - private: \ + try { \ + return RunImpl(valueBuilder, args); \ + } catch (const std::exception& ex) { \ + TStringBuilder sb; \ + APPEND_SOURCE_LOCATION(sb, valueBuilder, Pos_) \ + sb << ex.what(); \ + UdfTerminate(sb.c_str()); \ + } \ + } \ + \ + private: \ ::NYql::NUdf::TUnboxedValue RunConfig; \ ::NYql::NUdf::TSourcePosition Pos_; \ - }; \ + }; \ ::NYql::NUdf::TUnboxedValue Run( \ const ::NYql::NUdf::IValueBuilder* valueBuilder, \ const ::NYql::NUdf::TUnboxedValuePod* args) const override { \ - Y_UNUSED(valueBuilder); \ + Y_UNUSED(valueBuilder); \ return ::NYql::NUdf::TUnboxedValuePod(new TImpl(*this, args[0], Pos_)); \ - } \ - static bool DeclareSignature( \ + } \ + static bool DeclareSignature( \ const ::NYql::NUdf::TStringRef& name, \ ::NYql::NUdf::TType* userType, \ ::NYql::NUdf::IFunctionTypeInfoBuilder& builder, \ - bool typesOnly) { \ - Y_UNUSED(userType); \ - if (Name() == name) { \ - typeBody if (!typesOnly) { \ - builder.Implementation(new udfName(builder)); \ - SetIRImplementation(builder, irResourceId, irFunctionName); \ - } \ - return true; \ - } \ - return false; \ - } \ - private: \ + bool typesOnly) { \ + Y_UNUSED(userType); \ + if (Name() == name) { \ + typeBody if (!typesOnly) { \ + builder.Implementation(new udfName(builder)); \ + SetIRImplementation(builder, irResourceId, irFunctionName); \ + } \ + return true; \ + } \ + return false; \ + } \ + \ + private: \ ::NYql::NUdf::TSourcePosition Pos_; \ - }; \ + }; \ ::NYql::NUdf::TUnboxedValue udfName::TImpl::RunImpl( \ const ::NYql::NUdf::IValueBuilder* valueBuilder, \ const ::NYql::NUdf::TUnboxedValuePod* args) const @@ -230,7 +234,6 @@ namespace NUdf { #define SIMPLE_UDF_IMPL(udfName, typeBody, signature, irResourceId, irFunctionName, blockType) \ SIMPLE_UDF_IMPL_EX(udfName, typeBody, signature, irResourceId, irFunctionName, blockType, builder.Implementation(new udfName(builder));) - #define SIMPLE_UDF(udfName, signature) \ SIMPLE_UDF_IMPL(udfName, builder.SimpleSignature<signature>();, signature, "", "", void) @@ -261,25 +264,25 @@ namespace NUdf { #define SIMPLE_UDF_RUN(udfName, signature, runConfigSignature) \ SIMPLE_UDF_RUN_OPTIONS(udfName, signature, builder.RunConfig<runConfigSignature>()) -#define SIMPLE_MODULE(moduleName, ...) \ - class moduleName: public ::NYql::NUdf::TSimpleUdfModuleHelper<__VA_ARGS__> { \ - public: \ - ::NYql::NUdf::TStringRef Name() const { \ - auto name = ::NYql::NUdf::TStringRef::Of(#moduleName); \ - return name.Substring(1, name.Size() - 7); \ - } \ +#define SIMPLE_MODULE(moduleName, ...) \ + class moduleName: public ::NYql::NUdf::TSimpleUdfModuleHelper<__VA_ARGS__> { \ + public: \ + ::NYql::NUdf::TStringRef Name() const { \ + auto name = ::NYql::NUdf::TStringRef::Of(#moduleName); \ + return name.Substring(1, name.Size() - 7); \ + } \ }; -#define EMPTY_RESULT_ON_EMPTY_ARG(n) \ - if (!args[n]) { \ +#define EMPTY_RESULT_ON_EMPTY_ARG(n) \ + if (!args[n]) { \ return ::NYql::NUdf::TUnboxedValue(); \ } namespace NYql { namespace NUdf { -template<bool CheckOptional, bool CheckBlock, const char* TFuncName, template<class> class TFunc, typename... TUserTypes> -class TUserDataTypeFuncFactory : public ::NYql::NUdf::TBoxedValue { +template <bool CheckOptional, bool CheckBlock, const char* TFuncName, template <class> class TFunc, typename... TUserTypes> +class TUserDataTypeFuncFactory: public ::NYql::NUdf::TBoxedValue { public: typedef bool TTypeAwareMarker; @@ -308,8 +311,7 @@ public: return userType; } - - template<typename TUserType> + template <typename TUserType> static bool DeclareSignatureImpl( const ::NYql::NUdf::TStringRef& name, TDataTypeId typeId, @@ -324,7 +326,7 @@ public: return true; } - template<typename TUserType, typename THead, typename... TTail> + template <typename TUserType, typename THead, typename... TTail> static bool DeclareSignatureImpl( const ::NYql::NUdf::TStringRef& name, TDataTypeId typeId, @@ -390,9 +392,8 @@ public: } }; -template<CUDF... TUdfs> -class TSimpleUdfModuleHelper : public IUdfModule -{ +template <CUDF... TUdfs> +class TSimpleUdfModuleHelper: public IUdfModule { Y_HAS_SUBTYPE(TTypeAwareMarker); Y_HAS_SUBTYPE(TBlockType); @@ -400,7 +401,7 @@ public: void CleanupOnTerminate() const override { } - template<typename TUdfType> + template <typename TUdfType> void GetAllFunctionsImpl(IFunctionNamesSink& names) const { auto r = names.Add(TUdfType::Name()); if (THasTTypeAwareMarker<TUdfType>::value) { @@ -415,20 +416,19 @@ public: } } - template<typename THead1, typename THead2, typename... TTail> + template <typename THead1, typename THead2, typename... TTail> void GetAllFunctionsImpl(IFunctionNamesSink& names) const { GetAllFunctionsImpl<THead1>(names); GetAllFunctionsImpl<THead2, TTail...>(names); } - template<typename TUdfType> + template <typename TUdfType> bool BuildFunctionTypeInfoImpl( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, - IFunctionTypeInfoBuilder& builder) const - { + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, + IFunctionTypeInfoBuilder& builder) const { Y_UNUSED(typeConfig); bool typesOnly = (flags & TFlags::TypesOnly); bool found = TUdfType::DeclareSignature(name, userType, builder, typesOnly); @@ -443,14 +443,13 @@ public: return found; } - template<typename THead1, typename THead2, typename... TTail> + template <typename THead1, typename THead2, typename... TTail> bool BuildFunctionTypeInfoImpl( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, - IFunctionTypeInfoBuilder& builder) const - { + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, + IFunctionTypeInfoBuilder& builder) const { bool found = BuildFunctionTypeInfoImpl<THead1>(name, userType, typeConfig, flags, builder); if (!found) { found = BuildFunctionTypeInfoImpl<THead2, TTail...>(name, userType, typeConfig, flags, builder); @@ -463,12 +462,11 @@ public: } void BuildFunctionTypeInfo( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, - IFunctionTypeInfoBuilder& builder) const override - { + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, + IFunctionTypeInfoBuilder& builder) const override { try { bool found = BuildFunctionTypeInfoImpl<TUdfs...>(name, userType, typeConfig, flags, builder); if (!found) { @@ -480,8 +478,7 @@ public: builder.SetError(CurrentExceptionMessage()); } } - }; -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_log.cpp b/yql/essentials/public/udf/udf_log.cpp index d9f1327acb2..25983f7e686 100644 --- a/yql/essentials/public/udf/udf_log.cpp +++ b/yql/essentials/public/udf/udf_log.cpp @@ -9,7 +9,7 @@ namespace NUdf { namespace { -class TNullLogger : public ILogger { +class TNullLogger: public ILogger { public: TLogComponentId RegisterComponent(const TStringRef& component) final { Y_UNUSED(component); @@ -38,38 +38,39 @@ public: } }; -class TSynchronizedLogger : public ILogger { +class TSynchronizedLogger: public ILogger { public: TSynchronizedLogger(const TLoggerPtr& inner) : Inner_(inner) - {} + { + } TLogComponentId RegisterComponent(const TStringRef& component) final { - with_lock(Mutex_) { + with_lock (Mutex_) { return Inner_->RegisterComponent(component); } } void SetDefaultLevel(ELogLevel level) final { - with_lock(Mutex_) { + with_lock (Mutex_) { Inner_->SetDefaultLevel(level); } } void SetComponentLevel(TLogComponentId component, ELogLevel level) final { - with_lock(Mutex_) { + with_lock (Mutex_) { Inner_->SetComponentLevel(component, level); } } bool IsActive(TLogComponentId component, ELogLevel level) const final { - with_lock(Mutex_) { + with_lock (Mutex_) { return Inner_->IsActive(component, level); } } void Log(TLogComponentId component, ELogLevel level, const TStringRef& message) final { - with_lock(Mutex_) { + with_lock (Mutex_) { Inner_->Log(component, level, message); } } @@ -139,12 +140,13 @@ private: THashMap<TLogComponentId, ELogLevel> CompLevels_; }; -class TLogProvider : public ILogProvider { +class TLogProvider: public ILogProvider { public: TLogProvider(TLogProviderFunc func, TMaybe<ELogLevel> filter) : Func_(func) , Filter_(filter) - {} + { + } TLoggerPtr MakeLogger() const final { return new TLogger(Func_, Filter_); @@ -155,7 +157,7 @@ private: const TMaybe<ELogLevel> Filter_; }; -} +} // namespace TLoggerPtr MakeNullLogger() { return new TNullLogger(); @@ -166,7 +168,8 @@ TLoggerPtr MakeSynchronizedLogger(const TLoggerPtr& inner) { } #define SWITCH_ENUM_TYPE_TO_STR(name, val) \ - case val: return TStringBuf(#name); + case val: \ + return TStringBuf(#name); TStringBuf LevelToString(ELogLevel level) { switch (static_cast<ui32>(level)) { @@ -177,7 +180,8 @@ TStringBuf LevelToString(ELogLevel level) { } #define PARSE_ENUM_TYPE_FROM_STR(name, val) \ - if (#name == str) return static_cast<ELogLevel>(val); + if (#name == str) \ + return static_cast<ELogLevel>(val); TMaybe<ELogLevel> TryLevelFromString(TStringBuf str) { UDF_LOG_LEVEL(PARSE_ENUM_TYPE_FROM_STR) @@ -189,13 +193,12 @@ TMaybe<ELogLevel> TryLevelFromString(TStringBuf str) { TString LogLevelAvailables() { return JoinSeq(", ", - {UDF_LOG_LEVEL(ENUM_STR_JOIN)} - ); + {UDF_LOG_LEVEL(ENUM_STR_JOIN)}); } TUniquePtr<ILogProvider> MakeLogProvider(TLogProviderFunc func, TMaybe<ELogLevel> filter) { return new TLogProvider(func, filter); } -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_log.h b/yql/essentials/public/udf/udf_log.h index b09edef148f..d76c7031509 100644 --- a/yql/essentials/public/udf/udf_log.h +++ b/yql/essentials/public/udf/udf_log.h @@ -11,30 +11,30 @@ namespace NYql { namespace NUdf { -#define UDF_LOG(logger, component, level, msg) \ -do { \ - if ((logger) && (logger)->IsActive(component, level)) { \ - (logger)->Log(component, level, msg); \ - } \ -} while (0) - -#define UDF_LOG_IF(condition, logger, component, level, msg) \ -do { \ - if ((logger) && (condition) && (logger)->IsActive(component, level)) { \ - (logger)->Log(component, level, msg); \ - } \ -} while (0) +#define UDF_LOG(logger, component, level, msg) \ + do { \ + if ((logger) && (logger)->IsActive(component, level)) { \ + (logger)->Log(component, level, msg); \ + } \ + } while (0) + +#define UDF_LOG_IF(condition, logger, component, level, msg) \ + do { \ + if ((logger) && (condition) && (logger)->IsActive(component, level)) { \ + (logger)->Log(component, level, msg); \ + } \ + } while (0) #define UDF_LOG_LEVEL(XX) \ - XX(Fatal, 0) \ - XX(Error, 1) \ - XX(Warn, 2) \ - XX(Notice, 3) \ - XX(Info, 4) \ - XX(Debug, 5) \ + XX(Fatal, 0) \ + XX(Error, 1) \ + XX(Warn, 2) \ + XX(Notice, 3) \ + XX(Info, 4) \ + XX(Debug, 5) \ XX(Trace, 6) -enum class ELogLevel : ui32 { +enum class ELogLevel: ui32 { UDF_LOG_LEVEL(ENUM_VALUE_GEN) }; @@ -48,7 +48,7 @@ TMaybe<ELogLevel> TryLevelFromString(TStringBuf str); using TLogComponentId = ui32; -class ILogger : public IRefCounted { +class ILogger: public IRefCounted { public: virtual TLogComponentId RegisterComponent(const TStringRef& component) = 0; virtual void SetDefaultLevel(ELogLevel level) = 0; @@ -75,10 +75,10 @@ TLoggerPtr MakeSynchronizedLogger(const TLoggerPtr& inner); using TLogProviderFunc = std::function<void(const TStringRef&, ELogLevel, const TStringRef&)>; TUniquePtr<ILogProvider> MakeLogProvider(TLogProviderFunc func, TMaybe<ELogLevel> filter = Nothing()); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql -template<> -inline void Out<NYql::NUdf::ELogLevel>(IOutputStream &o, NYql::NUdf::ELogLevel value) { +template <> +inline void Out<NYql::NUdf::ELogLevel>(IOutputStream& o, NYql::NUdf::ELogLevel value) { o << NYql::NUdf::LevelToString(value); } diff --git a/yql/essentials/public/udf/udf_pg_type_description.h b/yql/essentials/public/udf/udf_pg_type_description.h index 859dba6c54d..6e242f675b9 100644 --- a/yql/essentials/public/udf/udf_pg_type_description.h +++ b/yql/essentials/public/udf/udf_pg_type_description.h @@ -24,5 +24,5 @@ using TPgTypeDescription = TPgTypeDescription1; UDF_ASSERT_TYPE_SIZE(TPgTypeDescription1, 40); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_ptr.h b/yql/essentials/public/udf/udf_ptr.h index 98945c5f5fe..e0a62ce4d30 100644 --- a/yql/essentials/public/udf/udf_ptr.h +++ b/yql/essentials/public/udf/udf_ptr.h @@ -13,14 +13,13 @@ struct TDelete { delete ptr; } }; -} +} // namespace NDetails /////////////////////////////////////////////////////////////////////////////// // TUniquePtr /////////////////////////////////////////////////////////////////////////////// template <typename T, typename D = NDetails::TDelete> -class TUniquePtr -{ +class TUniquePtr { public: inline TUniquePtr(T* ptr = nullptr) : Ptr_(ptr) @@ -66,15 +65,24 @@ public: rhs.Ptr_ = tmp; } - inline T* Get() const { return Ptr_; } - inline T& operator*() const { return *Ptr_; } - inline T* operator->() const { return Ptr_; } - inline explicit operator bool() const { return Ptr_ != nullptr; } + inline T* Get() const { + return Ptr_; + } + inline T& operator*() const { + return *Ptr_; + } + inline T* operator->() const { + return Ptr_; + } + inline explicit operator bool() const { + return Ptr_ != nullptr; + } private: inline void DoDestroy() { - if (Ptr_) + if (Ptr_) { D::DoDelete(Ptr_); + } } private: @@ -85,8 +93,7 @@ private: // TRefCountedPtr /////////////////////////////////////////////////////////////////////////////// template <class T> -class TDefaultRefCountedPtrOps -{ +class TDefaultRefCountedPtrOps { public: static inline void Ref(T* t) { Y_ASSERT(t); @@ -105,11 +112,14 @@ public: }; template <typename T, typename Ops = TDefaultRefCountedPtrOps<T>> -class TRefCountedPtr -{ +class TRefCountedPtr { public: - enum AddRef { ADD_REF }; - enum StealRef { STEAL_REF }; + enum AddRef { + ADD_REF + }; + enum StealRef { + STEAL_REF + }; public: inline TRefCountedPtr(T* ptr = nullptr) @@ -188,10 +198,18 @@ public: return tmp; } - inline T* Get() const { return Ptr_; } - inline T& operator*() const { return *Ptr_; } - inline T* operator->() const { return Ptr_; } - inline explicit operator bool() const { return Ptr_ != nullptr; } + inline T* Get() const { + return Ptr_; + } + inline T& operator*() const { + return *Ptr_; + } + inline T* operator->() const { + return Ptr_; + } + inline explicit operator bool() const { + return Ptr_ != nullptr; + } inline ui32 RefCount() const { return Ptr_ ? Ops::RefCount(Ptr_) : 0; @@ -199,7 +217,7 @@ public: private: inline void Ref() { - if (Ptr_){ + if (Ptr_) { Ops::Ref(Ptr_); } } @@ -244,5 +262,5 @@ private: UDF_ASSERT_TYPE_SIZE(IRefCounted, 16); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_registrator.h b/yql/essentials/public/udf/udf_registrator.h index 535b9b5199c..9da08ffbb02 100644 --- a/yql/essentials/public/udf/udf_registrator.h +++ b/yql/essentials/public/udf/udf_registrator.h @@ -12,37 +12,37 @@ #include <stdarg.h> #ifdef _win_ -# ifdef BUILD_UDF -# define YQL_UDF_API __declspec(dllexport) -# else -# define YQL_UDF_API __declspec(dllimport) -# endif + #ifdef BUILD_UDF + #define YQL_UDF_API __declspec(dllexport) + #else + #define YQL_UDF_API __declspec(dllimport) + #endif #else -# define YQL_UDF_API __attribute__ ((visibility("default"))) + #define YQL_UDF_API __attribute__((visibility("default"))) #endif #ifdef BUILD_UDF -#define REGISTER_MODULES(...) \ - extern "C" YQL_UDF_API void Register( \ - ::NYql::NUdf::IRegistrator& registrator, ui32 flags) { \ - Y_UNUSED(flags); \ - ::NYql::NUdf::RegisterHelper<__VA_ARGS__>(registrator); \ - } \ - extern "C" YQL_UDF_API ui32 AbiVersion() { \ - return ::NYql::NUdf::CurrentAbiVersion(); \ - }\ - extern "C" YQL_UDF_API void SetBackTraceCallback(::NYql::NUdf::TBackTraceCallback callback) { \ - ::NYql::NUdf::SetBackTraceCallbackImpl(callback); \ - } + #define REGISTER_MODULES(...) \ + extern "C" YQL_UDF_API void Register( \ + ::NYql::NUdf::IRegistrator& registrator, ui32 flags) { \ + Y_UNUSED(flags); \ + ::NYql::NUdf::RegisterHelper<__VA_ARGS__>(registrator); \ + } \ + extern "C" YQL_UDF_API ui32 AbiVersion() { \ + return ::NYql::NUdf::CurrentAbiVersion(); \ + } \ + extern "C" YQL_UDF_API void SetBackTraceCallback(::NYql::NUdf::TBackTraceCallback callback) { \ + ::NYql::NUdf::SetBackTraceCallbackImpl(callback); \ + } #else -#define REGISTER_MODULES(...) \ - namespace { \ - struct TYqlStaticUdfRegistrator { \ - inline TYqlStaticUdfRegistrator() { \ + #define REGISTER_MODULES(...) \ + namespace { \ + struct TYqlStaticUdfRegistrator { \ + inline TYqlStaticUdfRegistrator() { \ ::NYql::NUdf::AddToStaticUdfRegistry<__VA_ARGS__>(); \ - } \ - } YQL_REGISTRATOR; \ - } + } \ + } YQL_REGISTRATOR; \ + } #endif namespace NYql { @@ -80,8 +80,7 @@ inline TStaticSymbols GetStaticSymbols(); ////////////////////////////////////////////////////////////////////////////// // IFunctionNamesSink ////////////////////////////////////////////////////////////////////////////// -class IFunctionDescriptor -{ +class IFunctionDescriptor { public: typedef TUniquePtr<IFunctionDescriptor> TPtr; @@ -95,8 +94,7 @@ UDF_ASSERT_TYPE_SIZE(IFunctionDescriptor, 8); ////////////////////////////////////////////////////////////////////////////// // IFunctionNamesSink ////////////////////////////////////////////////////////////////////////////// -class IFunctionsSink -{ +class IFunctionsSink { public: virtual ~IFunctionsSink() = default; @@ -110,8 +108,7 @@ typedef IFunctionsSink IFunctionNamesSink; ////////////////////////////////////////////////////////////////////////////// // IUdfModule ////////////////////////////////////////////////////////////////////////////// -class IUdfModule -{ +class IUdfModule { public: struct TFlags { enum { @@ -125,11 +122,11 @@ public: virtual void GetAllFunctions(IFunctionsSink& sink) const = 0; virtual void BuildFunctionTypeInfo( - const TStringRef& name, - TType* userType, - const TStringRef& typeConfig, - ui32 flags, - IFunctionTypeInfoBuilder& builder) const = 0; + const TStringRef& name, + TType* userType, + const TStringRef& typeConfig, + ui32 flags, + IFunctionTypeInfoBuilder& builder) const = 0; virtual void CleanupOnTerminate() const = 0; }; @@ -139,8 +136,7 @@ UDF_ASSERT_TYPE_SIZE(IUdfModule, 8); ////////////////////////////////////////////////////////////////////////////// // TRegistrator ////////////////////////////////////////////////////////////////////////////// -class IRegistrator -{ +class IRegistrator { public: struct TFlags { enum { @@ -152,27 +148,27 @@ public: virtual ~IRegistrator() = default; virtual void AddModule( - const TStringRef& name, - TUniquePtr<IUdfModule> module) = 0; + const TStringRef& name, + TUniquePtr<IUdfModule> module) = 0; }; UDF_ASSERT_TYPE_SIZE(IRegistrator, 8); -typedef void(*TBackTraceCallback)(); +typedef void (*TBackTraceCallback)(); using TRegisterFunctionPtr = void (*)(IRegistrator& registrator, ui32 flags); using TAbiVersionFunctionPtr = ui32 (*)(); using TBindSymbolsFunctionPtr = void (*)(const TStaticSymbols& symbols); -using TSetBackTraceCallbackPtr = void(*)(TBackTraceCallback callback); +using TSetBackTraceCallbackPtr = void (*)(TBackTraceCallback callback); -template<typename TModule> +template <typename TModule> static inline void RegisterHelper(IRegistrator& registrator) { TUniquePtr<TModule> ptr(new TModule()); auto name = ptr->Name(); registrator.AddModule(name, ptr.Release()); } -template<typename THead1, typename THead2, typename... TTail> +template <typename THead1, typename THead2, typename... TTail> static inline void RegisterHelper(IRegistrator& registrator) { RegisterHelper<THead1>(registrator); RegisterHelper<THead2, TTail...>(registrator); @@ -180,12 +176,11 @@ static inline void RegisterHelper(IRegistrator& registrator) { void SetBackTraceCallbackImpl(TBackTraceCallback callback); - using TUdfModuleWrapper = std::function<std::pair<TStringRef, TUniquePtr<IUdfModule>>()>; void AddToStaticUdfRegistry(TUdfModuleWrapper&&); -template<typename TModule> +template <typename TModule> static inline void AddToStaticUdfRegistry() { AddToStaticUdfRegistry([]() { TUniquePtr<TModule> ptr(new TModule()); @@ -194,15 +189,14 @@ static inline void AddToStaticUdfRegistry() { }); } -template<typename THead1, typename THead2, typename... TTail> +template <typename THead1, typename THead2, typename... TTail> static inline void AddToStaticUdfRegistry() { AddToStaticUdfRegistry<THead1>(); AddToStaticUdfRegistry<THead2, TTail...>(); } - -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql extern "C" YQL_UDF_API void Register(NYql::NUdf::IRegistrator& registrator, ui32 flags); extern "C" YQL_UDF_API ui32 AbiVersion(); @@ -221,12 +215,14 @@ Y_PRAGMA_NO_DEPRECATED inline TStaticSymbols GetStaticSymbols() { return {&UdfAllocate, &UdfFree, &UdfTerminate, &UdfRegisterObject, &UdfUnregisterObject -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) - ,&UdfAllocateWithSize, &UdfFreeWithSize -#endif -#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 37) - ,&UdfArrowAllocate, &UdfArrowReallocate, &UdfArrowFree -#endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) + , + &UdfAllocateWithSize, &UdfFreeWithSize + #endif + #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 37) + , + &UdfArrowAllocate, &UdfArrowReallocate, &UdfArrowFree + #endif }; } @@ -234,5 +230,5 @@ Y_PRAGMA_DIAGNOSTIC_POP #endif -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_static_registry.cpp b/yql/essentials/public/udf/udf_static_registry.cpp index b6efa4872cf..05be2ef33dc 100644 --- a/yql/essentials/public/udf/udf_static_registry.cpp +++ b/yql/essentials/public/udf/udf_static_registry.cpp @@ -18,5 +18,5 @@ void AddToStaticUdfRegistry(TUdfModuleWrapper&& wrapper) { StaticUdfModuleWrapperList()->emplace_back(wrapper); }; -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_static_registry.h b/yql/essentials/public/udf/udf_static_registry.h index e50b05ffaca..75d405bad7b 100644 --- a/yql/essentials/public/udf/udf_static_registry.h +++ b/yql/essentials/public/udf/udf_static_registry.h @@ -10,5 +10,5 @@ namespace NUdf { using TUdfModuleWrapperList = TVector<TUdfModuleWrapper>; const TUdfModuleWrapperList& GetStaticUdfModuleWrapperList(); -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_string.h b/yql/essentials/public/udf/udf_string.h index 3416f4dce9c..9e7ce0ea7da 100644 --- a/yql/essentials/public/udf/udf_string.h +++ b/yql/essentials/public/udf/udf_string.h @@ -16,8 +16,7 @@ namespace NUdf { ////////////////////////////////////////////////////////////////////////////// // TStringValue ////////////////////////////////////////////////////////////////////////////// -class TStringValue -{ +class TStringValue { friend class TUnboxedValue; friend class TUnboxedValuePod; friend class TBlockItem; @@ -25,25 +24,31 @@ class TStringValue class TData { friend class TStringValue; + public: inline TData(ui32 size, ui64 cap) : Size_(size) , Refs_(1) , Capacity_(cap) - {} + { + } - inline i32 RefCount() const { return Refs_; } + inline i32 RefCount() const { + return Refs_; + } inline void Ref() { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif Refs_++; } inline void UnRef() { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif Y_DEBUG_ABORT_UNLESS(Refs_ > 0); if (!--Refs_) { @@ -56,16 +61,18 @@ class TStringValue } inline void ReleaseRef() { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif Y_DEBUG_ABORT_UNLESS(Refs_ > 0); --Refs_; } inline void DeleteUnreferenced() { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif if (!Refs_) { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) @@ -76,10 +83,18 @@ class TStringValue } } - inline char* Data() const { return (char*)(this+1); } - inline ui32 Size() const { return Size_; } - inline bool Empty() const { return Size_ == 0; } - inline ui64 Capacity() const { return Capacity_; } + inline char* Data() const { + return (char*)(this + 1); + } + inline ui32 Size() const { + return Size_; + } + inline bool Empty() const { + return Size_ == 0; + } + inline ui64 Capacity() const { + return Capacity_; + } inline i32 LockRef() noexcept { Y_DEBUG_ABORT_UNLESS(Refs_ != -1); @@ -110,23 +125,42 @@ class TStringValue private: ui32 Size_; - i32 Refs_; + i32 Refs_; ui64 Capacity_; }; UDF_ASSERT_TYPE_SIZE(TData, 16); public: - inline explicit TStringValue(ui32 size): Data_(AllocateData(size, size)) {} - inline explicit TStringValue(TData* data): Data_(data) { Ref(); } - inline explicit TStringValue(TStringRef data): Data_(AllocateData(data.Size(), data.Size())) { + inline explicit TStringValue(ui32 size) + : Data_(AllocateData(size, size)) + { + } + inline explicit TStringValue(TData* data) + : Data_(data) + { + Ref(); + } + inline explicit TStringValue(TStringRef data) + : Data_(AllocateData(data.Size(), data.Size())) + { memcpy(Data_->Data(), data.Data(), data.Size()); } - inline TStringValue(const TStringValue& rhs): Data_(rhs.Data_) { Ref(); } - inline TStringValue(TStringValue&& rhs): Data_(rhs.Data_) { rhs.Data_ = nullptr; } + inline TStringValue(const TStringValue& rhs) + : Data_(rhs.Data_) + { + Ref(); + } + inline TStringValue(TStringValue&& rhs) + : Data_(rhs.Data_) + { + rhs.Data_ = nullptr; + } - inline ~TStringValue() { UnRef(); } + inline ~TStringValue() { + UnRef(); + } inline TStringValue& operator=(const TStringValue& rhs) { if (this != &rhs) { @@ -148,12 +182,22 @@ public: return *this; } - inline ui64 Capacity() const { return Data_->Capacity(); } - inline ui32 Size() const { return Data_->Size(); } - inline char* Data() const { return Data_->Data(); } + inline ui64 Capacity() const { + return Data_->Capacity(); + } + inline ui32 Size() const { + return Data_->Size(); + } + inline char* Data() const { + return Data_->Data(); + } - inline ui32 size() const { return Data_->Size(); } - inline char* data() const { return Data_->Data(); } + inline ui32 size() const { + return Data_->Size(); + } + inline char* data() const { + return Data_->Data(); + } inline void Ref() { if (Data_ != nullptr) { @@ -180,8 +224,9 @@ public: } bool TryExpandOn(ui32 len) { - if (RefCount() < 0) - return false; + if (RefCount() < 0) { + return false; + } return Data_->TryExpandOn(len); } @@ -213,9 +258,9 @@ public: const auto alligned = AlignUp<ui64>(cap, 16ULL); const auto dataSize = sizeof(TData) + alligned; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 8) - return ::new(UdfAllocateWithSize(dataSize)) TData(len, alligned); + return ::new (UdfAllocateWithSize(dataSize)) TData(len, alligned); #else - return ::new(UdfAllocate(dataSize)) TData(len, alligned); + return ::new (UdfAllocate(dataSize)) TData(len, alligned); #endif } @@ -225,5 +270,5 @@ private: UDF_ASSERT_TYPE_SIZE(TStringValue, 8); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_string_ref.h b/yql/essentials/public/udf/udf_string_ref.h index cee321a8b66..cfe6addc14b 100644 --- a/yql/essentials/public/udf/udf_string_ref.h +++ b/yql/essentials/public/udf/udf_string_ref.h @@ -14,9 +14,8 @@ namespace NUdf { ////////////////////////////////////////////////////////////////////////////// // TStringRefBase ////////////////////////////////////////////////////////////////////////////// -template<bool Const> -class TStringRefBase -{ +template <bool Const> +class TStringRefBase { public: typedef std::conditional_t<Const, const char*, char*> TDataType; @@ -26,18 +25,35 @@ protected: inline constexpr TStringRefBase(TDataType data, ui32 size) noexcept : Data_(data) , Size_(size) - {} + { + } public: - inline constexpr operator std::string_view() const noexcept { return { Data_, Size_ }; } - inline constexpr operator TStringBuf() const noexcept { return { Data_, Size_ }; } - inline constexpr TDataType Data() const noexcept { return Data_; } - inline constexpr ui32 Size() const noexcept { return Size_; } - inline constexpr bool Empty() const noexcept { return Size_ == 0; } + inline constexpr operator std::string_view() const noexcept { + return {Data_, Size_}; + } + inline constexpr operator TStringBuf() const noexcept { + return {Data_, Size_}; + } + inline constexpr TDataType Data() const noexcept { + return Data_; + } + inline constexpr ui32 Size() const noexcept { + return Size_; + } + inline constexpr bool Empty() const noexcept { + return Size_ == 0; + } - inline constexpr TDataType data() const noexcept { return Data_; } - inline constexpr ui32 size() const noexcept { return Size_; } - inline constexpr bool empty() const noexcept { return Size_ == 0; } + inline constexpr TDataType data() const noexcept { + return Data_; + } + inline constexpr ui32 size() const noexcept { + return Size_; + } + inline constexpr bool empty() const noexcept { + return Size_ == 0; + } protected: TDataType Data_ = nullptr; @@ -48,14 +64,14 @@ protected: ////////////////////////////////////////////////////////////////////////////// // TMutableStringRef ////////////////////////////////////////////////////////////////////////////// -class TMutableStringRef : public TStringRefBase<false> -{ +class TMutableStringRef: public TStringRefBase<false> { public: typedef TStringRefBase<false> TBase; inline constexpr TMutableStringRef(TDataType data, ui32 size) noexcept : TBase(data, size) - {} + { + } }; UDF_ASSERT_TYPE_SIZE(TMutableStringRef, 16); @@ -63,8 +79,7 @@ UDF_ASSERT_TYPE_SIZE(TMutableStringRef, 16); ////////////////////////////////////////////////////////////////////////////// // TStringRef ////////////////////////////////////////////////////////////////////////////// -class TStringRef : public TStringRefBase<true> -{ +class TStringRef: public TStringRefBase<true> { public: typedef TStringRefBase<true> TBase; @@ -72,24 +87,28 @@ public: inline constexpr TStringRef(TDataType data, ui32 size) noexcept : TBase(data, size) - {} + { + } - template<size_t Size> + template <size_t Size> inline constexpr TStringRef(const char (&data)[Size]) noexcept : TBase(data, Size - 1) - {} + { + } inline constexpr TStringRef(const TMutableStringRef& buf) noexcept : TBase(buf.Data(), buf.Size()) - {} + { + } template <typename TStringType> inline constexpr TStringRef(const TStringType& buf) noexcept : TBase(TGetData<TStringType>::Get(buf), TGetSize<TStringType>::Get(buf)) - {} + { + } template <size_t size> - inline static constexpr TStringRef Of(const char(&str)[size]) noexcept { + inline static constexpr TStringRef Of(const char (&str)[size]) noexcept { return TStringRef(str); } @@ -137,50 +156,51 @@ public: private: inline static constexpr i64 Compare(const TStringRef& s1, const TStringRef& s2) noexcept { auto minSize = std::min(s1.Size(), s2.Size()); - if (const auto result = minSize > 0 ? std::memcmp(s1.Data(), s2.Data(), minSize) : 0) + if (const auto result = minSize > 0 ? std::memcmp(s1.Data(), s2.Data(), minSize) : 0) { return result; + } return i64(s1.Size()) - i64(s2.Size()); } Y_HAS_MEMBER(Data); Y_HAS_MEMBER(Size); - template<typename TStringType> + template <typename TStringType> struct TByData { static constexpr auto Get(const TStringType& buf) noexcept { return buf.data(); } }; - template<typename TStringType> + template <typename TStringType> struct TBySize { static constexpr auto Get(const TStringType& buf) noexcept { return buf.size(); } }; - template<typename TStringType> + template <typename TStringType> struct TBydata { static constexpr auto Get(const TStringType& buf) noexcept { return buf.data(); } }; - template<typename TStringType> + template <typename TStringType> struct TBysize { static constexpr auto Get(const TStringType& buf) noexcept { return buf.size(); } }; - template<typename TStringType> + template <typename TStringType> using TGetData = std::conditional_t<THasData<TStringType>::value, TByData<TStringType>, TBydata<TStringType>>; - template<typename TStringType> + template <typename TStringType> using TGetSize = std::conditional_t<THasSize<TStringType>::value, TBySize<TStringType>, TBysize<TStringType>>; }; UDF_ASSERT_TYPE_SIZE(TStringRef, 16); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_terminator.h b/yql/essentials/public/udf/udf_terminator.h index b94c7975d33..f8b4767674e 100644 --- a/yql/essentials/public/udf/udf_terminator.h +++ b/yql/essentials/public/udf/udf_terminator.h @@ -3,9 +3,9 @@ namespace NYql { namespace NUdf { - class TBoxedValue; -} -} +class TBoxedValue; +} // namespace NUdf +} // namespace NYql extern "C" [[noreturn]] void UdfTerminate(const char* message); extern "C" void UdfRegisterObject(::NYql::NUdf::TBoxedValue* object); diff --git a/yql/essentials/public/udf/udf_type_builder.cpp b/yql/essentials/public/udf/udf_type_builder.cpp index 321d36685c7..70e73297307 100644 --- a/yql/essentials/public/udf/udf_type_builder.cpp +++ b/yql/essentials/public/udf/udf_type_builder.cpp @@ -4,7 +4,8 @@ namespace NYql { namespace NUdf { IFunctionTypeInfoBuilder::IFunctionTypeInfoBuilder() -{} +{ +} } // namespace NUdf } // namespace NYql diff --git a/yql/essentials/public/udf/udf_type_builder.h b/yql/essentials/public/udf/udf_type_builder.h index 4cb08c89ecc..3b8b6c7dc6f 100644 --- a/yql/essentials/public/udf/udf_type_builder.h +++ b/yql/essentials/public/udf/udf_type_builder.h @@ -27,16 +27,24 @@ public: }; template <typename T> -struct TAutoMap { using ItemType = T; }; +struct TAutoMap { + using ItemType = T; +}; template <typename T> -struct TOptional { using ItemType = T; }; +struct TOptional { + using ItemType = T; +}; template <typename T, const char* Name> -struct TNamedArg { using ItemType = T; }; +struct TNamedArg { + using ItemType = T; +}; template <typename T> -struct TListType { using ItemType = T; }; +struct TListType { + using ItemType = T; +}; template <typename TKey, typename TValue> struct TDict { @@ -59,30 +67,39 @@ template <typename... TArgs> struct TVariant; template <typename T> -struct TStream { using ItemType = T; }; +struct TStream { + using ItemType = T; +}; template <typename T, const char* Tag> -struct TTagged { using BaseType = T; }; +struct TTagged { + using BaseType = T; +}; template <ui32 TypeId> struct TPg; template <typename T> -struct TBlockType { using ItemType = T; }; +struct TBlockType { + using ItemType = T; +}; template <typename T> -struct TScalarType { using ItemType = T; }; +struct TScalarType { + using ItemType = T; +}; template <typename T, bool IsDynamic = false> -struct TLinear { using ItemType = T; }; +struct TLinear { + using ItemType = T; +}; struct TVoid {}; ////////////////////////////////////////////////////////////////////////////// // ITypeBuilder ////////////////////////////////////////////////////////////////////////////// -class ITypeBuilder -{ +class ITypeBuilder { public: virtual ~ITypeBuilder() = default; @@ -94,8 +111,7 @@ UDF_ASSERT_TYPE_SIZE(ITypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IOptionalTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IOptionalTypeBuilder: public ITypeBuilder -{ +class IOptionalTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IOptionalTypeBuilder>; @@ -115,8 +131,7 @@ UDF_ASSERT_TYPE_SIZE(IOptionalTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IListTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IListTypeBuilder: public ITypeBuilder -{ +class IListTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IListTypeBuilder>; @@ -136,8 +151,7 @@ UDF_ASSERT_TYPE_SIZE(IListTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IVariantTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IVariantTypeBuilder : public ITypeBuilder -{ +class IVariantTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IVariantTypeBuilder>; @@ -152,8 +166,7 @@ UDF_ASSERT_TYPE_SIZE(IVariantTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IStreamTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IStreamTypeBuilder : public ITypeBuilder -{ +class IStreamTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IStreamTypeBuilder>; @@ -173,8 +186,7 @@ UDF_ASSERT_TYPE_SIZE(IStreamTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IDictTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IDictTypeBuilder: public ITypeBuilder -{ +class IDictTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IDictTypeBuilder>; @@ -203,8 +215,7 @@ UDF_ASSERT_TYPE_SIZE(IDictTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // ISetTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class ISetTypeBuilder: public ITypeBuilder -{ +class ISetTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<ISetTypeBuilder>; @@ -224,29 +235,28 @@ UDF_ASSERT_TYPE_SIZE(ISetTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IStructTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IStructTypeBuilder: public ITypeBuilder -{ +class IStructTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IStructTypeBuilder>; public: template <typename T, typename = std::enable_if_t<TKnownDataType<T>::Result>> inline IStructTypeBuilder& AddField( - const TStringRef& name, ui32* index) + const TStringRef& name, ui32* index) { return AddField(name, TDataType<T>::Id, index); } virtual IStructTypeBuilder& AddField( - const TStringRef& name, TDataTypeId type, ui32* index) = 0; + const TStringRef& name, TDataTypeId type, ui32* index) = 0; virtual IStructTypeBuilder& AddField( - const TStringRef& name, const TType* type, ui32* index) = 0; + const TStringRef& name, const TType* type, ui32* index) = 0; virtual IStructTypeBuilder& AddField( - const TStringRef& name, - const ITypeBuilder& typeBuilder, - ui32* index) = 0; + const TStringRef& name, + const ITypeBuilder& typeBuilder, + ui32* index) = 0; }; UDF_ASSERT_TYPE_SIZE(IStructTypeBuilder, 8); @@ -254,14 +264,13 @@ UDF_ASSERT_TYPE_SIZE(IStructTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // IEnumTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IEnumTypeBuilder: public ITypeBuilder -{ +class IEnumTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IEnumTypeBuilder>; public: virtual IEnumTypeBuilder& AddField( - const TStringRef& name, ui32* index) = 0; + const TStringRef& name, ui32* index) = 0; }; UDF_ASSERT_TYPE_SIZE(IEnumTypeBuilder, 8); @@ -269,8 +278,7 @@ UDF_ASSERT_TYPE_SIZE(IEnumTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // ITupleTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class ITupleTypeBuilder: public ITypeBuilder -{ +class ITupleTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<ITupleTypeBuilder>; @@ -290,8 +298,7 @@ UDF_ASSERT_TYPE_SIZE(ITupleTypeBuilder, 8); ////////////////////////////////////////////////////////////////////////////// // ICallableTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class ICallableTypeBuilder: public ITypeBuilder -{ +class ICallableTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<ICallableTypeBuilder>; @@ -332,8 +339,7 @@ class IFunctionTypeInfoBuilder; ////////////////////////////////////////////////////////////////////////////// // IFunctionArgTypesBuilder ////////////////////////////////////////////////////////////////////////////// -class IFunctionArgTypesBuilder -{ +class IFunctionArgTypesBuilder { public: using TPtr = TUniquePtr<IFunctionArgTypesBuilder>; @@ -373,7 +379,7 @@ UDF_ASSERT_TYPE_SIZE(IFunctionArgTypesBuilder, 16); ////////////////////////////////////////////////////////////////////////////// // IHash ////////////////////////////////////////////////////////////////////////////// -class IHash : public IRefCounted { +class IHash: public IRefCounted { public: using TPtr = TRefCountedPtr<IHash>; @@ -385,7 +391,7 @@ UDF_ASSERT_TYPE_SIZE(IHash, 16); ////////////////////////////////////////////////////////////////////////////// // IEquate ////////////////////////////////////////////////////////////////////////////// -class IEquate : public IRefCounted { +class IEquate: public IRefCounted { public: using TPtr = TRefCountedPtr<IEquate>; @@ -397,7 +403,7 @@ UDF_ASSERT_TYPE_SIZE(IEquate, 16); ////////////////////////////////////////////////////////////////////////////// // ICompare ////////////////////////////////////////////////////////////////////////////// -class ICompare : public IRefCounted { +class ICompare: public IRefCounted { public: using TPtr = TRefCountedPtr<ICompare>; @@ -411,14 +417,15 @@ UDF_ASSERT_TYPE_SIZE(ICompare, 16); ////////////////////////////////////////////////////////////////////////////// // IBlockTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class IBlockTypeBuilder: public ITypeBuilder -{ +class IBlockTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<IBlockTypeBuilder>; explicit IBlockTypeBuilder(bool isScalar) : IsScalar_(isScalar) - {} + { + } + public: template <typename T, typename = std::enable_if_t<TKnownDataType<T>::Result>> inline IBlockTypeBuilder& Item() { @@ -438,14 +445,15 @@ UDF_ASSERT_TYPE_SIZE(IBlockTypeBuilder, 16); ////////////////////////////////////////////////////////////////////////////// // ILinearTypeBuilder ////////////////////////////////////////////////////////////////////////////// -class ILinearTypeBuilder: public ITypeBuilder -{ +class ILinearTypeBuilder: public ITypeBuilder { public: using TPtr = TUniquePtr<ILinearTypeBuilder>; explicit ILinearTypeBuilder(bool isDynamic) : IsDynamic_(isDynamic) - {} + { + } + public: template <typename T, typename = std::enable_if_t<TKnownDataType<T>::Result>> inline ILinearTypeBuilder& Item() { @@ -467,21 +475,28 @@ UDF_ASSERT_TYPE_SIZE(ILinearTypeBuilder, 16); ////////////////////////////////////////////////////////////////////////////// namespace NImpl { -template <typename T> struct TSimpleSignatureHelper; -template <typename T> struct TSimpleSignatureTypeHelper; -template <typename T> struct TTypeBuilderHelper; -template <typename... TArgs> struct TArgsHelper; -template <typename... TArgs> struct TTupleHelper; -template <typename... TArgs> struct TCallableArgsHelper; +template <typename T> +struct TSimpleSignatureHelper; +template <typename T> +struct TSimpleSignatureTypeHelper; +template <typename T> +struct TTypeBuilderHelper; +template <typename... TArgs> +struct TArgsHelper; +template <typename... TArgs> +struct TTupleHelper; +template <typename... TArgs> +struct TCallableArgsHelper; -} // namspace NImpl +} // namespace NImpl struct TSourcePosition { TSourcePosition(ui32 row = 0, ui32 column = 0, TStringRef file = {}) : Row(row) , Column(column) , File(file) - {} + { + } ui32 Row; ui32 Column; @@ -495,14 +510,13 @@ inline IOutputStream& operator<<(IOutputStream& os, const TSourcePosition& pos) return os; } -class IFunctionTypeInfoBuilder1 -{ +class IFunctionTypeInfoBuilder1 { public: virtual ~IFunctionTypeInfoBuilder1() = default; // function implementation virtual IFunctionTypeInfoBuilder1& ImplementationImpl( - TUniquePtr<IBoxedValue> impl) = 0; + TUniquePtr<IBoxedValue> impl) = 0; virtual IFunctionTypeInfoBuilder1& ReturnsImpl(TDataTypeId type) = 0; virtual IFunctionTypeInfoBuilder1& ReturnsImpl(const TType* type) = 0; @@ -594,8 +608,7 @@ public: virtual IFunctionTypeInfoBuilder7& IRImplementationImpl( const TStringRef& moduleIR, const TStringRef& moduleIRUniqId, - const TStringRef& functionName - ) = 0; + const TStringRef& functionName) = 0; }; #endif @@ -740,7 +753,7 @@ public: IFunctionTypeInfoBuilder(); IFunctionTypeInfoBuilder& Implementation( - TUniquePtr<IBoxedValue> impl) { + TUniquePtr<IBoxedValue> impl) { ImplementationImpl(std::move(impl)); return *this; } @@ -852,8 +865,7 @@ public: IFunctionTypeInfoBuilder& IRImplementation( const TStringRef& moduleIR, const TStringRef& moduleIRUniqId, - const TStringRef& functionName - ) { + const TStringRef& functionName) { IRImplementationImpl(moduleIR, moduleIRUniqId, functionName); return *this; } @@ -869,8 +881,8 @@ namespace NImpl { template <typename T> struct TTypeBuilderHelper { static TType* Build( - const IFunctionTypeInfoBuilder& builder, - std::enable_if_t<TKnownDataType<T>::Result>* = nullptr) + const IFunctionTypeInfoBuilder& builder, + std::enable_if_t<TKnownDataType<T>::Result>* = nullptr) { return builder.Primitive(TDataType<T>::Id); } @@ -902,38 +914,28 @@ struct TTypeBuilderHelper<TResource<Tag>> { template <typename T> struct TTypeBuilderHelper<TListType<T>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.List()-> - Item(TTypeBuilderHelper<T>::Build(builder)) - .Build(); + return builder.List()->Item(TTypeBuilderHelper<T>::Build(builder)).Build(); } }; template <typename T> struct TTypeBuilderHelper<TOptional<T>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Optional()-> - Item(TTypeBuilderHelper<T>::Build(builder)) - .Build(); + return builder.Optional()->Item(TTypeBuilderHelper<T>::Build(builder)).Build(); } }; template <typename TKey, typename TValue> struct TTypeBuilderHelper<TDict<TKey, TValue>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Dict()-> - Key(TTypeBuilderHelper<TKey>::Build(builder)) - .Value(TTypeBuilderHelper<TValue>::Build(builder)) - .Build(); + return builder.Dict()->Key(TTypeBuilderHelper<TKey>::Build(builder)).Value(TTypeBuilderHelper<TValue>::Build(builder)).Build(); } }; template <typename TKey> struct TTypeBuilderHelper<TSetType<TKey>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Dict()-> - Key(TTypeBuilderHelper<TKey>::Build(builder)) - .Value(builder.Void()) - .Build(); + return builder.Dict()->Key(TTypeBuilderHelper<TKey>::Build(builder)).Value(builder.Void()).Build(); } }; @@ -947,9 +949,7 @@ struct TTypeBuilderHelper<TVoid> { template <typename T> struct TTypeBuilderHelper<TStream<T>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Stream()-> - Item(TTypeBuilderHelper<T>::Build(builder)) - .Build(); + return builder.Stream()->Item(TTypeBuilderHelper<T>::Build(builder)).Build(); } }; @@ -958,7 +958,7 @@ template <typename T, const char* Tag> struct TTypeBuilderHelper<TTagged<T, Tag>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { return builder.Tagged(TTypeBuilderHelper<T>::Build(builder), - TStringRef(Tag, std::strlen(Tag))); + TStringRef(Tag, std::strlen(Tag))); } }; #endif @@ -976,18 +976,14 @@ struct TTypeBuilderHelper<TPg<TypeId>> { template <typename T> struct TTypeBuilderHelper<TBlockType<T>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Block(false)-> - Item(TTypeBuilderHelper<T>::Build(builder)) - .Build(); + return builder.Block(false)->Item(TTypeBuilderHelper<T>::Build(builder)).Build(); } }; template <typename T> struct TTypeBuilderHelper<TScalarType<T>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Block(true)-> - Item(TTypeBuilderHelper<T>::Build(builder)) - .Build(); + return builder.Block(true)->Item(TTypeBuilderHelper<T>::Build(builder)).Build(); } }; #endif @@ -996,9 +992,7 @@ struct TTypeBuilderHelper<TScalarType<T>> { template <typename T, bool IsDynamic> struct TTypeBuilderHelper<TLinear<T, IsDynamic>> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { - return builder.Linear(IsDynamic)-> - Item(TTypeBuilderHelper<T>::Build(builder)) - .Build(); + return builder.Linear(IsDynamic)->Item(TTypeBuilderHelper<T>::Build(builder)).Build(); } }; @@ -1007,18 +1001,19 @@ struct TTypeBuilderHelper<TLinear<T, IsDynamic>> { template <> struct TCallableArgsHelper<> { static void Arg( - ICallableTypeBuilder& callableBuilder, - const IFunctionTypeInfoBuilder& builder) + ICallableTypeBuilder& callableBuilder, + const IFunctionTypeInfoBuilder& builder) { - Y_UNUSED(callableBuilder); Y_UNUSED(builder); + Y_UNUSED(callableBuilder); + Y_UNUSED(builder); } }; template <typename TArg, typename... TArgs> struct TCallableArgsHelper<TArg, TArgs...> { static void Arg( - ICallableTypeBuilder& callableBuilder, - const IFunctionTypeInfoBuilder& builder) + ICallableTypeBuilder& callableBuilder, + const IFunctionTypeInfoBuilder& builder) { callableBuilder.Arg(TTypeBuilderHelper<TArg>::Build(builder)); TCallableArgsHelper<TArgs...>::Arg(callableBuilder, builder); @@ -1028,28 +1023,28 @@ struct TCallableArgsHelper<TArg, TArgs...> { template <typename TArg, typename... TArgs> struct TCallableArgsHelper<TAutoMap<TArg>, TArgs...> { static void Arg( - ICallableTypeBuilder& callableBuilder, - const IFunctionTypeInfoBuilder& builder) + ICallableTypeBuilder& callableBuilder, + const IFunctionTypeInfoBuilder& builder) { callableBuilder.Arg(TTypeBuilderHelper<TArg>::Build(builder)) #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 38) - // XXX: Unfortunately, ICallableTypeBuilder provides Flags - // method only since UDF ABI 2.38. However, AutoMap flag - // has been silently ignored by this builder and also by - // the TTypeBuilderHelper specialization for AutoMap type. - // Hence, the correct AutoMap type processing is wrapped - // with this compatibility macro, since the caller of - // ICallableTypeBuilder has to explicitly set AutoMap flag - // for the particular argument anyway. - .Flags(ICallablePayload::TArgumentFlags::AutoMap) + // XXX: Unfortunately, ICallableTypeBuilder provides Flags + // method only since UDF ABI 2.38. However, AutoMap flag + // has been silently ignored by this builder and also by + // the TTypeBuilderHelper specialization for AutoMap type. + // Hence, the correct AutoMap type processing is wrapped + // with this compatibility macro, since the caller of + // ICallableTypeBuilder has to explicitly set AutoMap flag + // for the particular argument anyway. + .Flags(ICallablePayload::TArgumentFlags::AutoMap) #endif - ; + ; TCallableArgsHelper<TArgs...>::Arg(callableBuilder, builder); } }; template <typename TReturn, typename... TArgs> -struct TTypeBuilderHelper<TReturn(*)(TArgs...)> { +struct TTypeBuilderHelper<TReturn (*)(TArgs...)> { static TType* Build(const IFunctionTypeInfoBuilder& builder) { auto callableBuilder = builder.Callable(sizeof...(TArgs)); callableBuilder->Returns(TTypeBuilderHelper<TReturn>::Build(builder)); @@ -1061,8 +1056,8 @@ struct TTypeBuilderHelper<TReturn(*)(TArgs...)> { template <typename TArg, typename... TArgs> struct TTupleHelper<TArg, TArgs...> { static void Add( - ITupleTypeBuilder& tupleBuilder, - const IFunctionTypeInfoBuilder& builder) + ITupleTypeBuilder& tupleBuilder, + const IFunctionTypeInfoBuilder& builder) { tupleBuilder.Add(TTypeBuilderHelper<TArg>::Build(builder)); TTupleHelper<TArgs...>::Add(tupleBuilder, builder); @@ -1072,10 +1067,11 @@ struct TTupleHelper<TArg, TArgs...> { template <> struct TTupleHelper<> { static void Add( - ITupleTypeBuilder& tupleBuilder, - const IFunctionTypeInfoBuilder& builder) + ITupleTypeBuilder& tupleBuilder, + const IFunctionTypeInfoBuilder& builder) { - Y_UNUSED(tupleBuilder); Y_UNUSED(builder); + Y_UNUSED(tupleBuilder); + Y_UNUSED(builder); } }; @@ -1108,10 +1104,10 @@ struct TArgsHelper<> { template <typename TArg, typename... TArgs> struct TArgsHelper<TAutoMap<TArg>, TArgs...> { - static void Add(IFunctionArgTypesBuilder& builder, const char* name = nullptr, ui64 flags = 0) { - TArgsHelper<TArg>::Add(builder, name, flags | ICallablePayload::TArgumentFlags::AutoMap); - TArgsHelper<TArgs...>::Add(builder); - } + static void Add(IFunctionArgTypesBuilder& builder, const char* name = nullptr, ui64 flags = 0) { + TArgsHelper<TArg>::Add(builder, name, flags | ICallablePayload::TArgumentFlags::AutoMap); + TArgsHelper<TArgs...>::Add(builder); + } }; template <const char* Name, typename TArg, typename... TArgs> @@ -1120,7 +1116,7 @@ struct TArgsHelper<TNamedArg<TArg, Name>, TArgs...> { Y_UNUSED(name); TArgsHelper<TOptional<TArg>>::Add(builder, Name, flags); TArgsHelper<TArgs...>::Add(builder); - } + } }; template <typename TArg, typename... TArgs> @@ -1156,7 +1152,7 @@ struct TSimpleSignatureTypeHelper<TReturn(TArgs...)> { } }; -} // namspace NImpl +} // namespace NImpl template <typename T> inline IFunctionArgTypesBuilder& IFunctionArgTypesBuilder::Add() diff --git a/yql/essentials/public/udf/udf_type_inspection.cpp b/yql/essentials/public/udf/udf_type_inspection.cpp index 88b5940a584..2e096e83427 100644 --- a/yql/essentials/public/udf/udf_type_inspection.cpp +++ b/yql/essentials/public/udf/udf_type_inspection.cpp @@ -128,5 +128,5 @@ TLinearTypeInspector::TLinearTypeInspector(const ITypeInfoHelper1& typeHelper, c } } -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_type_inspection.h b/yql/essentials/public/udf/udf_type_inspection.h index 31aa66cdd96..93c9a45c8bf 100644 --- a/yql/essentials/public/udf/udf_type_inspection.h +++ b/yql/essentials/public/udf/udf_type_inspection.h @@ -6,42 +6,40 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> // TSmallVec #include <util/generic/algorithm.h> -#include <util/system/yassert.h> // FAIL, VERIFY_DEBUG +#include <util/system/yassert.h> // FAIL, VERIFY_DEBUG #include <util/generic/ylimits.h> // Max - namespace NYql { namespace NUdf { ////////////////////////////////////////////////////////////////////////////// // TStubTypeVisitor ////////////////////////////////////////////////////////////////////////////// -class TStubTypeVisitor1: public ITypeVisitor -{ +class TStubTypeVisitor1: public ITypeVisitor { protected: using ITypeVisitor::ITypeVisitor; + private: void OnDataType(TDataTypeId typeId) override; void OnStruct( - ui32 membersCount, - TStringRef* membersNames, - const TType** membersTypes) override; + ui32 membersCount, + TStringRef* membersNames, + const TType** membersTypes) override; void OnList(const TType* itemType) override; void OnOptional(const TType* itemType) override; void OnTuple(ui32 elementsCount, const TType** elementsTypes) override; void OnDict(const TType* keyType, const TType* valueType) override; void OnCallable( - const TType* returnType, - ui32 argsCount, const TType** argsTypes, - ui32 optionalArgsCount, - const ICallablePayload* payload) override; + const TType* returnType, + ui32 argsCount, const TType** argsTypes, + ui32 optionalArgsCount, + const ICallablePayload* payload) override; void OnVariant(const TType* underlyingType) override; void OnStream(const TType* itemType) override; }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13) -class TStubTypeVisitor2: public TStubTypeVisitor1 -{ +class TStubTypeVisitor2: public TStubTypeVisitor1 { public: using TStubTypeVisitor1::TStubTypeVisitor1; @@ -49,8 +47,7 @@ public: }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) -class TStubTypeVisitor3: public TStubTypeVisitor2 -{ +class TStubTypeVisitor3: public TStubTypeVisitor2 { public: using TStubTypeVisitor2::TStubTypeVisitor2; @@ -58,8 +55,7 @@ public: }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) -class TStubTypeVisitor4: public TStubTypeVisitor3 -{ +class TStubTypeVisitor4: public TStubTypeVisitor3 { public: using TStubTypeVisitor3::TStubTypeVisitor3; @@ -68,8 +64,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -class TStubTypeVisitor5: public TStubTypeVisitor4 -{ +class TStubTypeVisitor5: public TStubTypeVisitor4 { public: using TStubTypeVisitor4::TStubTypeVisitor4; @@ -78,8 +73,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -class TStubTypeVisitor6: public TStubTypeVisitor5 -{ +class TStubTypeVisitor6: public TStubTypeVisitor5 { public: using TStubTypeVisitor5::TStubTypeVisitor5; @@ -88,8 +82,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 44) -class TStubTypeVisitor7: public TStubTypeVisitor6 -{ +class TStubTypeVisitor7: public TStubTypeVisitor6 { public: using TStubTypeVisitor6::TStubTypeVisitor6; @@ -98,52 +91,45 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 44) -class TStubTypeVisitor : public TStubTypeVisitor7 -{ +class TStubTypeVisitor: public TStubTypeVisitor7 { public: - using TBase = TStubTypeVisitor7; + using TBase = TStubTypeVisitor7; TStubTypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -class TStubTypeVisitor : public TStubTypeVisitor6 -{ +class TStubTypeVisitor: public TStubTypeVisitor6 { public: - using TBase = TStubTypeVisitor6; + using TBase = TStubTypeVisitor6; TStubTypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -class TStubTypeVisitor : public TStubTypeVisitor5 -{ +class TStubTypeVisitor: public TStubTypeVisitor5 { public: - using TBase = TStubTypeVisitor5; + using TBase = TStubTypeVisitor5; TStubTypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) -class TStubTypeVisitor : public TStubTypeVisitor4 -{ +class TStubTypeVisitor: public TStubTypeVisitor4 { public: - using TBase = TStubTypeVisitor4; + using TBase = TStubTypeVisitor4; TStubTypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) -class TStubTypeVisitor : public TStubTypeVisitor3 -{ +class TStubTypeVisitor: public TStubTypeVisitor3 { public: - using TBase = TStubTypeVisitor3; + using TBase = TStubTypeVisitor3; TStubTypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13) -class TStubTypeVisitor : public TStubTypeVisitor2 -{ +class TStubTypeVisitor: public TStubTypeVisitor2 { public: - using TBase = TStubTypeVisitor2; + using TBase = TStubTypeVisitor2; TStubTypeVisitor(ui16 compatibilityVersion); }; #else -class TStubTypeVisitor : public TStubTypeVisitor1 -{ +class TStubTypeVisitor: public TStubTypeVisitor1 { public: - using TBase = TStubTypeVisitor1; + using TBase = TStubTypeVisitor1; TStubTypeVisitor(ui16 compatibilityVersion); }; #endif @@ -151,19 +137,24 @@ public: ////////////////////////////////////////////////////////////////////////////// // TDataTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TDataTypeInspector: public TStubTypeVisitor -{ +class TDataTypeInspector: public TStubTypeVisitor { public: TDataTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return TypeId_ != 0; } - TDataTypeId GetTypeId() const { return TypeId_; } + explicit operator bool() const { + return TypeId_ != 0; + } + TDataTypeId GetTypeId() const { + return TypeId_; + } + private: void OnDataType(TDataTypeId typeId) override { TypeId_ = typeId; } #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13) - void OnDecimal(ui8, ui8) override {} + void OnDecimal(ui8, ui8) override { + } #endif TDataTypeId TypeId_ = 0; }; @@ -171,13 +162,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TDataAndDecimalTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TDataAndDecimalTypeInspector: public TStubTypeVisitor -{ +class TDataAndDecimalTypeInspector: public TStubTypeVisitor { public: TDataAndDecimalTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return TypeId_ != 0; } - TDataTypeId GetTypeId() const { return TypeId_; } + explicit operator bool() const { + return TypeId_ != 0; + } + TDataTypeId GetTypeId() const { + return TypeId_; + } ui8 GetPrecision() const { return Precision_; } @@ -203,16 +197,19 @@ private: ////////////////////////////////////////////////////////////////////////////// // TStructTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TStructTypeInspector: public TStubTypeVisitor -{ +class TStructTypeInspector: public TStubTypeVisitor { public: TStructTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return MembersCount_ != Max<ui32>(); } - ui32 GetMembersCount() const { return MembersCount_; } + explicit operator bool() const { + return MembersCount_ != Max<ui32>(); + } + ui32 GetMembersCount() const { + return MembersCount_; + } ui32 GetMemberIndex(TStringRef name) const { ui32 index = 0; - for (TStringRef memberName: MembersNames_) { + for (TStringRef memberName : MembersNames_) { if (memberName == name) { return index; } @@ -220,17 +217,24 @@ public: } return Max<ui32>(); } - const TStringRef& GetMemberName(ui32 i) const { return MembersNames_[i]; } - const TType* GetMemberType(ui32 i) const { return MembersTypes_[i]; } - TStringRef* GetMemberNames() { return MembersNames_.data(); } - const TType** GetMemberTypes() { return MembersTypes_.data(); } + const TStringRef& GetMemberName(ui32 i) const { + return MembersNames_[i]; + } + const TType* GetMemberType(ui32 i) const { + return MembersTypes_[i]; + } + TStringRef* GetMemberNames() { + return MembersNames_.data(); + } + const TType** GetMemberTypes() { + return MembersTypes_.data(); + } private: void OnStruct( - ui32 membersCount, - TStringRef* membersNames, - const TType** membersTypes) override - { + ui32 membersCount, + TStringRef* membersNames, + const TType** membersTypes) override { MembersCount_ = membersCount; MembersNames_.reserve(membersCount); MembersTypes_.reserve(membersCount); @@ -250,13 +254,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TListTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TListTypeInspector: public TStubTypeVisitor -{ +class TListTypeInspector: public TStubTypeVisitor { public: TListTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ItemType_ != nullptr; } - const TType* GetItemType() const { return ItemType_; } + explicit operator bool() const { + return ItemType_ != nullptr; + } + const TType* GetItemType() const { + return ItemType_; + } private: void OnList(const TType* itemType) override { @@ -270,13 +277,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TOptionalTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TOptionalTypeInspector: public TStubTypeVisitor -{ +class TOptionalTypeInspector: public TStubTypeVisitor { public: TOptionalTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ItemType_ != nullptr; } - const TType* GetItemType() const { return ItemType_; } + explicit operator bool() const { + return ItemType_ != nullptr; + } + const TType* GetItemType() const { + return ItemType_; + } private: void OnOptional(const TType* itemType) override { @@ -290,15 +300,22 @@ private: ////////////////////////////////////////////////////////////////////////////// // TTupleTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TTupleTypeInspector: public TStubTypeVisitor -{ +class TTupleTypeInspector: public TStubTypeVisitor { public: TTupleTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ElementsCount_ != Max<ui32>(); } - ui32 GetElementsCount() const { return ElementsCount_; } - const TType* GetElementType(ui32 i) const { return ElementsTypes_[i]; } - const TType** GetElementTypes() { return ElementsTypes_.data(); } + explicit operator bool() const { + return ElementsCount_ != Max<ui32>(); + } + ui32 GetElementsCount() const { + return ElementsCount_; + } + const TType* GetElementType(ui32 i) const { + return ElementsTypes_[i]; + } + const TType** GetElementTypes() { + return ElementsTypes_.data(); + } private: void OnTuple(ui32 elementsCount, const TType** elementsTypes) override { @@ -318,14 +335,19 @@ private: ////////////////////////////////////////////////////////////////////////////// // TDictTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TDictTypeInspector: public TStubTypeVisitor -{ +class TDictTypeInspector: public TStubTypeVisitor { public: TDictTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return KeyType_ != nullptr; } - const TType* GetKeyType() const { return KeyType_; } - const TType* GetValueType() const { return ValueType_; } + explicit operator bool() const { + return KeyType_ != nullptr; + } + const TType* GetKeyType() const { + return KeyType_; + } + const TType* GetValueType() const { + return ValueType_; + } private: void OnDict(const TType* keyType, const TType* valueType) override { @@ -341,27 +363,41 @@ private: ////////////////////////////////////////////////////////////////////////////// // TCallableTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TCallableTypeInspector: public TStubTypeVisitor -{ +class TCallableTypeInspector: public TStubTypeVisitor { public: TCallableTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ReturnType_ != nullptr; } - const TType* GetReturnType() const { return ReturnType_; } - ui32 GetArgsCount() const { return ArgsCount_; } - const TType* GetArgType(ui32 i) const { return ArgsTypes_[i]; } - ui32 GetOptionalArgsCount() const { return OptionalArgsCount_; } - TStringRef GetPayload() const { return HasPayload_ ? Payload_ : TStringRef(); } - TStringRef GetArgumentName(ui32 i) const { return HasPayload_ ? ArgsNames_[i] : TStringRef(); } - ui64 GetArgumentFlags(ui32 i) const { return HasPayload_ ? ArgsFlags_[i] : 0; } + explicit operator bool() const { + return ReturnType_ != nullptr; + } + const TType* GetReturnType() const { + return ReturnType_; + } + ui32 GetArgsCount() const { + return ArgsCount_; + } + const TType* GetArgType(ui32 i) const { + return ArgsTypes_[i]; + } + ui32 GetOptionalArgsCount() const { + return OptionalArgsCount_; + } + TStringRef GetPayload() const { + return HasPayload_ ? Payload_ : TStringRef(); + } + TStringRef GetArgumentName(ui32 i) const { + return HasPayload_ ? ArgsNames_[i] : TStringRef(); + } + ui64 GetArgumentFlags(ui32 i) const { + return HasPayload_ ? ArgsFlags_[i] : 0; + } private: void OnCallable( - const TType* returnType, - ui32 argsCount, const TType** argsTypes, - ui32 optionalArgsCount, - const ICallablePayload* payload) override - { + const TType* returnType, + ui32 argsCount, const TType** argsTypes, + ui32 optionalArgsCount, + const ICallablePayload* payload) override { ReturnType_ = returnType; ArgsCount_ = argsCount; OptionalArgsCount_ = optionalArgsCount; @@ -397,13 +433,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TStreamTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TStreamTypeInspector : public TStubTypeVisitor -{ +class TStreamTypeInspector: public TStubTypeVisitor { public: TStreamTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ItemType_ != nullptr; } - const TType* GetItemType() const { return ItemType_; } + explicit operator bool() const { + return ItemType_ != nullptr; + } + const TType* GetItemType() const { + return ItemType_; + } private: void OnStream(const TType* itemType) override { @@ -417,13 +456,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TVariantTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TVariantTypeInspector : public TStubTypeVisitor -{ +class TVariantTypeInspector: public TStubTypeVisitor { public: TVariantTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return UnderlyingType_ != nullptr; } - const TType* GetUnderlyingType() const { return UnderlyingType_; } + explicit operator bool() const { + return UnderlyingType_ != nullptr; + } + const TType* GetUnderlyingType() const { + return UnderlyingType_; + } private: void OnVariant(const TType* underlyingType) override { @@ -438,13 +480,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TResourceTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TResourceTypeInspector: public TStubTypeVisitor -{ +class TResourceTypeInspector: public TStubTypeVisitor { public: TResourceTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return Tag_.Data() != nullptr; } - TStringRef GetTag() const { return Tag_; } + explicit operator bool() const { + return Tag_.Data() != nullptr; + } + TStringRef GetTag() const { + return Tag_; + } private: void OnResource(TStringRef tag) override { @@ -459,14 +504,19 @@ private: ////////////////////////////////////////////////////////////////////////////// // TTaggedTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TTaggedTypeInspector: public TStubTypeVisitor -{ +class TTaggedTypeInspector: public TStubTypeVisitor { public: TTaggedTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return BaseType_ != nullptr; } - const TType* GetBaseType() const { return BaseType_; } - TStringRef GetTag() const { return Tag_; } + explicit operator bool() const { + return BaseType_ != nullptr; + } + const TType* GetBaseType() const { + return BaseType_; + } + TStringRef GetTag() const { + return Tag_; + } private: void OnTagged(const TType* baseType, TStringRef tag) override { @@ -483,13 +533,16 @@ private: ////////////////////////////////////////////////////////////////////////////// // TPgTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TPgTypeInspector: public TStubTypeVisitor -{ +class TPgTypeInspector: public TStubTypeVisitor { public: TPgTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return TypeId_ != 0; } - ui32 GetTypeId() const { return TypeId_; } + explicit operator bool() const { + return TypeId_ != 0; + } + ui32 GetTypeId() const { + return TypeId_; + } private: void OnPg(ui32 typeId) override { @@ -504,14 +557,19 @@ private: ////////////////////////////////////////////////////////////////////////////// // TBlockTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TBlockTypeInspector: public TStubTypeVisitor -{ +class TBlockTypeInspector: public TStubTypeVisitor { public: TBlockTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ItemType_ != 0; } - const TType* GetItemType() const { return ItemType_; } - bool IsScalar() const { return IsScalar_; } + explicit operator bool() const { + return ItemType_ != 0; + } + const TType* GetItemType() const { + return ItemType_; + } + bool IsScalar() const { + return IsScalar_; + } private: void OnBlock(const TType* itemType, bool isScalar) override { @@ -529,14 +587,19 @@ private: ////////////////////////////////////////////////////////////////////////////// // TLinearTypeInspector ////////////////////////////////////////////////////////////////////////////// -class TLinearTypeInspector: public TStubTypeVisitor -{ +class TLinearTypeInspector: public TStubTypeVisitor { public: TLinearTypeInspector(const ITypeInfoHelper1& typeHelper, const TType* type); - explicit operator bool() const { return ItemType_ != 0; } - const TType* GetItemType() const { return ItemType_; } - bool IsDynamic() const { return IsDynamic_; } + explicit operator bool() const { + return ItemType_ != 0; + } + const TType* GetItemType() const { + return ItemType_; + } + bool IsDynamic() const { + return IsDynamic_; + } private: void OnLinear(const TType* itemType, bool isDynamic) override { @@ -557,11 +620,13 @@ inline void TStubTypeVisitor1::OnDataType(TDataTypeId typeId) } inline void TStubTypeVisitor1::OnStruct( - ui32 membersCount, - TStringRef* membersNames, - const TType** membersTypes) + ui32 membersCount, + TStringRef* membersNames, + const TType** membersTypes) { - Y_UNUSED(membersCount); Y_UNUSED(membersNames); Y_UNUSED(membersTypes); + Y_UNUSED(membersCount); + Y_UNUSED(membersNames); + Y_UNUSED(membersTypes); Y_ABORT("Not implemented"); } @@ -579,24 +644,28 @@ inline void TStubTypeVisitor1::OnOptional(const TType* itemType) inline void TStubTypeVisitor1::OnTuple(ui32 elementsCount, const TType** elementsTypes) { - Y_UNUSED(elementsCount); Y_UNUSED(elementsTypes); + Y_UNUSED(elementsCount); + Y_UNUSED(elementsTypes); Y_ABORT("Not implemented"); } inline void TStubTypeVisitor1::OnDict(const TType* keyType, const TType* valueType) { - Y_UNUSED(keyType); Y_UNUSED(valueType); + Y_UNUSED(keyType); + Y_UNUSED(valueType); Y_ABORT("Not implemented"); } inline void TStubTypeVisitor1::OnCallable( - const TType* returnType, - ui32 argsCount, const TType** argsTypes, - ui32 optionalArgsCount, - const ICallablePayload* payload) -{ - Y_UNUSED(returnType); Y_UNUSED(argsCount); - Y_UNUSED(argsTypes); Y_UNUSED(optionalArgsCount); + const TType* returnType, + ui32 argsCount, const TType** argsTypes, + ui32 optionalArgsCount, + const ICallablePayload* payload) +{ + Y_UNUSED(returnType); + Y_UNUSED(argsCount); + Y_UNUSED(argsTypes); + Y_UNUSED(optionalArgsCount); Y_UNUSED(payload); Y_ABORT("Not implemented"); } @@ -645,5 +714,5 @@ inline void TStubTypeVisitor7::OnLinear(const TType* itemType, bool isDynamic) { } #endif -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_type_ops.h b/yql/essentials/public/udf/udf_type_ops.h index ac848d07c8b..38ed738cd5f 100644 --- a/yql/essentials/public/udf/udf_type_ops.h +++ b/yql/essentials/public/udf/udf_type_ops.h @@ -235,7 +235,7 @@ inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value) { return GetValueHash<EDataSlot::slot>(value); switch (type) { - UDF_TYPE_ID_MAP(HASH_TYPE) + UDF_TYPE_ID_MAP(HASH_TYPE) } #undef HASH_TYPE @@ -259,7 +259,8 @@ inline int CompareFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rh if (std::isunordered(x, y)) { const auto xn = std::isnan(x); const auto yn = std::isnan(y); - return (xn == yn) ? 0 : xn ? 1 : -1; + return (xn == yn) ? 0 : xn ? 1 + : -1; } return (x == y) ? 0 : (x < y ? -1 : 1); } @@ -462,7 +463,7 @@ inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnb return CompareValues<EDataSlot::slot>(lhs, rhs); switch (type) { - UDF_TYPE_ID_MAP(COMPARE_TYPE) + UDF_TYPE_ID_MAP(COMPARE_TYPE) } #undef COMPARE_TYPE @@ -665,7 +666,7 @@ inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnb return EquateValues<EDataSlot::slot>(lhs, rhs); switch (type) { - UDF_TYPE_ID_MAP(EQUATE_TYPE) + UDF_TYPE_ID_MAP(EQUATE_TYPE) } #undef EQUATE_TYPE diff --git a/yql/essentials/public/udf/udf_type_printer.cpp b/yql/essentials/public/udf/udf_type_printer.cpp index fcc804699ea..422ad98be75 100644 --- a/yql/essentials/public/udf/udf_type_printer.cpp +++ b/yql/essentials/public/udf/udf_type_printer.cpp @@ -6,12 +6,11 @@ namespace NYql { namespace NUdf { -class TTypePrinter1 : private TStubTypeVisitor -{ +class TTypePrinter1: private TStubTypeVisitor { public: TTypePrinter1(const ITypeInfoHelper1& typeHelper, const TType* type, ui16 compatibilityVersion); - void Out(IOutputStream &o) const; + void Out(IOutputStream& o) const; protected: void OnDataType(TDataTypeId typeId) final; @@ -40,7 +39,7 @@ protected: }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13) -class TTypePrinter2 : public TTypePrinter1 { +class TTypePrinter2: public TTypePrinter1 { public: using TTypePrinter1::TTypePrinter1; @@ -52,7 +51,7 @@ protected: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) -class TTypePrinter3 : public TTypePrinter2 { +class TTypePrinter3: public TTypePrinter2 { public: using TTypePrinter2::TTypePrinter2; @@ -64,7 +63,7 @@ protected: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) -class TTypePrinter4 : public TTypePrinter3 { +class TTypePrinter4: public TTypePrinter3 { public: using TTypePrinter3::TTypePrinter3; @@ -76,7 +75,7 @@ protected: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -class TTypePrinter5 : public TTypePrinter4 { +class TTypePrinter5: public TTypePrinter4 { public: TTypePrinter5(const ITypeInfoHelper2& typeHelper, const TType* type, ui16 compatibilityVersion); @@ -94,7 +93,7 @@ private: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -class TTypePrinter6 : public TTypePrinter5 { +class TTypePrinter6: public TTypePrinter5 { public: using TTypePrinter5::TTypePrinter5; @@ -109,7 +108,7 @@ private: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 44) -class TTypePrinter7 : public TTypePrinter6 { +class TTypePrinter7: public TTypePrinter6 { public: using TTypePrinter6::TTypePrinter6; @@ -125,10 +124,13 @@ private: TTypePrinter1::TTypePrinter1(const ITypeInfoHelper1& typeHelper, const TType* type, ui16 compatibilityVersion) : TStubTypeVisitor(compatibilityVersion) - , TypeHelper1_(typeHelper), Type_(type), Output_(nullptr) -{} + , TypeHelper1_(typeHelper) + , Type_(type) + , Output_(nullptr) +{ +} -void TTypePrinter1::Out(IOutputStream &o) const { +void TTypePrinter1::Out(IOutputStream& o) const { Output_ = &o; Y_DEFER { Output_ = nullptr; @@ -138,12 +140,23 @@ void TTypePrinter1::Out(IOutputStream &o) const { void TTypePrinter1::OutImpl(const TType* type) const { switch (TypeHelper1_.GetTypeKind(type)) { - case ETypeKind::Null: *Output_ << "Null"; break; - case ETypeKind::Void: *Output_ << "Void"; break; - case ETypeKind::EmptyList: *Output_ << "EmptyList"; break; - case ETypeKind::EmptyDict: *Output_ << "EmptyDict"; break; - case ETypeKind::Unknown: *Output_ << "Unknown"; break; - default: TypeHelper1_.VisitType(type, const_cast<TTypePrinter1*>(this)); + case ETypeKind::Null: + *Output_ << "Null"; + break; + case ETypeKind::Void: + *Output_ << "Void"; + break; + case ETypeKind::EmptyList: + *Output_ << "EmptyList"; + break; + case ETypeKind::EmptyDict: + *Output_ << "EmptyDict"; + break; + case ETypeKind::Unknown: + *Output_ << "Unknown"; + break; + default: + TypeHelper1_.VisitType(type, const_cast<TTypePrinter1*>(this)); } } @@ -161,8 +174,9 @@ void TTypePrinter1::OutStructPayload(ui32 membersCount, TStringRef* membersNames for (ui32 i = 0U; i < membersCount; ++i) { *Output_ << "'" << std::string_view(membersNames[i]) << "':"; OutImpl(membersTypes[i]); - if (i < membersCount - 1U) + if (i < membersCount - 1U) { *Output_ << ','; + } } } @@ -186,8 +200,9 @@ void TTypePrinter1::OnTuple(ui32 elementsCount, const TType** elementsTypes) { void TTypePrinter1::OutTuplePayload(ui32 elementsCount, const TType** elementsTypes) { for (ui32 i = 0U; i < elementsCount; ++i) { OutImpl(elementsTypes[i]); - if (i < elementsCount - 1U) + if (i < elementsCount - 1U) { *Output_ << ','; + } } } @@ -209,20 +224,24 @@ void TTypePrinter1::OnDict(const TType* keyType, const TType* valueType) { void TTypePrinter1::OnCallable(const TType* returnType, ui32 argsCount, const TType** argsTypes, ui32 optionalArgsCount, const ICallablePayload* payload) { *Output_ << "Callable<("; for (ui32 i = 0U; i < argsCount; ++i) { - if (optionalArgsCount && i == argsCount - optionalArgsCount) + if (optionalArgsCount && i == argsCount - optionalArgsCount) { *Output_ << '['; + } if (payload) { const std::string_view name = payload->GetArgumentName(i); - if (!name.empty()) + if (!name.empty()) { *Output_ << "'" << name << "':"; + } } OutImpl(argsTypes[i]); if (payload) { - if (ICallablePayload::TArgumentFlags::AutoMap == payload->GetArgumentFlags(i)) + if (ICallablePayload::TArgumentFlags::AutoMap == payload->GetArgumentFlags(i)) { *Output_ << "{Flags:AutoMap}"; + } } - if (i < argsCount - 1U) + if (i < argsCount - 1U) { *Output_ << ','; + } } *Output_ << (optionalArgsCount ? "])->" : ")->"); OutImpl(returnType); @@ -231,29 +250,29 @@ void TTypePrinter1::OnCallable(const TType* returnType, ui32 argsCount, const TT void TTypePrinter1::OnVariant(const TType* underlyingType) { switch (TypeHelper1_.GetTypeKind(underlyingType)) { - case ETypeKind::Struct: { - TStructTypeInspector s(TypeHelper1_, underlyingType); - const bool isEnum = std::all_of(s.GetMemberTypes(), s.GetMemberTypes() + s.GetMembersCount(), [this](auto memberType) { - return TypeHelper1_.GetTypeKind(memberType) == ETypeKind::Void; - }); - - if (isEnum) { - *Output_ << "Enum<"; - OutEnumValues(s.GetMembersCount(), s.GetMemberNames()); - } else { + case ETypeKind::Struct: { + TStructTypeInspector s(TypeHelper1_, underlyingType); + const bool isEnum = std::all_of(s.GetMemberTypes(), s.GetMemberTypes() + s.GetMembersCount(), [this](auto memberType) { + return TypeHelper1_.GetTypeKind(memberType) == ETypeKind::Void; + }); + + if (isEnum) { + *Output_ << "Enum<"; + OutEnumValues(s.GetMembersCount(), s.GetMemberNames()); + } else { + *Output_ << "Variant<"; + OutStructPayload(s.GetMembersCount(), s.GetMemberNames(), s.GetMemberTypes()); + } + break; + } + case ETypeKind::Tuple: { + TTupleTypeInspector s(TypeHelper1_, underlyingType); *Output_ << "Variant<"; - OutStructPayload(s.GetMembersCount(), s.GetMemberNames(), s.GetMemberTypes()); + OutTuplePayload(s.GetElementsCount(), s.GetElementTypes()); + break; } - break; - } - case ETypeKind::Tuple: { - TTupleTypeInspector s(TypeHelper1_, underlyingType); - *Output_ << "Variant<"; - OutTuplePayload(s.GetElementsCount(), s.GetElementTypes()); - break; - } - default: - Y_ABORT_UNLESS(false, "Unexpected underlying type in Variant"); + default: + Y_ABORT_UNLESS(false, "Unexpected underlying type in Variant"); } *Output_ << '>'; } @@ -261,8 +280,9 @@ void TTypePrinter1::OnVariant(const TType* underlyingType) { void TTypePrinter1::OutEnumValues(ui32 membersCount, TStringRef* membersNames) { for (ui32 i = 0U; i < membersCount; ++i) { *Output_ << "'" << std::string_view(membersNames[i]) << '\''; - if (i < membersCount - 1U) + if (i < membersCount - 1U) { *Output_ << ','; + } } } @@ -289,7 +309,8 @@ void TTypePrinter1::OnTaggedImpl(const TType* baseType, TStringRef tag) { TTypePrinter5::TTypePrinter5(const ITypeInfoHelper2& typeHelper, const TType* type, ui16 compatibilityVersion) : TTypePrinter4(typeHelper, type, compatibilityVersion) , TypeHelper2_(typeHelper) -{} +{ +} void TTypePrinter5::OnPgImpl(ui32 typeId) { auto* description = TypeHelper2_.FindPgTypeDescription(typeId); @@ -317,10 +338,10 @@ void TTypePrinter7::OnLinearImpl(const TType* itemType, bool isDynamic) { *Output_ << '>'; } -void TTypePrinter::Out(IOutputStream &o) const { +void TTypePrinter::Out(IOutputStream& o) const { TTypePrinter7 p(TypeHelper_, Type_, UDF_ABI_COMPATIBILITY_VERSION(2, 44)); p.Out(o); } -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_type_printer.h b/yql/essentials/public/udf/udf_type_printer.h index ea7c79ec5e7..436ff415efa 100644 --- a/yql/essentials/public/udf/udf_type_printer.h +++ b/yql/essentials/public/udf/udf_type_printer.h @@ -7,8 +7,7 @@ namespace NYql { namespace NUdf { -class TTypePrinter -{ +class TTypePrinter { public: #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) TTypePrinter(const ITypeInfoHelper2& typeHelper, const TType* type) @@ -17,9 +16,10 @@ public: #endif : TypeHelper_(typeHelper) , Type_(type) - {} + { + } - void Out(IOutputStream &o) const; + void Out(IOutputStream& o) const; private: #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) @@ -30,5 +30,5 @@ private: const TType* Type_; }; -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_type_printer_ut.cpp b/yql/essentials/public/udf/udf_type_printer_ut.cpp index 39e0fd7d5f3..886e98b11b4 100644 --- a/yql/essentials/public/udf/udf_type_printer_ut.cpp +++ b/yql/essentials/public/udf/udf_type_printer_ut.cpp @@ -6,26 +6,26 @@ using namespace NYql::NUdf; Y_UNIT_TEST_SUITE(TUdfTypePrinter) { - template <typename F> - void TestType(const TString& expected, F&& f) { - NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__); - NKikimr::NMiniKQL::TTypeEnvironment env(alloc); - NKikimr::NMiniKQL::TTypeBuilder typeBuilder(env); - NKikimr::NMiniKQL::TTypeInfoHelper typeHelper; +template <typename F> +void TestType(const TString& expected, F&& f) { + NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__); + NKikimr::NMiniKQL::TTypeEnvironment env(alloc); + NKikimr::NMiniKQL::TTypeBuilder typeBuilder(env); + NKikimr::NMiniKQL::TTypeInfoHelper typeHelper; - auto type = f(typeBuilder); - TStringStream stream; - TTypePrinter(typeHelper, type).Out(stream); - UNIT_ASSERT_STRINGS_EQUAL(stream.Str(), expected); - } + auto type = f(typeBuilder); + TStringStream stream; + TTypePrinter(typeHelper, type).Out(stream); + UNIT_ASSERT_STRINGS_EQUAL(stream.Str(), expected); +} - Y_UNIT_TEST(TestLinear) { - TestType("Linear<Int32>", [](const NKikimr::NMiniKQL::TTypeBuilder& typeBuilder) { - return typeBuilder.NewLinearType(typeBuilder.NewDataType(EDataSlot::Int32), false); - }); +Y_UNIT_TEST(TestLinear) { + TestType("Linear<Int32>", [](const NKikimr::NMiniKQL::TTypeBuilder& typeBuilder) { + return typeBuilder.NewLinearType(typeBuilder.NewDataType(EDataSlot::Int32), false); + }); - TestType("DynamicLinear<Int32>", [](const NKikimr::NMiniKQL::TTypeBuilder& typeBuilder) { - return typeBuilder.NewLinearType(typeBuilder.NewDataType(EDataSlot::Int32), true); - }); - } + TestType("DynamicLinear<Int32>", [](const NKikimr::NMiniKQL::TTypeBuilder& typeBuilder) { + return typeBuilder.NewLinearType(typeBuilder.NewDataType(EDataSlot::Int32), true); + }); } +} // Y_UNIT_TEST_SUITE(TUdfTypePrinter) diff --git a/yql/essentials/public/udf/udf_type_size_check.h b/yql/essentials/public/udf/udf_type_size_check.h index 1433abf3620..0c80b8de405 100644 --- a/yql/essentials/public/udf/udf_type_size_check.h +++ b/yql/essentials/public/udf/udf_type_size_check.h @@ -1,7 +1,6 @@ #pragma once -#define UDF_ASSERT_TYPE_SIZE(type, expectedSize) \ - static_assert(sizeof(type) <= (expectedSize), \ - "Size of " #type " mismatches expected size. " \ - "Expected size is " #expectedSize) - +#define UDF_ASSERT_TYPE_SIZE(type, expectedSize) \ + static_assert(sizeof(type) <= (expectedSize), \ + "Size of " #type " mismatches expected size. " \ + "Expected size is " #expectedSize) diff --git a/yql/essentials/public/udf/udf_types.cpp b/yql/essentials/public/udf/udf_types.cpp index 1c61fd2713c..dfba87785a7 100644 --- a/yql/essentials/public/udf/udf_types.cpp +++ b/yql/essentials/public/udf/udf_types.cpp @@ -5,10 +5,12 @@ namespace NUdf { ITypeVisitor::ITypeVisitor(ui16 compatibilityVersion) : TBase(compatibilityVersion) -{} +{ +} ITypeInfoHelper::ITypeInfoHelper() -{} +{ +} -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_types.h b/yql/essentials/public/udf/udf_types.h index ba889feabd0..8a9fb51280f 100644 --- a/yql/essentials/public/udf/udf_types.h +++ b/yql/essentials/public/udf/udf_types.h @@ -16,128 +16,127 @@ using TType = void; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 44) -#define UDF_TYPE_KIND_MAP(XX) \ - XX(Unknown) \ - XX(Data) \ - XX(Struct) \ - XX(List) \ - XX(Optional) \ - XX(Tuple) \ - XX(Dict) \ - XX(Callable) \ - XX(Resource) \ - XX(Void) \ - XX(Variant) \ - XX(Stream) \ - XX(Null) \ - XX(EmptyList) \ - XX(EmptyDict) \ - XX(Tagged) \ - XX(Pg) \ - XX(Block) \ - XX(Linear) + #define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) \ + XX(Null) \ + XX(EmptyList) \ + XX(EmptyDict) \ + XX(Tagged) \ + XX(Pg) \ + XX(Block) \ + XX(Linear) #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -#define UDF_TYPE_KIND_MAP(XX) \ - XX(Unknown) \ - XX(Data) \ - XX(Struct) \ - XX(List) \ - XX(Optional) \ - XX(Tuple) \ - XX(Dict) \ - XX(Callable) \ - XX(Resource) \ - XX(Void) \ - XX(Variant) \ - XX(Stream) \ - XX(Null) \ - XX(EmptyList) \ - XX(EmptyDict) \ - XX(Tagged) \ - XX(Pg) \ - XX(Block) + #define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) \ + XX(Null) \ + XX(EmptyList) \ + XX(EmptyDict) \ + XX(Tagged) \ + XX(Pg) \ + XX(Block) #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -#define UDF_TYPE_KIND_MAP(XX) \ - XX(Unknown) \ - XX(Data) \ - XX(Struct) \ - XX(List) \ - XX(Optional) \ - XX(Tuple) \ - XX(Dict) \ - XX(Callable) \ - XX(Resource) \ - XX(Void) \ - XX(Variant) \ - XX(Stream) \ - XX(Null) \ - XX(EmptyList) \ - XX(EmptyDict) \ - XX(Tagged) \ - XX(Pg) + #define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) \ + XX(Null) \ + XX(EmptyList) \ + XX(EmptyDict) \ + XX(Tagged) \ + XX(Pg) #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) -#define UDF_TYPE_KIND_MAP(XX) \ - XX(Unknown) \ - XX(Data) \ - XX(Struct) \ - XX(List) \ - XX(Optional) \ - XX(Tuple) \ - XX(Dict) \ - XX(Callable) \ - XX(Resource) \ - XX(Void) \ - XX(Variant) \ - XX(Stream) \ - XX(Null) \ - XX(EmptyList) \ - XX(EmptyDict) \ - XX(Tagged) + #define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) \ + XX(Null) \ + XX(EmptyList) \ + XX(EmptyDict) \ + XX(Tagged) #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 18) -#define UDF_TYPE_KIND_MAP(XX) \ - XX(Unknown) \ - XX(Data) \ - XX(Struct) \ - XX(List) \ - XX(Optional) \ - XX(Tuple) \ - XX(Dict) \ - XX(Callable) \ - XX(Resource) \ - XX(Void) \ - XX(Variant) \ - XX(Stream) \ - XX(Null) \ - XX(EmptyList) \ - XX(EmptyDict) + #define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) \ + XX(Null) \ + XX(EmptyList) \ + XX(EmptyDict) #else -#define UDF_TYPE_KIND_MAP(XX) \ - XX(Unknown) \ - XX(Data) \ - XX(Struct) \ - XX(List) \ - XX(Optional) \ - XX(Tuple) \ - XX(Dict) \ - XX(Callable) \ - XX(Resource) \ - XX(Void) \ - XX(Variant) \ - XX(Stream) + #define UDF_TYPE_KIND_MAP(XX) \ + XX(Unknown) \ + XX(Data) \ + XX(Struct) \ + XX(List) \ + XX(Optional) \ + XX(Tuple) \ + XX(Dict) \ + XX(Callable) \ + XX(Resource) \ + XX(Void) \ + XX(Variant) \ + XX(Stream) #endif -enum ETypeKind -{ +enum ETypeKind { UDF_TYPE_KIND_MAP(ENUM_VALUE_GEN_NO_VALUE) }; @@ -146,8 +145,7 @@ ENUM_TO_STRING(ETypeKind, UDF_TYPE_KIND_MAP) ////////////////////////////////////////////////////////////////////////////// // ICallablePayload ////////////////////////////////////////////////////////////////////////////// -class ICallablePayload -{ +class ICallablePayload { public: virtual ~ICallablePayload() = default; @@ -170,12 +168,13 @@ UDF_ASSERT_TYPE_SIZE(ICallablePayload, 8); ////////////////////////////////////////////////////////////////////////////// // ITypeVisitor ////////////////////////////////////////////////////////////////////////////// -class ITypeVisitor1 -{ +class ITypeVisitor1 { protected: ITypeVisitor1(ui16 compatibilityVersion) : AbiCompatibility_(compatibilityVersion) - {} + { + } + public: inline bool IsCompatibleTo(ui16 compatibilityVersion) const { return AbiCompatibility_ >= compatibilityVersion; @@ -185,19 +184,20 @@ public: virtual void OnDataType(TDataTypeId typeId) = 0; virtual void OnStruct( - ui32 membersCount, - TStringRef* membersNames, - const TType** membersTypes) = 0; + ui32 membersCount, + TStringRef* membersNames, + const TType** membersTypes) = 0; virtual void OnList(const TType* itemType) = 0; virtual void OnOptional(const TType* itemType) = 0; virtual void OnTuple(ui32 elementsCount, const TType** elementsTypes) = 0; virtual void OnDict(const TType* keyType, const TType* valueType) = 0; virtual void OnCallable( - const TType* returnType, - ui32 argsCount, const TType** argsTypes, - ui32 optionalArgsCount, const ICallablePayload* payload) = 0; + const TType* returnType, + ui32 argsCount, const TType** argsTypes, + ui32 optionalArgsCount, const ICallablePayload* payload) = 0; virtual void OnVariant(const TType* underlyingType) = 0; virtual void OnStream(const TType* itemType) = 0; + private: ui16 AbiCompatibility_ = 0; ui16 Reserved1_ = 0; @@ -258,43 +258,43 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 44) -class ITypeVisitor : public ITypeVisitor7 { +class ITypeVisitor: public ITypeVisitor7 { protected: using TBase = ITypeVisitor7; ITypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -class ITypeVisitor : public ITypeVisitor6 { +class ITypeVisitor: public ITypeVisitor6 { protected: using TBase = ITypeVisitor6; ITypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -class ITypeVisitor : public ITypeVisitor5 { +class ITypeVisitor: public ITypeVisitor5 { protected: using TBase = ITypeVisitor5; ITypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 21) -class ITypeVisitor : public ITypeVisitor4 { +class ITypeVisitor: public ITypeVisitor4 { protected: using TBase = ITypeVisitor4; ITypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 15) -class ITypeVisitor : public ITypeVisitor3 { +class ITypeVisitor: public ITypeVisitor3 { protected: using TBase = ITypeVisitor3; ITypeVisitor(ui16 compatibilityVersion); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 13) -class ITypeVisitor : public ITypeVisitor2 { +class ITypeVisitor: public ITypeVisitor2 { protected: using TBase = ITypeVisitor2; ITypeVisitor(ui16 compatibilityVersion); }; #else -class ITypeVisitor : public ITypeVisitor1 { +class ITypeVisitor: public ITypeVisitor1 { protected: using TBase = ITypeVisitor1; ITypeVisitor(ui16 compatibilityVersion); @@ -306,8 +306,7 @@ UDF_ASSERT_TYPE_SIZE(ITypeVisitor, 16); ////////////////////////////////////////////////////////////////////////////// // ITypeInfoHelper ////////////////////////////////////////////////////////////////////////////// -class ITypeInfoHelper1 -{ +class ITypeInfoHelper1 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper1>; @@ -344,7 +343,7 @@ private: }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -class ITypeInfoHelper2 : public ITypeInfoHelper1 { +class ITypeInfoHelper2: public ITypeInfoHelper1 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper2>; @@ -356,8 +355,7 @@ public: ////////////////////////////////////////////////////////////////////////////// // IArrowType ////////////////////////////////////////////////////////////////////////////// -class IArrowType -{ +class IArrowType { public: using TPtr = TUniquePtr<IArrowType>; @@ -369,7 +367,7 @@ public: UDF_ASSERT_TYPE_SIZE(IArrowType, 8); #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -class ITypeInfoHelper3 : public ITypeInfoHelper2 { +class ITypeInfoHelper3: public ITypeInfoHelper2 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper3>; @@ -382,7 +380,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 29) -class ITypeInfoHelper4 : public ITypeInfoHelper3 { +class ITypeInfoHelper4: public ITypeInfoHelper3 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper4>; @@ -393,25 +391,25 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 29) -class ITypeInfoHelper : public ITypeInfoHelper4 { +class ITypeInfoHelper: public ITypeInfoHelper4 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper>; ITypeInfoHelper(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 26) -class ITypeInfoHelper : public ITypeInfoHelper3 { +class ITypeInfoHelper: public ITypeInfoHelper3 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper>; ITypeInfoHelper(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 25) -class ITypeInfoHelper : public ITypeInfoHelper2 { +class ITypeInfoHelper: public ITypeInfoHelper2 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper>; ITypeInfoHelper(); }; #else -class ITypeInfoHelper : public ITypeInfoHelper1 { +class ITypeInfoHelper: public ITypeInfoHelper1 { public: using TPtr = TRefCountedPtr<ITypeInfoHelper>; ITypeInfoHelper(); @@ -421,5 +419,5 @@ public: UDF_ASSERT_TYPE_SIZE(ITypeInfoHelper, 16); UDF_ASSERT_TYPE_SIZE(ITypeInfoHelper::TPtr, 8); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_ut_helpers.h b/yql/essentials/public/udf/udf_ut_helpers.h index af95e727dc3..fb56eae790c 100644 --- a/yql/essentials/public/udf/udf_ut_helpers.h +++ b/yql/essentials/public/udf/udf_ut_helpers.h @@ -12,11 +12,13 @@ namespace NYql { namespace NUdf { -template<bool HasLength = true> +template <bool HasLength = true> class TLazyList: public NUdf::TBoxedValue { struct TIterator: public NUdf::TBoxedValue { TIterator(i32 from, i32 to) - : From_(from), To_(to), Curr_(Max<i32>()) + : From_(from) + , To_(to) + , Curr_(Max<i32>()) { if (To_ >= From_) { To_--; // exclude last @@ -24,6 +26,7 @@ class TLazyList: public NUdf::TBoxedValue { From_--; // exclude first } } + private: bool Skip() override { if (Curr_ == Max<i32>()) { @@ -45,16 +48,19 @@ class TLazyList: public NUdf::TBoxedValue { } bool Next(NUdf::TUnboxedValue& value) override { - if (!Skip()) + if (!Skip()) { return false; + } value = NUdf::TUnboxedValuePod(Curr_); return true; } i32 From_, To_, Curr_; }; + public: TLazyList(i32 from, i32 to) - : From_(from), To_(to) + : From_(from) + , To_(to) { } @@ -64,8 +70,9 @@ private: } ui64 GetListLength() const override { - if (HasLength) + if (HasLength) { return Abs(To_ - From_); + } Y_ABORT("No length!"); } @@ -109,5 +116,5 @@ private: i32 From_, To_; }; -} -} +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_validate.cpp b/yql/essentials/public/udf/udf_validate.cpp index d6fe5524f9a..10858ba09fd 100644 --- a/yql/essentials/public/udf/udf_validate.cpp +++ b/yql/essentials/public/udf/udf_validate.cpp @@ -6,14 +6,14 @@ namespace NYql { namespace NUdf { #define SWITCH_ENUM_TYPE_TO_STR(name, val) \ - case val: return TStringBuf(#name); + case val: \ + return TStringBuf(#name); TString ValidateModeAvailables() { return Join(", ", - ValidateModeAsStr(EValidateMode::None), - ValidateModeAsStr(EValidateMode::Lazy), - ValidateModeAsStr(EValidateMode::Greedy) - ); + ValidateModeAsStr(EValidateMode::None), + ValidateModeAsStr(EValidateMode::Lazy), + ValidateModeAsStr(EValidateMode::Greedy)); } TStringBuf ValidateModeAsStr(EValidateMode validateMode) { @@ -63,5 +63,5 @@ EValidateDatumMode ToDatumValidateMode(EValidateMode validateMode) { return EValidateDatumMode::Expensive; } } -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_validate.h b/yql/essentials/public/udf/udf_validate.h index 361615770df..306b301fd0f 100644 --- a/yql/essentials/public/udf/udf_validate.h +++ b/yql/essentials/public/udf/udf_validate.h @@ -7,14 +7,14 @@ namespace NYql { namespace NUdf { #define UDF_VALIDATE_MODE(XX) \ - XX(None, 0) \ - XX(Lazy, 1) \ - XX(Greedy, 2) \ - XX(Max, 3) \ + XX(None, 0) \ + XX(Lazy, 1) \ + XX(Greedy, 2) \ + XX(Max, 3) #define UDF_VALIDATE_POLICY(XX) \ - XX(Fail, 0) \ - XX(Exception, 1) \ + XX(Fail, 0) \ + XX(Exception, 1) \ XX(Max, 2) enum class EValidateDatumMode { @@ -23,11 +23,11 @@ enum class EValidateDatumMode { Expensive, }; -enum class EValidateMode : ui8 { +enum class EValidateMode: ui8 { UDF_VALIDATE_MODE(ENUM_VALUE_GEN) }; -enum class EValidatePolicy : ui8 { +enum class EValidatePolicy: ui8 { UDF_VALIDATE_POLICY(ENUM_VALUE_GEN) }; @@ -38,5 +38,5 @@ TStringBuf ValidatePolicyAsStr(EValidatePolicy verifyPolicy); EValidatePolicy ValidatePolicyByStr(const TString& verifyPolicy); EValidateDatumMode ToDatumValidateMode(EValidateMode validateMode); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_value.cpp b/yql/essentials/public/udf/udf_value.cpp index 8d8add7393e..c2c096a1c0a 100644 --- a/yql/essentials/public/udf/udf_value.cpp +++ b/yql/essentials/public/udf/udf_value.cpp @@ -4,7 +4,8 @@ namespace NYql { namespace NUdf { IBoxedValue::IBoxedValue() -{} - +{ } -}
\ No newline at end of file + +} // namespace NUdf +} // namespace NYql diff --git a/yql/essentials/public/udf/udf_value.h b/yql/essentials/public/udf/udf_value.h index a895d0f8ea7..4b3e3cfbe55 100644 --- a/yql/essentials/public/udf/udf_value.h +++ b/yql/essentials/public/udf/udf_value.h @@ -8,10 +8,10 @@ #include <yql/essentials/public/decimal/yql_decimal.h> -#include <util/system/yassert.h> // FAIL, VERIFY_DEBUG -#include <util/generic/utility.h> // Min, Max +#include <util/system/yassert.h> // FAIL, VERIFY_DEBUG +#include <util/generic/utility.h> // Min, Max #include <util/generic/yexception.h> // Y_ENSURE -#include <util/system/compiler.h> // Y_FORCE_INLINE +#include <util/system/compiler.h> // Y_FORCE_INLINE #include <algorithm> #include <type_traits> @@ -26,7 +26,7 @@ class TUnboxedValuePod; class TOpaqueListRepresentation; class IValueBuilder; -enum class EFetchStatus : ui32 { +enum class EFetchStatus: ui32 { Ok, Finish, Yield @@ -48,9 +48,9 @@ UDF_ASSERT_TYPE_SIZE(IApplyContext, 8); class IBoxedValue; using IBoxedValuePtr = TRefCountedPtr<IBoxedValue>; -class IBoxedValue1 -{ -friend struct TBoxedValueAccessor; +class IBoxedValue1 { + friend struct TBoxedValueAccessor; + public: inline bool IsCompatibleTo(ui16 compatibilityVersion) const { return AbiCompatibility_ >= compatibilityVersion; @@ -79,7 +79,7 @@ private: // Dict accessors virtual ui64 GetDictLength() const = 0; virtual TUnboxedValue GetDictIterator() const = 0; - virtual TUnboxedValue GetKeysIterator() const = 0; // May return empty. + virtual TUnboxedValue GetKeysIterator() const = 0; // May return empty. virtual TUnboxedValue GetPayloadsIterator() const = 0; // May return empty. virtual bool Contains(const TUnboxedValuePod& key) const = 0; virtual TUnboxedValue Lookup(const TUnboxedValuePod& key) const = 0; @@ -133,8 +133,9 @@ private: }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) -class IBoxedValue2 : public IBoxedValue1 { -friend struct TBoxedValueAccessor; +class IBoxedValue2: public IBoxedValue1 { + friend struct TBoxedValueAccessor; + private: // Save/Load state virtual ui32 GetTraverseCount() const = 0; @@ -145,24 +146,27 @@ private: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) -class IBoxedValue3 : public IBoxedValue2 { -friend struct TBoxedValueAccessor; +class IBoxedValue3: public IBoxedValue2 { + friend struct TBoxedValueAccessor; + private: virtual void Push(const TUnboxedValuePod& value) = 0; }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12) -class IBoxedValue4 : public IBoxedValue3 { -friend struct TBoxedValueAccessor; +class IBoxedValue4: public IBoxedValue3 { + friend struct TBoxedValueAccessor; + private: virtual bool IsSortedDict() const = 0; }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) -class IBoxedValue5 : public IBoxedValue4 { -friend struct TBoxedValueAccessor; +class IBoxedValue5: public IBoxedValue4 { + friend struct TBoxedValueAccessor; + private: virtual void Unused1() = 0; virtual void Unused2() = 0; @@ -174,53 +178,55 @@ private: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 30) -class IBoxedValue6 : public IBoxedValue5 { -friend struct TBoxedValueAccessor; +class IBoxedValue6: public IBoxedValue5 { + friend struct TBoxedValueAccessor; + private: virtual EFetchStatus WideFetch(TUnboxedValue* result, ui32 width) = 0; }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 36) -class IBoxedValue7 : public IBoxedValue6 { -friend struct TBoxedValueAccessor; +class IBoxedValue7: public IBoxedValue6 { + friend struct TBoxedValueAccessor; + private: virtual bool Load2(const TUnboxedValue& state) = 0; }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 36) -class IBoxedValue : public IBoxedValue7 { +class IBoxedValue: public IBoxedValue7 { protected: IBoxedValue(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 30) -class IBoxedValue : public IBoxedValue6 { +class IBoxedValue: public IBoxedValue6 { protected: IBoxedValue(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) -class IBoxedValue : public IBoxedValue5 { +class IBoxedValue: public IBoxedValue5 { protected: IBoxedValue(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12) -class IBoxedValue : public IBoxedValue4 { +class IBoxedValue: public IBoxedValue4 { protected: IBoxedValue(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) -class IBoxedValue : public IBoxedValue3 { +class IBoxedValue: public IBoxedValue3 { protected: IBoxedValue(); }; #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) -class IBoxedValue : public IBoxedValue2 { +class IBoxedValue: public IBoxedValue2 { protected: IBoxedValue(); }; #else -class IBoxedValue : public IBoxedValue1 { +class IBoxedValue: public IBoxedValue1 { protected: IBoxedValue(); }; @@ -233,302 +239,304 @@ UDF_ASSERT_TYPE_SIZE(IBoxedValuePtr, 8); /////////////////////////////////////////////////////////////////////////////// // TBoxedValueAccessor /////////////////////////////////////////////////////////////////////////////// -struct TBoxedValueAccessor -{ +struct TBoxedValueAccessor { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 36) - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ - xx(Load) \ - xx(Push) \ - xx(IsSortedDict) \ - xx(Unused1) \ - xx(Unused2) \ - xx(Unused3) \ - xx(Unused4) \ - xx(Unused5) \ - xx(Unused6) \ - xx(WideFetch) \ - xx(Load2) + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ + xx(Load) \ + xx(Push) \ + xx(IsSortedDict) \ + xx(Unused1) \ + xx(Unused2) \ + xx(Unused3) \ + xx(Unused4) \ + xx(Unused5) \ + xx(Unused6) \ + xx(WideFetch) \ + xx(Load2) + // clang-format on #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 30) - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ - xx(Load) \ - xx(Push) \ - xx(IsSortedDict) \ - xx(Unused1) \ - xx(Unused2) \ - xx(Unused3) \ - xx(Unused4) \ - xx(Unused5) \ - xx(Unused6) \ - xx(WideFetch) \ + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ + xx(Load) \ + xx(Push) \ + xx(IsSortedDict) \ + xx(Unused1) \ + xx(Unused2) \ + xx(Unused3) \ + xx(Unused4) \ + xx(Unused5) \ + xx(Unused6) \ + xx(WideFetch) + // clang-format on #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ - xx(Load) \ - xx(Push) \ - xx(IsSortedDict) \ - xx(Unused1) \ - xx(Unused2) \ - xx(Unused3) \ - xx(Unused4) \ - xx(Unused5) \ - xx(Unused6) + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ + xx(Load) \ + xx(Push) \ + xx(IsSortedDict) \ + xx(Unused1) \ + xx(Unused2) \ + xx(Unused3) \ + xx(Unused4) \ + xx(Unused5) \ + xx(Unused6) + // clang-format on #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12) - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ - xx(Load) \ - xx(Push) \ - xx(IsSortedDict) - + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ + xx(Load) \ + xx(Push) \ + xx(IsSortedDict) + // clang-format on #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ - xx(Load) \ - xx(Push) - + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ + xx(Load) \ + xx(Push) + // clang-format on #elif UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) \ - xx(GetTraverseCount) \ - xx(GetTraverseItem) \ - xx(Save) \ - xx(Load) - + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) \ + xx(GetTraverseCount) \ + xx(GetTraverseItem) \ + xx(Save) \ + xx(Load) + // clang-format on #else - -#define METHOD_MAP(xx) \ - xx(HasFastListLength) \ - xx(GetListLength) \ - xx(GetEstimatedListLength) \ - xx(GetListIterator) \ - xx(GetListRepresentation) \ - xx(ReverseListImpl) \ - xx(SkipListImpl) \ - xx(TakeListImpl) \ - xx(ToIndexDictImpl) \ - xx(GetDictLength) \ - xx(GetDictIterator) \ - xx(GetKeysIterator) \ - xx(GetPayloadsIterator) \ - xx(Contains) \ - xx(Lookup) \ - xx(GetElement) \ - xx(GetElements) \ - xx(Run) \ - xx(GetResourceTag) \ - xx(GetResource) \ - xx(HasListItems) \ - xx(HasDictItems) \ - xx(GetVariantIndex) \ - xx(GetVariantItem) \ - xx(Fetch) \ - xx(Skip) \ - xx(Next) \ - xx(NextPair) \ - xx(Apply) - + // clang-format off + #define METHOD_MAP(xx) \ + xx(HasFastListLength) \ + xx(GetListLength) \ + xx(GetEstimatedListLength) \ + xx(GetListIterator) \ + xx(GetListRepresentation) \ + xx(ReverseListImpl) \ + xx(SkipListImpl) \ + xx(TakeListImpl) \ + xx(ToIndexDictImpl) \ + xx(GetDictLength) \ + xx(GetDictIterator) \ + xx(GetKeysIterator) \ + xx(GetPayloadsIterator) \ + xx(Contains) \ + xx(Lookup) \ + xx(GetElement) \ + xx(GetElements) \ + xx(Run) \ + xx(GetResourceTag) \ + xx(GetResource) \ + xx(HasListItems) \ + xx(HasDictItems) \ + xx(GetVariantIndex) \ + xx(GetVariantItem) \ + xx(Fetch) \ + xx(Skip) \ + xx(Next) \ + xx(NextPair) \ + xx(Apply) + // clang-format on #endif - enum class EMethod : ui32 { + enum class EMethod: ui32 { #define MAP_HANDLER(xx) xx, METHOD_MAP(MAP_HANDLER) #undef MAP_HANDLER }; - template<typename Method> + template <typename Method> static uintptr_t GetMethodPtr(Method method) { uintptr_t ret; memcpy(&ret, &method, sizeof(uintptr_t)); @@ -537,7 +545,9 @@ struct TBoxedValueAccessor static uintptr_t GetMethodPtr(EMethod method) { switch (method) { -#define MAP_HANDLER(xx) case EMethod::xx: return GetMethodPtr(&IBoxedValue::xx); +#define MAP_HANDLER(xx) \ + case EMethod::xx: \ + return GetMethodPtr(&IBoxedValue::xx); METHOD_MAP(MAP_HANDLER) #undef MAP_HANDLER } @@ -545,7 +555,8 @@ struct TBoxedValueAccessor Y_ABORT("unknown method"); } - template<EMethod Method> static uintptr_t GetMethodPtr(); + template <EMethod Method> + static uintptr_t GetMethodPtr(); // List accessors static inline bool HasFastListLength(const IBoxedValue& value); @@ -621,7 +632,11 @@ struct TBoxedValueAccessor #endif }; -#define MAP_HANDLER(xx) template<> inline uintptr_t TBoxedValueAccessor::GetMethodPtr<TBoxedValueAccessor::EMethod::xx>() { return GetMethodPtr(&IBoxedValue::xx); } +#define MAP_HANDLER(xx) \ + template <> \ + inline uintptr_t TBoxedValueAccessor::GetMethodPtr<TBoxedValueAccessor::EMethod::xx>() { \ + return GetMethodPtr(&IBoxedValue::xx); \ + } METHOD_MAP(MAP_HANDLER) #undef MAP_HANDLER @@ -713,28 +728,32 @@ private: #endif }; -class TBoxedValueLink: public TBoxedValueBase -{ +class TBoxedValueLink: public TBoxedValueBase { public: void Link(TBoxedValueLink* root); void Unlink(); - void InitLinks() { Left_ = Right_ = this; } - TBoxedValueLink* GetLeft() const { return Left_; } - TBoxedValueLink* GetRight() const { return Right_; } + void InitLinks() { + Left_ = Right_ = this; + } + TBoxedValueLink* GetLeft() const { + return Left_; + } + TBoxedValueLink* GetRight() const { + return Right_; + } + private: - TBoxedValueLink *Left_ = nullptr; - TBoxedValueLink *Right_ = nullptr; + TBoxedValueLink* Left_ = nullptr; + TBoxedValueLink* Right_ = nullptr; }; -class TBoxedValue: public TBoxedValueLink, public TWithUdfAllocator -{ +class TBoxedValue: public TBoxedValueLink, public TWithUdfAllocator { public: TBoxedValue(); ~TBoxedValue(); }; -class TManagedBoxedValue: public TBoxedValueBase, public TWithUdfAllocator -{ +class TManagedBoxedValue: public TBoxedValueBase, public TWithUdfAllocator { }; UDF_ASSERT_TYPE_SIZE(TBoxedValue, 32); @@ -769,11 +788,11 @@ struct TRawStringValue { }; }; -class TUnboxedValuePod -{ -friend class TUnboxedValue; +class TUnboxedValuePod { + friend class TUnboxedValue; + public: - enum class EMarkers : ui8 { + enum class EMarkers: ui8 { Empty = 0, Embedded, String, @@ -799,13 +818,23 @@ public: void Dump(IOutputStream& out) const; // meta information - inline explicit operator bool() const { return bool(Raw); } + inline explicit operator bool() const { + return bool(Raw); + } - inline bool HasValue() const { return EMarkers::Empty != Raw.GetMarkers(); } + inline bool HasValue() const { + return EMarkers::Empty != Raw.GetMarkers(); + } - inline bool IsString() const { return EMarkers::String == Raw.GetMarkers(); } - inline bool IsBoxed() const { return EMarkers::Boxed == Raw.GetMarkers(); } - inline bool IsEmbedded() const { return EMarkers::Embedded == Raw.GetMarkers(); } + inline bool IsString() const { + return EMarkers::String == Raw.GetMarkers(); + } + inline bool IsBoxed() const { + return EMarkers::Boxed == Raw.GetMarkers(); + } + inline bool IsEmbedded() const { + return EMarkers::Embedded == Raw.GetMarkers(); + } // Data accessors template <typename T, typename = std::enable_if_t<TPrimitiveDataType<T>::Result || std::is_same_v<T, NYql::NDecimal::TInt128>>> @@ -847,8 +876,10 @@ public: inline TUnboxedValuePod MakeOptional() const; inline TUnboxedValuePod GetOptionalValue() const; - template<bool IsOptional> inline TUnboxedValuePod GetOptionalValueIf() const; - template<bool IsOptional> inline TUnboxedValuePod MakeOptionalIf() const; + template <bool IsOptional> + inline TUnboxedValuePod GetOptionalValueIf() const; + template <bool IsOptional> + inline TUnboxedValuePod MakeOptionalIf() const; // List accessors inline bool HasFastListLength() const; @@ -908,7 +939,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 30) - inline EFetchStatus WideFetch(TUnboxedValue *result, ui32 width) const; + inline EFetchStatus WideFetch(TUnboxedValue* result, ui32 width) const; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 36) @@ -932,9 +963,9 @@ protected: struct { union { - #define FIELD(type) type type##_; +#define FIELD(type) type type##_; PRIMITIVE_VALUE_TYPES(FIELD); - #undef FIELD +#undef FIELD const void* Void; ui64 Count; }; @@ -957,7 +988,9 @@ protected: return Simple.Meta >> 2; } - explicit operator bool() const { return Simple.FullMeta | Simple.Count; } + explicit operator bool() const { + return Simple.FullMeta | Simple.Count; + } } Raw; public: @@ -984,8 +1017,7 @@ static_assert(std::is_trivially_move_constructible<TUnboxedValuePod>::value, "In ////////////////////////////////////////////////////////////////////////////// // TUnboxedValue /////////////////////////////////////////////////////////////////////////////// -class TUnboxedValue : public TUnboxedValuePod -{ +class TUnboxedValue: public TUnboxedValuePod { public: inline TUnboxedValue() noexcept = default; inline ~TUnboxedValue() noexcept; @@ -1012,8 +1044,7 @@ UDF_ASSERT_TYPE_SIZE(TUnboxedValue, 16); // TBoxedResource /////////////////////////////////////////////////////////////////////////////// template <typename TResourceData, const char* ResourceTag> -class TBoxedResource: public TBoxedValue -{ +class TBoxedResource: public TBoxedValue { public: template <typename... Args> inline TBoxedResource(Args&&... args) @@ -1048,17 +1079,17 @@ private: } // namespace NUdf } // namespace NYql -template<> -inline void Out<NYql::NUdf::TUnboxedValuePod>(class IOutputStream &o, const NYql::NUdf::TUnboxedValuePod& value); +template <> +inline void Out<NYql::NUdf::TUnboxedValuePod>(class IOutputStream& o, const NYql::NUdf::TUnboxedValuePod& value); -template<> -inline void Out<NYql::NUdf::TUnboxedValue>(class IOutputStream &o, const NYql::NUdf::TUnboxedValue& value); +template <> +inline void Out<NYql::NUdf::TUnboxedValue>(class IOutputStream& o, const NYql::NUdf::TUnboxedValue& value); -template<> -inline void Out<NYql::NUdf::EFetchStatus>(class IOutputStream &o, NYql::NUdf::EFetchStatus value); +template <> +inline void Out<NYql::NUdf::EFetchStatus>(class IOutputStream& o, NYql::NUdf::EFetchStatus value); -template<> -inline void Out<NYql::NUdf::TStringRef>(class IOutputStream &o, const NYql::NUdf::TStringRef& value); +template <> +inline void Out<NYql::NUdf::TStringRef>(class IOutputStream& o, const NYql::NUdf::TStringRef& value); #include "udf_terminator.h" #include <util/stream/output.h> @@ -1070,18 +1101,15 @@ namespace NUdf { ////////////////////////////////////////////////////////////////////////////// // TBoxedValue ////////////////////////////////////////////////////////////////////////////// -inline bool TBoxedValueBase::HasFastListLength() const -{ +inline bool TBoxedValueBase::HasFastListLength() const { Y_ABORT("Not implemented"); } -inline ui64 TBoxedValueBase::GetListLength() const -{ +inline ui64 TBoxedValueBase::GetListLength() const { Y_ABORT("Not implemented"); } -inline ui64 TBoxedValueBase::GetEstimatedListLength() const -{ +inline ui64 TBoxedValueBase::GetEstimatedListLength() const { Y_ABORT("Not implemented"); } @@ -1111,8 +1139,7 @@ inline IBoxedValuePtr TBoxedValueBase::ToIndexDictImpl(const IValueBuilder& buil return nullptr; } -inline ui64 TBoxedValueBase::GetDictLength() const -{ +inline ui64 TBoxedValueBase::GetDictLength() const { Y_ABORT("Not implemented"); } @@ -1137,23 +1164,19 @@ inline void TBoxedValueLink::Unlink() { Left_ = Right_ = nullptr; } -inline TUnboxedValue TBoxedValueBase::GetDictIterator() const -{ +inline TUnboxedValue TBoxedValueBase::GetDictIterator() const { Y_ABORT("Not implemented"); } -inline TUnboxedValue TBoxedValueBase::GetListIterator() const -{ +inline TUnboxedValue TBoxedValueBase::GetListIterator() const { Y_ABORT("Not implemented"); } -inline TUnboxedValue TBoxedValueBase::GetKeysIterator() const -{ +inline TUnboxedValue TBoxedValueBase::GetKeysIterator() const { Y_ABORT("Not implemented"); } -inline TUnboxedValue TBoxedValueBase::GetPayloadsIterator() const -{ +inline TUnboxedValue TBoxedValueBase::GetPayloadsIterator() const { Y_ABORT("Not implemented"); } @@ -1173,19 +1196,16 @@ inline bool TBoxedValueBase::NextPair(TUnboxedValue&, TUnboxedValue&) Y_ABORT("Not implemented"); } -inline TUnboxedValue TBoxedValueBase::GetElement(ui32 index) const -{ +inline TUnboxedValue TBoxedValueBase::GetElement(ui32 index) const { Y_UNUSED(index); Y_ABORT("Not implemented"); } -inline const TUnboxedValue* TBoxedValueBase::GetElements() const -{ +inline const TUnboxedValue* TBoxedValueBase::GetElements() const { return nullptr; } -inline void TBoxedValueBase::Apply(IApplyContext&) const -{ +inline void TBoxedValueBase::Apply(IApplyContext&) const { Y_ABORT("Not implemented"); } @@ -1210,20 +1230,17 @@ inline ui32 TBoxedValueBase::GetVariantIndex() const { Y_ABORT("Not implemented"); } -inline bool TBoxedValueBase::Contains(const TUnboxedValuePod& key) const -{ +inline bool TBoxedValueBase::Contains(const TUnboxedValuePod& key) const { Y_UNUSED(key); Y_ABORT("Not implemented"); } -inline TUnboxedValue TBoxedValueBase::Lookup(const TUnboxedValuePod& key) const -{ +inline TUnboxedValue TBoxedValueBase::Lookup(const TUnboxedValuePod& key) const { Y_UNUSED(key); Y_ABORT("Not implemented"); } -inline TUnboxedValue TBoxedValueBase::Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const -{ +inline TUnboxedValue TBoxedValueBase::Run(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { Y_UNUSED(valueBuilder); Y_UNUSED(args); Y_ABORT("Not implemented"); @@ -1299,7 +1316,7 @@ inline void TBoxedValueBase::Unused6() { #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 30) -inline EFetchStatus TBoxedValueBase::WideFetch(TUnboxedValue *result, ui32 width) { +inline EFetchStatus TBoxedValueBase::WideFetch(TUnboxedValue* result, ui32 width) { Y_UNUSED(result); Y_UNUSED(width); Y_ABORT("Not implemented"); @@ -1315,54 +1332,53 @@ inline bool TBoxedValueBase::Load2(const TUnboxedValue& value) { inline void TUnboxedValuePod::Dump(IOutputStream& out) const { switch (Raw.GetMarkers()) { - case EMarkers::Empty: - out << "Empty, count: " << (i64)Raw.Simple.Count; - break; - case EMarkers::Embedded: - out << "Embedded, size: " << (ui32)Raw.Embedded.Size << ", buffer: " << - TString(Raw.Embedded.Buffer, sizeof(Raw.Embedded.Buffer)).Quote(); - break; - case EMarkers::String: { - out << "String, size: " << Raw.String.Size << ", offset: " << (Raw.String.Offset & 0xffffff) << ", buffer: "; - auto ref = AsStringRef(); - out << TString(ref.Data(), ref.Size()).Quote(); - break; - } - case EMarkers::Boxed: - out << "Boxed, pointer: " << (void*)Raw.Boxed.Value; - break; + case EMarkers::Empty: + out << "Empty, count: " << (i64)Raw.Simple.Count; + break; + case EMarkers::Embedded: + out << "Embedded, size: " << (ui32)Raw.Embedded.Size << ", buffer: " << TString(Raw.Embedded.Buffer, sizeof(Raw.Embedded.Buffer)).Quote(); + break; + case EMarkers::String: { + out << "String, size: " << Raw.String.Size << ", offset: " << (Raw.String.Offset & 0xffffff) << ", buffer: "; + auto ref = AsStringRef(); + out << TString(ref.Data(), ref.Size()).Quote(); + break; + } + case EMarkers::Boxed: + out << "Boxed, pointer: " << (void*)Raw.Boxed.Value; + break; } } } // namespace NUdf } // namespace NYql -template<> -inline void Out<NYql::NUdf::TUnboxedValuePod>(class IOutputStream &o, const NYql::NUdf::TUnboxedValuePod& value) { +template <> +inline void Out<NYql::NUdf::TUnboxedValuePod>(class IOutputStream& o, const NYql::NUdf::TUnboxedValuePod& value) { value.Dump(o); } -template<> -inline void Out<NYql::NUdf::TUnboxedValue>(class IOutputStream &o, const NYql::NUdf::TUnboxedValue& value) { +template <> +inline void Out<NYql::NUdf::TUnboxedValue>(class IOutputStream& o, const NYql::NUdf::TUnboxedValue& value) { value.Dump(o); } -template<> -inline void Out<NYql::NUdf::EFetchStatus>(class IOutputStream &o, NYql::NUdf::EFetchStatus value) { +template <> +inline void Out<NYql::NUdf::EFetchStatus>(class IOutputStream& o, NYql::NUdf::EFetchStatus value) { switch (value) { - case NYql::NUdf::EFetchStatus::Ok: - o << "Ok"; - break; - case NYql::NUdf::EFetchStatus::Yield: - o << "Yield"; - break; - case NYql::NUdf::EFetchStatus::Finish: - o << "Finish"; - break; + case NYql::NUdf::EFetchStatus::Ok: + o << "Ok"; + break; + case NYql::NUdf::EFetchStatus::Yield: + o << "Yield"; + break; + case NYql::NUdf::EFetchStatus::Finish: + o << "Finish"; + break; } } -template<> -inline void Out<NYql::NUdf::TStringRef>(class IOutputStream &o, const NYql::NUdf::TStringRef& value) { +template <> +inline void Out<NYql::NUdf::TStringRef>(class IOutputStream& o, const NYql::NUdf::TStringRef& value) { o << TStringBuf(value.Data(), value.Size()); } diff --git a/yql/essentials/public/udf/udf_value_builder.cpp b/yql/essentials/public/udf/udf_value_builder.cpp index 1d0662f7702..d615e79e32c 100644 --- a/yql/essentials/public/udf/udf_value_builder.cpp +++ b/yql/essentials/public/udf/udf_value_builder.cpp @@ -4,13 +4,16 @@ namespace NYql { namespace NUdf { IDateBuilder::IDateBuilder() -{} +{ +} IPgBuilder::IPgBuilder() -{} +{ +} IValueBuilder::IValueBuilder() -{} +{ +} } // namespace NUdf } // namespace NYql diff --git a/yql/essentials/public/udf/udf_value_builder.h b/yql/essentials/public/udf/udf_value_builder.h index a1c48016447..2f154821cd3 100644 --- a/yql/essentials/public/udf/udf_value_builder.h +++ b/yql/essentials/public/udf/udf_value_builder.h @@ -27,8 +27,7 @@ struct TDictFlags { }; }; -class IDictValueBuilder -{ +class IDictValueBuilder { public: using TPtr = TUniquePtr<IDictValueBuilder>; @@ -63,8 +62,7 @@ UDF_ASSERT_TYPE_SIZE(IListValueBuilder, 8); /////////////////////////////////////////////////////////////////////////////// // IDateBuilder /////////////////////////////////////////////////////////////////////////////// -class IDateBuilder1 -{ +class IDateBuilder1 { public: virtual ~IDateBuilder1() = default; @@ -72,31 +70,30 @@ public: virtual bool SplitDate(ui16 value, ui32& year, ui32& month, ui32& day) const = 0; virtual bool MakeDatetime(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, ui32& value, - ui16 timezoneId = 0) const = 0; + ui16 timezoneId = 0) const = 0; virtual bool SplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, - ui16 timezoneId = 0) const = 0; + ui16 timezoneId = 0) const = 0; // deprecated virtual bool EnrichDate(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek) const = 0; // in minutes virtual bool GetTimezoneShift(ui32 year, ui32 month, ui32 day, ui32 hour, ui32 minute, ui32 second, - ui16 timezoneId, i32& value) const = 0; + ui16 timezoneId, i32& value) const = 0; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT < UDF_ABI_COMPATIBILITY_VERSION(2, 23) virtual void Unused7() const = 0; virtual void Unused8() const = 0; #else virtual bool FullSplitDate(ui16 value, ui32& year, ui32& month, ui32& day, - ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; + ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; virtual bool FullSplitDatetime(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, - ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; + ui32& dayOfYear, ui32& weekOfYear, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; #endif }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 7) -class IDateBuilder2: public IDateBuilder1 -{ +class IDateBuilder2: public IDateBuilder1 { public: virtual bool FindTimezoneName(ui32 id, TStringRef& name) const = 0; virtual bool FindTimezoneId(const TStringRef& name, ui32& id) const = 0; @@ -104,29 +101,27 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 24) -class IDateBuilder3: public IDateBuilder2 -{ +class IDateBuilder3: public IDateBuilder2 { public: virtual bool EnrichDate2(ui16 date, ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek) const = 0; virtual bool FullSplitDate2(ui16 value, ui32& year, ui32& month, ui32& day, - ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; + ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; virtual bool FullSplitDatetime2(ui32 value, ui32& year, ui32& month, ui32& day, ui32& hour, ui32& minute, ui32& second, - ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; + ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; }; #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 40) -class IDateBuilder4: public IDateBuilder3 -{ +class IDateBuilder4: public IDateBuilder3 { public: virtual bool SplitTzDate32(i32 date, i32& year, ui32& month, ui32& day, - ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; + ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; virtual bool SplitTzDatetime64(i64 datetime, i32& year, ui32& month, ui32& day, - ui32& hour, ui32& minute, ui32& second, - ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; + ui32& hour, ui32& minute, ui32& second, + ui32& dayOfYear, ui32& weekOfYear, ui32& weekOfYearIso8601, ui32& dayOfWeek, ui16 timezoneId = 0) const = 0; virtual bool MakeTzDate32(i32 year, ui32 month, ui32 day, i32& date, ui16 timezoneId = 0) const = 0; virtual bool MakeTzDatetime64(i32 year, ui32 month, ui32 day, - ui32 hour, ui32 minute, ui32 second, i64& datetime, ui16 timezoneId = 0) const = 0; + ui32 hour, ui32 minute, ui32 second, i64& datetime, ui16 timezoneId = 0) const = 0; }; #endif @@ -177,8 +172,7 @@ public: }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 31) -class IPgBuilder2: public IPgBuilder1 -{ +class IPgBuilder2: public IPgBuilder1 { public: virtual TStringRef AsCStringBuffer(const TUnboxedValue& value) const = 0; virtual TStringRef AsTextBuffer(const TUnboxedValue& value) const = 0; @@ -186,8 +180,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 33) -class IPgBuilder3: public IPgBuilder2 -{ +class IPgBuilder3: public IPgBuilder2 { public: virtual TUnboxedValue MakeCString(const char* value) const = 0; virtual TUnboxedValue MakeText(const char* value) const = 0; @@ -195,8 +188,7 @@ public: #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 35) -class IPgBuilder4: public IPgBuilder3 -{ +class IPgBuilder4: public IPgBuilder3 { public: virtual TStringRef AsFixedStringBuffer(const TUnboxedValue& value, ui32 length) const = 0; }; @@ -229,8 +221,7 @@ UDF_ASSERT_TYPE_SIZE(IPgBuilder, 8); /////////////////////////////////////////////////////////////////////////////// // IValueBuilder /////////////////////////////////////////////////////////////////////////////// -class IValueBuilder1 -{ +class IValueBuilder1 { public: virtual ~IValueBuilder1() = default; @@ -263,7 +254,9 @@ public: virtual TUnboxedValue NewVariant(ui32 index, TUnboxedValue&& value) const = 0; - inline TUnboxedValue NewEmptyList() const { return NewList(nullptr, 0); } + inline TUnboxedValue NewEmptyList() const { + return NewList(nullptr, 0); + } }; #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 1) @@ -385,8 +378,13 @@ private: std::array<TUnboxedValue, 2U> Cached_; std::array<TUnboxedValue*, 2U> CachedItems_; ui8 CacheIndex_ = 0U; + public: - TPlainArrayCache(ui32 size): Size_(size) { Clear(); } + TPlainArrayCache(ui32 size) + : Size_(size) + { + Clear(); + } TPlainArrayCache(TPlainArrayCache&&) = delete; TPlainArrayCache(const TPlainArrayCache&) = delete; diff --git a/yql/essentials/public/udf/udf_value_builder_ut.cpp b/yql/essentials/public/udf/udf_value_builder_ut.cpp index 64084a37e75..4b9d88e715a 100644 --- a/yql/essentials/public/udf/udf_value_builder_ut.cpp +++ b/yql/essentials/public/udf/udf_value_builder_ut.cpp @@ -7,30 +7,30 @@ using namespace NYql::NUdf; Y_UNIT_TEST_SUITE(TUdfValueBuilder) { - Y_UNIT_TEST(LockMethodsTable) { - using NYql::GetMethodIndex; - UNIT_ASSERT_VALUES_EQUAL(2, GetMethodIndex(&IValueBuilder::NewStringNotFilled)); - UNIT_ASSERT_VALUES_EQUAL(3, GetMethodIndex(&IValueBuilder::NewString)); - UNIT_ASSERT_VALUES_EQUAL(4, GetMethodIndex(&IValueBuilder::ConcatStrings)); - UNIT_ASSERT_VALUES_EQUAL(5, GetMethodIndex(&IValueBuilder::AppendString)); - UNIT_ASSERT_VALUES_EQUAL(6, GetMethodIndex(&IValueBuilder::PrependString)); - UNIT_ASSERT_VALUES_EQUAL(7, GetMethodIndex(&IValueBuilder::SubString)); - UNIT_ASSERT_VALUES_EQUAL(8, GetMethodIndex(&IValueBuilder::NewDict)); - UNIT_ASSERT_VALUES_EQUAL(9, GetMethodIndex(&IValueBuilder::NewList)); - UNIT_ASSERT_VALUES_EQUAL(10, GetMethodIndex(&IValueBuilder::ReverseList)); - UNIT_ASSERT_VALUES_EQUAL(11, GetMethodIndex(&IValueBuilder::SkipList)); - UNIT_ASSERT_VALUES_EQUAL(12, GetMethodIndex(&IValueBuilder::TakeList)); - UNIT_ASSERT_VALUES_EQUAL(13, GetMethodIndex(&IValueBuilder::ToIndexDict)); - UNIT_ASSERT_VALUES_EQUAL(14, GetMethodIndex(&IValueBuilder::NewArray32)); - UNIT_ASSERT_VALUES_EQUAL(15, GetMethodIndex(&IValueBuilder::NewVariant)); - UNIT_ASSERT_VALUES_EQUAL(16, GetMethodIndex(&IValueBuilder::GetDateBuilder)); - UNIT_ASSERT_VALUES_EQUAL(17, GetMethodIndex(&IValueBuilder::GetSecureParam)); - UNIT_ASSERT_VALUES_EQUAL(18, GetMethodIndex(&IValueBuilder::CalleePosition)); - UNIT_ASSERT_VALUES_EQUAL(19, GetMethodIndex(&IValueBuilder::Run)); - UNIT_ASSERT_VALUES_EQUAL(20, GetMethodIndex(&IValueBuilder::ExportArrowBlock)); - UNIT_ASSERT_VALUES_EQUAL(21, GetMethodIndex(&IValueBuilder::ImportArrowBlock)); - UNIT_ASSERT_VALUES_EQUAL(22, GetMethodIndex(&IValueBuilder::GetArrowBlockChunks)); - UNIT_ASSERT_VALUES_EQUAL(23, GetMethodIndex(&IValueBuilder::GetPgBuilder)); - UNIT_ASSERT_VALUES_EQUAL(24, GetMethodIndex(&IValueBuilder::NewArray64)); - } +Y_UNIT_TEST(LockMethodsTable) { + using NYql::GetMethodIndex; + UNIT_ASSERT_VALUES_EQUAL(2, GetMethodIndex(&IValueBuilder::NewStringNotFilled)); + UNIT_ASSERT_VALUES_EQUAL(3, GetMethodIndex(&IValueBuilder::NewString)); + UNIT_ASSERT_VALUES_EQUAL(4, GetMethodIndex(&IValueBuilder::ConcatStrings)); + UNIT_ASSERT_VALUES_EQUAL(5, GetMethodIndex(&IValueBuilder::AppendString)); + UNIT_ASSERT_VALUES_EQUAL(6, GetMethodIndex(&IValueBuilder::PrependString)); + UNIT_ASSERT_VALUES_EQUAL(7, GetMethodIndex(&IValueBuilder::SubString)); + UNIT_ASSERT_VALUES_EQUAL(8, GetMethodIndex(&IValueBuilder::NewDict)); + UNIT_ASSERT_VALUES_EQUAL(9, GetMethodIndex(&IValueBuilder::NewList)); + UNIT_ASSERT_VALUES_EQUAL(10, GetMethodIndex(&IValueBuilder::ReverseList)); + UNIT_ASSERT_VALUES_EQUAL(11, GetMethodIndex(&IValueBuilder::SkipList)); + UNIT_ASSERT_VALUES_EQUAL(12, GetMethodIndex(&IValueBuilder::TakeList)); + UNIT_ASSERT_VALUES_EQUAL(13, GetMethodIndex(&IValueBuilder::ToIndexDict)); + UNIT_ASSERT_VALUES_EQUAL(14, GetMethodIndex(&IValueBuilder::NewArray32)); + UNIT_ASSERT_VALUES_EQUAL(15, GetMethodIndex(&IValueBuilder::NewVariant)); + UNIT_ASSERT_VALUES_EQUAL(16, GetMethodIndex(&IValueBuilder::GetDateBuilder)); + UNIT_ASSERT_VALUES_EQUAL(17, GetMethodIndex(&IValueBuilder::GetSecureParam)); + UNIT_ASSERT_VALUES_EQUAL(18, GetMethodIndex(&IValueBuilder::CalleePosition)); + UNIT_ASSERT_VALUES_EQUAL(19, GetMethodIndex(&IValueBuilder::Run)); + UNIT_ASSERT_VALUES_EQUAL(20, GetMethodIndex(&IValueBuilder::ExportArrowBlock)); + UNIT_ASSERT_VALUES_EQUAL(21, GetMethodIndex(&IValueBuilder::ImportArrowBlock)); + UNIT_ASSERT_VALUES_EQUAL(22, GetMethodIndex(&IValueBuilder::GetArrowBlockChunks)); + UNIT_ASSERT_VALUES_EQUAL(23, GetMethodIndex(&IValueBuilder::GetPgBuilder)); + UNIT_ASSERT_VALUES_EQUAL(24, GetMethodIndex(&IValueBuilder::NewArray64)); } +} // Y_UNIT_TEST_SUITE(TUdfValueBuilder) diff --git a/yql/essentials/public/udf/udf_value_inl.h b/yql/essentials/public/udf/udf_value_inl.h index 43c4c84140f..20445a5d4f7 100644 --- a/yql/essentials/public/udf/udf_value_inl.h +++ b/yql/essentials/public/udf/udf_value_inl.h @@ -1,69 +1,69 @@ #pragma once #if !defined(INCLUDE_UDF_VALUE_INL_H) -#error "you should never include udf_value_inl.h directly" -#endif // INCLUDE_UDF_VALUE_INL_H + #error "you should never include udf_value_inl.h directly" +#endif // INCLUDE_UDF_VALUE_INL_H #ifdef LLVM_BC -#define UDF_VERIFY(expr, ...) \ - do { \ - if (false) { \ - bool __xxx = static_cast<bool>(expr); \ - Y_UNUSED(__xxx); \ - } \ - } while (false) + #define UDF_VERIFY(expr, ...) \ + do { \ + if (false) { \ + bool __xxx = static_cast<bool>(expr); \ + Y_UNUSED(__xxx); \ + } \ + } while (false) -#define UDF_ALWAYS_INLINE __attribute__((always_inline)) + #define UDF_ALWAYS_INLINE __attribute__((always_inline)) #else -#define UDF_VERIFY Y_DEBUG_ABORT_UNLESS -#define UDF_ALWAYS_INLINE Y_FORCE_INLINE + #define UDF_VERIFY Y_DEBUG_ABORT_UNLESS + #define UDF_ALWAYS_INLINE Y_FORCE_INLINE #endif ////////////////////////////////////////////////////////////////////////////// // IBoxedValue ////////////////////////////////////////////////////////////////////////////// -inline void IBoxedValue1::Ref() noexcept -{ +inline void IBoxedValue1::Ref() noexcept { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif ++Refs_; } -inline void IBoxedValue1::UnRef() noexcept -{ +inline void IBoxedValue1::UnRef() noexcept { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif Y_DEBUG_ABORT_UNLESS(Refs_ > 0); - if (!--Refs_) + if (!--Refs_) { delete this; + } } -inline void IBoxedValue1::ReleaseRef() noexcept -{ +inline void IBoxedValue1::ReleaseRef() noexcept { #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) - if (Refs_ < 0) + if (Refs_ < 0) { return; + } #endif Y_DEBUG_ABORT_UNLESS(Refs_ > 0); --Refs_; } -inline void IBoxedValue1::DeleteUnreferenced() noexcept -{ - if (!Refs_) +inline void IBoxedValue1::DeleteUnreferenced() noexcept { + if (!Refs_) { delete this; + } } -inline i32 IBoxedValue1::RefCount() const noexcept -{ +inline i32 IBoxedValue1::RefCount() const noexcept { return Refs_; } @@ -76,16 +76,16 @@ inline ui8 IBoxedValue1::UserMark() const noexcept { } inline i32 IBoxedValue1::LockRef() noexcept { - Y_DEBUG_ABORT_UNLESS(Refs_ != -1); - auto ret = Refs_; - Refs_ = -1; - return ret; + Y_DEBUG_ABORT_UNLESS(Refs_ != -1); + auto ret = Refs_; + Refs_ = -1; + return ret; } inline void IBoxedValue1::UnlockRef(i32 prev) noexcept { - Y_DEBUG_ABORT_UNLESS(Refs_ == -1); - Y_DEBUG_ABORT_UNLESS(prev != -1); - Refs_ = prev; + Y_DEBUG_ABORT_UNLESS(Refs_ == -1); + Y_DEBUG_ABORT_UNLESS(prev != -1); + Refs_ = prev; } ////////////////////////////////////////////////////////////////////////////// @@ -93,113 +93,113 @@ inline void IBoxedValue1::UnlockRef(i32 prev) noexcept { ////////////////////////////////////////////////////////////////////////////// inline bool TBoxedValueAccessor::HasFastListLength(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.HasFastListLength(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.HasFastListLength(); } inline ui64 TBoxedValueAccessor::GetListLength(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetListLength(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetListLength(); } inline ui64 TBoxedValueAccessor::GetEstimatedListLength(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetEstimatedListLength(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetEstimatedListLength(); } inline TUnboxedValue TBoxedValueAccessor::GetListIterator(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetListIterator(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetListIterator(); } inline const TOpaqueListRepresentation* TBoxedValueAccessor::GetListRepresentation(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetListRepresentation(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetListRepresentation(); } inline IBoxedValuePtr TBoxedValueAccessor::ReverseListImpl(const IBoxedValue& value, const IValueBuilder& builder) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.ReverseListImpl(builder); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.ReverseListImpl(builder); } inline IBoxedValuePtr TBoxedValueAccessor::SkipListImpl(const IBoxedValue& value, const IValueBuilder& builder, ui64 count) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.SkipListImpl(builder, count); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.SkipListImpl(builder, count); } inline IBoxedValuePtr TBoxedValueAccessor::TakeListImpl(const IBoxedValue& value, const IValueBuilder& builder, ui64 count) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.TakeListImpl(builder, count); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.TakeListImpl(builder, count); } inline IBoxedValuePtr TBoxedValueAccessor::ToIndexDictImpl(const IBoxedValue& value, const IValueBuilder& builder) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.ToIndexDictImpl(builder); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.ToIndexDictImpl(builder); } inline ui64 TBoxedValueAccessor::GetDictLength(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetDictLength(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetDictLength(); } inline TUnboxedValue TBoxedValueAccessor::GetDictIterator(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetDictIterator(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetDictIterator(); } inline TUnboxedValue TBoxedValueAccessor::GetKeysIterator(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetKeysIterator(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetKeysIterator(); } inline TUnboxedValue TBoxedValueAccessor::GetPayloadsIterator(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetPayloadsIterator(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetPayloadsIterator(); } inline bool TBoxedValueAccessor::Contains(const IBoxedValue& value, const TUnboxedValuePod& key) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.Contains(key); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.Contains(key); } inline TUnboxedValue TBoxedValueAccessor::Lookup(const IBoxedValue& value, const TUnboxedValuePod& key) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.Lookup(key); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.Lookup(key); } inline TUnboxedValue TBoxedValueAccessor::GetElement(const IBoxedValue& value, ui32 index) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetElement(index); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetElement(index); } inline const TUnboxedValue* TBoxedValueAccessor::GetElements(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetElements(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetElements(); } inline TUnboxedValue TBoxedValueAccessor::Run(const IBoxedValue& value, const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.Run(valueBuilder, args); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.Run(valueBuilder, args); } inline TStringRef TBoxedValueAccessor::GetResourceTag(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetResourceTag(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetResourceTag(); } inline void* TBoxedValueAccessor::GetResource(IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.GetResource(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.GetResource(); } inline bool TBoxedValueAccessor::HasListItems(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.HasListItems(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.HasListItems(); } inline bool TBoxedValueAccessor::HasDictItems(const IBoxedValue& value) { - Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); - return value.HasDictItems(); + Y_DEBUG_ABORT_UNLESS(value.IsCompatibleTo(MakeAbiCompatibilityVersion(2, 0))); + return value.HasDictItems(); } inline ui32 TBoxedValueAccessor::GetVariantIndex(const IBoxedValue& value) { @@ -321,8 +321,7 @@ Y_FORCE_INLINE TUnboxedValue::TUnboxedValue(TUnboxedValue&& value) noexcept value.Raw = TRaw(); } -Y_FORCE_INLINE TUnboxedValue& TUnboxedValue::operator=(const TUnboxedValue& value) noexcept -{ +Y_FORCE_INLINE TUnboxedValue& TUnboxedValue::operator=(const TUnboxedValue& value) noexcept { if (this != &value) { value.Ref(); UnRef(); @@ -331,8 +330,7 @@ Y_FORCE_INLINE TUnboxedValue& TUnboxedValue::operator=(const TUnboxedValue& valu return *this; } -Y_FORCE_INLINE TUnboxedValue& TUnboxedValue::operator=(TUnboxedValue&& value) noexcept -{ +Y_FORCE_INLINE TUnboxedValue& TUnboxedValue::operator=(TUnboxedValue&& value) noexcept { if (this != &value) { UnRef(); Raw = value.Raw; @@ -348,14 +346,12 @@ Y_FORCE_INLINE TUnboxedValuePod TUnboxedValue::Release() noexcept { return value; } -Y_FORCE_INLINE void TUnboxedValue::Clear() noexcept -{ +Y_FORCE_INLINE void TUnboxedValue::Clear() noexcept { UnRef(); Raw = TRaw(); } -Y_FORCE_INLINE TUnboxedValue::~TUnboxedValue() noexcept -{ +Y_FORCE_INLINE TUnboxedValue::~TUnboxedValue() noexcept { UnRef(); } ////////////////////////////////////////////////////////////////////////////// @@ -378,32 +374,27 @@ Y_FORCE_INLINE TUnboxedValuePod::TUnboxedValuePod(TStringValue&& value, ui32 siz Raw.String.Meta = static_cast<ui8>(EMarkers::String); } -inline TStringValue TUnboxedValuePod::AsStringValue() const -{ +inline TStringValue TUnboxedValuePod::AsStringValue() const { UDF_VERIFY(IsString(), "Value is not a string"); return TStringValue(Raw.String.Value); } -inline IBoxedValuePtr TUnboxedValuePod::AsBoxed() const -{ +inline IBoxedValuePtr TUnboxedValuePod::AsBoxed() const { UDF_VERIFY(IsBoxed(), "Value is not boxed"); return IBoxedValuePtr(Raw.Boxed.Value); } -inline TStringValue::TData* TUnboxedValuePod::AsRawStringValue() const -{ +inline TStringValue::TData* TUnboxedValuePod::AsRawStringValue() const { UDF_VERIFY(IsString(), "Value is not a string"); return Raw.String.Value; } -inline IBoxedValue* TUnboxedValuePod::AsRawBoxed() const -{ +inline IBoxedValue* TUnboxedValuePod::AsRawBoxed() const { UDF_VERIFY(IsBoxed(), "Value is not boxed"); return Raw.Boxed.Value; } -inline bool TUnboxedValuePod::UniqueBoxed() const -{ +inline bool TUnboxedValuePod::UniqueBoxed() const { UDF_VERIFY(IsBoxed(), "Value is not boxed"); return Raw.Boxed.Value->RefCount() <= 1; } @@ -420,8 +411,7 @@ inline void TUnboxedValuePod::Push(const TUnboxedValuePod& value) const { } #endif -inline ui64 TUnboxedValuePod::GetListLength() const -{ +inline ui64 TUnboxedValuePod::GetListLength() const { UDF_VERIFY(IsBoxed(), "Value is not a list"); return TBoxedValueAccessor::GetListLength(*Raw.Boxed.Value); } @@ -436,26 +426,25 @@ inline bool TUnboxedValuePod::HasListItems() const { return TBoxedValueAccessor::HasListItems(*Raw.Boxed.Value); } -inline TUnboxedValue TUnboxedValuePod::GetListIterator() const -{ +inline TUnboxedValue TUnboxedValuePod::GetListIterator() const { UDF_VERIFY(IsBoxed(), "Value is not a list"); return TBoxedValueAccessor::GetListIterator(*Raw.Boxed.Value); } -inline TUnboxedValuePod TUnboxedValuePod::MakeOptional() const -{ - if (Raw.Simple.Meta) +inline TUnboxedValuePod TUnboxedValuePod::MakeOptional() const { + if (Raw.Simple.Meta) { return *this; + } TUnboxedValuePod result(*this); ++result.Raw.Simple.Count; return result; } -inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValue() const -{ - if (Raw.Simple.Meta) +inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValue() const { + if (Raw.Simple.Meta) { return *this; + } UDF_VERIFY(Raw.Simple.Count > 0U, "Can't get value from empty."); @@ -464,44 +453,50 @@ inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValue() const return result; } -template<> inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValueIf<false>() const { return *this; } -template<> inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValueIf<true>() const { return GetOptionalValue(); } +template <> +inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValueIf<false>() const { + return *this; +} +template <> +inline TUnboxedValuePod TUnboxedValuePod::GetOptionalValueIf<true>() const { + return GetOptionalValue(); +} -template<> inline TUnboxedValuePod TUnboxedValuePod::MakeOptionalIf<false>() const { return *this; } -template<> inline TUnboxedValuePod TUnboxedValuePod::MakeOptionalIf<true>() const { return MakeOptional(); } +template <> +inline TUnboxedValuePod TUnboxedValuePod::MakeOptionalIf<false>() const { + return *this; +} +template <> +inline TUnboxedValuePod TUnboxedValuePod::MakeOptionalIf<true>() const { + return MakeOptional(); +} -inline ui64 TUnboxedValuePod::GetDictLength() const -{ +inline ui64 TUnboxedValuePod::GetDictLength() const { UDF_VERIFY(IsBoxed(), "Value is not a dict"); return TBoxedValueAccessor::GetDictLength(*Raw.Boxed.Value); } -inline TUnboxedValue TUnboxedValuePod::GetDictIterator() const -{ +inline TUnboxedValue TUnboxedValuePod::GetDictIterator() const { UDF_VERIFY(IsBoxed(), "Value is not a dict"); return TBoxedValueAccessor::GetDictIterator(*Raw.Boxed.Value); } -inline TUnboxedValue TUnboxedValuePod::GetKeysIterator() const -{ +inline TUnboxedValue TUnboxedValuePod::GetKeysIterator() const { UDF_VERIFY(IsBoxed(), "Value is not a dict"); return TBoxedValueAccessor::GetKeysIterator(*Raw.Boxed.Value); } -inline TUnboxedValue TUnboxedValuePod::GetPayloadsIterator() const -{ +inline TUnboxedValue TUnboxedValuePod::GetPayloadsIterator() const { UDF_VERIFY(IsBoxed(), "Value is not a dict"); return TBoxedValueAccessor::GetPayloadsIterator(*Raw.Boxed.Value); } -inline bool TUnboxedValuePod::Contains(const TUnboxedValuePod& key) const -{ +inline bool TUnboxedValuePod::Contains(const TUnboxedValuePod& key) const { UDF_VERIFY(IsBoxed(), "Value is not a dict"); return TBoxedValueAccessor::Contains(*Raw.Boxed.Value, key); } -inline TUnboxedValue TUnboxedValuePod::Lookup(const TUnboxedValuePod& key) const -{ +inline TUnboxedValue TUnboxedValuePod::Lookup(const TUnboxedValuePod& key) const { UDF_VERIFY(IsBoxed(), "Value is not a dict"); return TBoxedValueAccessor::Lookup(*Raw.Boxed.Value, key); } @@ -511,21 +506,18 @@ inline bool TUnboxedValuePod::HasDictItems() const { return TBoxedValueAccessor::HasDictItems(*Raw.Boxed.Value); } -inline TUnboxedValue TUnboxedValuePod::GetElement(ui32 index) const -{ +inline TUnboxedValue TUnboxedValuePod::GetElement(ui32 index) const { UDF_VERIFY(IsBoxed(), "Value is not a tuple"); return TBoxedValueAccessor::GetElement(*Raw.Boxed.Value, index); } -inline const TUnboxedValue* TUnboxedValuePod::GetElements() const -{ +inline const TUnboxedValue* TUnboxedValuePod::GetElements() const { UDF_VERIFY(IsBoxed(), "Value is not a tuple"); return TBoxedValueAccessor::GetElements(*Raw.Boxed.Value); } inline TUnboxedValue TUnboxedValuePod::Run( - const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const -{ + const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) const { UDF_VERIFY(IsBoxed(), "Value is not a callable"); return TBoxedValueAccessor::Run(*Raw.Boxed.Value, valueBuilder, args); } @@ -541,8 +533,9 @@ inline void* TUnboxedValuePod::GetResource() const { } inline ui32 TUnboxedValuePod::GetVariantIndex() const { - if (auto index = Raw.GetIndex()) + if (auto index = Raw.GetIndex()) { return --index; + } UDF_VERIFY(IsBoxed(), "Value is not a variant"); return TBoxedValueAccessor::GetVariantIndex(*Raw.Boxed.Value); } @@ -559,8 +552,9 @@ inline TUnboxedValue TUnboxedValuePod::GetVariantItem() const { inline bool TUnboxedValuePod::TryMakeVariant(ui32 index) { static const ui32 limit = (1U << 6U) - 1U; - if (index >= limit || Raw.GetIndex()) + if (index >= limit || Raw.GetIndex()) { return false; + } Raw.Simple.Meta |= ui8(++index << 2); return true; @@ -644,89 +638,100 @@ inline bool TUnboxedValuePod::Load2(const TUnboxedValue& value) { } #endif -Y_FORCE_INLINE void TUnboxedValuePod::Ref() const noexcept -{ +Y_FORCE_INLINE void TUnboxedValuePod::Ref() const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->Ref(); - case EMarkers::Boxed: return Raw.Boxed.Value->Ref(); - default: return; + case EMarkers::String: + return Raw.String.Value->Ref(); + case EMarkers::Boxed: + return Raw.Boxed.Value->Ref(); + default: + return; } } -Y_FORCE_INLINE void TUnboxedValuePod::UnRef() const noexcept -{ +Y_FORCE_INLINE void TUnboxedValuePod::UnRef() const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->UnRef(); - case EMarkers::Boxed: return Raw.Boxed.Value->UnRef(); - default: return; + case EMarkers::String: + return Raw.String.Value->UnRef(); + case EMarkers::Boxed: + return Raw.Boxed.Value->UnRef(); + default: + return; } } -Y_FORCE_INLINE void TUnboxedValuePod::ReleaseRef() const noexcept -{ +Y_FORCE_INLINE void TUnboxedValuePod::ReleaseRef() const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->ReleaseRef(); - case EMarkers::Boxed: return Raw.Boxed.Value->ReleaseRef(); - default: return; + case EMarkers::String: + return Raw.String.Value->ReleaseRef(); + case EMarkers::Boxed: + return Raw.Boxed.Value->ReleaseRef(); + default: + return; } } -Y_FORCE_INLINE void TUnboxedValuePod::DeleteUnreferenced() const noexcept -{ +Y_FORCE_INLINE void TUnboxedValuePod::DeleteUnreferenced() const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->DeleteUnreferenced(); - case EMarkers::Boxed: return Raw.Boxed.Value->DeleteUnreferenced(); - default: return; + case EMarkers::String: + return Raw.String.Value->DeleteUnreferenced(); + case EMarkers::Boxed: + return Raw.Boxed.Value->DeleteUnreferenced(); + default: + return; } } -Y_FORCE_INLINE i32 TUnboxedValuePod::LockRef() const noexcept -{ +Y_FORCE_INLINE i32 TUnboxedValuePod::LockRef() const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->LockRef(); - case EMarkers::Boxed: return Raw.Boxed.Value->LockRef(); - default: return -1; + case EMarkers::String: + return Raw.String.Value->LockRef(); + case EMarkers::Boxed: + return Raw.Boxed.Value->LockRef(); + default: + return -1; } } -Y_FORCE_INLINE void TUnboxedValuePod::UnlockRef(i32 prev) const noexcept -{ +Y_FORCE_INLINE void TUnboxedValuePod::UnlockRef(i32 prev) const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->UnlockRef(prev); - case EMarkers::Boxed: return Raw.Boxed.Value->UnlockRef(prev); - default: return; + case EMarkers::String: + return Raw.String.Value->UnlockRef(prev); + case EMarkers::Boxed: + return Raw.Boxed.Value->UnlockRef(prev); + default: + return; } } -Y_FORCE_INLINE i32 TUnboxedValuePod::RefCount() const noexcept -{ +Y_FORCE_INLINE i32 TUnboxedValuePod::RefCount() const noexcept { switch (Raw.GetMarkers()) { - case EMarkers::String: return Raw.String.Value->RefCount(); - case EMarkers::Boxed: return Raw.Boxed.Value->RefCount(); - default: return -1; + case EMarkers::String: + return Raw.String.Value->RefCount(); + case EMarkers::Boxed: + return Raw.Boxed.Value->RefCount(); + default: + return -1; } } -#define VALUE_GET(xType) \ - template <> \ - inline xType TUnboxedValuePod::Get<xType>() const \ - { \ +#define VALUE_GET(xType) \ + template <> \ + inline xType TUnboxedValuePod::Get<xType>() const { \ UDF_VERIFY(EMarkers::Embedded == Raw.GetMarkers(), "Value is empty."); \ - return Raw.Simple.xType##_; \ + return Raw.Simple.xType##_; \ } -#define VALUE_GET_DEF(xType) \ - template <> \ - inline xType TUnboxedValuePod::GetOrDefault<xType>(xType def) const \ - { \ +#define VALUE_GET_DEF(xType) \ + template <> \ + inline xType TUnboxedValuePod::GetOrDefault<xType>(xType def) const { \ return EMarkers::Empty == Raw.GetMarkers() ? def : Raw.Simple.xType##_; \ } -#define VALUE_CONSTR(xType) \ - template <> \ - inline TUnboxedValuePod::TUnboxedValuePod(xType value) \ - { \ - Raw.Simple.xType##_ = value; \ +#define VALUE_CONSTR(xType) \ + template <> \ + inline TUnboxedValuePod::TUnboxedValuePod(xType value) { \ + Raw.Simple.xType##_ = value; \ Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); \ } @@ -739,21 +744,18 @@ PRIMITIVE_VALUE_TYPES(VALUE_CONSTR) #undef VALUE_CONSTR template <> -inline bool TUnboxedValuePod::Get<bool>() const -{ +inline bool TUnboxedValuePod::Get<bool>() const { UDF_VERIFY(EMarkers::Empty != Raw.GetMarkers(), "Value is empty."); return bool(Raw.Simple.ui8_); } template <> -inline bool TUnboxedValuePod::GetOrDefault<bool>(bool def) const -{ +inline bool TUnboxedValuePod::GetOrDefault<bool>(bool def) const { return EMarkers::Empty == Raw.GetMarkers() ? def : bool(Raw.Simple.ui8_); } template <> -inline NYql::NDecimal::TInt128 TUnboxedValuePod::Get<NYql::NDecimal::TInt128>() const -{ +inline NYql::NDecimal::TInt128 TUnboxedValuePod::Get<NYql::NDecimal::TInt128>() const { return GetInt128(); } @@ -764,8 +766,7 @@ inline TUnboxedValuePod::TUnboxedValuePod(bool value) Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); } -inline NYql::NDecimal::TInt128 TUnboxedValuePod::GetInt128() const -{ +inline NYql::NDecimal::TInt128 TUnboxedValuePod::GetInt128() const { UDF_VERIFY(EMarkers::Empty != Raw.GetMarkers(), "Value is empty."); auto v = *reinterpret_cast<const NYql::NDecimal::TInt128*>(&Raw); const auto p = reinterpret_cast<ui8*>(&v); @@ -773,8 +774,7 @@ inline NYql::NDecimal::TInt128 TUnboxedValuePod::GetInt128() const return v; } -inline NYql::NDecimal::TUint128 TUnboxedValuePod::GetUint128() const -{ +inline NYql::NDecimal::TUint128 TUnboxedValuePod::GetUint128() const { UDF_VERIFY(EMarkers::Empty != Raw.GetMarkers(), "Value is empty."); auto v = *reinterpret_cast<const NYql::NDecimal::TUint128*>(&Raw); const auto p = reinterpret_cast<ui8*>(&v); @@ -794,8 +794,7 @@ inline TUnboxedValuePod::TUnboxedValuePod(NYql::NDecimal::TUint128 value) Raw.Simple.Meta = static_cast<ui8>(EMarkers::Embedded); } -inline const void* TUnboxedValuePod::GetRawPtr() const -{ +inline const void* TUnboxedValuePod::GetRawPtr() const { return &Raw; } @@ -859,40 +858,40 @@ inline TUnboxedValuePod TUnboxedValuePod::MakeYield() return Invalid(); } -inline bool TUnboxedValuePod::IsInvalid() const -{ +inline bool TUnboxedValuePod::IsInvalid() const { return Raw.Simple.Count == std::numeric_limits<ui64>::max() && Raw.Simple.FullMeta == 0; } -inline bool TUnboxedValuePod::IsFinish() const -{ +inline bool TUnboxedValuePod::IsFinish() const { return Raw.Simple.Count == std::numeric_limits<ui64>::max() - 1U && Raw.Simple.FullMeta == 0; } -inline bool TUnboxedValuePod::IsYield() const -{ +inline bool TUnboxedValuePod::IsYield() const { return IsInvalid(); } -inline bool TUnboxedValuePod::IsSpecial() const -{ +inline bool TUnboxedValuePod::IsSpecial() const { return Raw.Simple.FullMeta == 0 && Raw.Simple.Count >= std::numeric_limits<ui64>::max() - 1U; } -inline TStringRef TUnboxedValuePod::AsStringRef() const& -{ +inline TStringRef TUnboxedValuePod::AsStringRef() const& { switch (Raw.GetMarkers()) { - case EMarkers::Embedded: return { Raw.Embedded.Buffer, Raw.Embedded.Size }; - case EMarkers::String: return { Raw.String.Value->Data() + (Raw.String.Offset & 0xFFFFFF), Raw.String.Size }; - default: Y_ABORT("Value is not a string."); + case EMarkers::Embedded: + return {Raw.Embedded.Buffer, Raw.Embedded.Size}; + case EMarkers::String: + return {Raw.String.Value->Data() + (Raw.String.Offset & 0xFFFFFF), Raw.String.Size}; + default: + Y_ABORT("Value is not a string."); } } -inline TMutableStringRef TUnboxedValuePod::AsStringRef() & -{ +inline TMutableStringRef TUnboxedValuePod::AsStringRef() & { switch (Raw.GetMarkers()) { - case EMarkers::Embedded: return { Raw.Embedded.Buffer, Raw.Embedded.Size }; - case EMarkers::String: return { Raw.String.Value->Data() + (Raw.String.Offset & 0xFFFFFF), Raw.String.Size }; - default: Y_ABORT("Value is not a string."); + case EMarkers::Embedded: + return {Raw.Embedded.Buffer, Raw.Embedded.Size}; + case EMarkers::String: + return {Raw.String.Value->Data() + (Raw.String.Offset & 0xFFFFFF), Raw.String.Size}; + default: + Y_ABORT("Value is not a string."); } } diff --git a/yql/essentials/public/udf/udf_value_ut.cpp b/yql/essentials/public/udf/udf_value_ut.cpp index 7f5489c7331..e296c6252c6 100644 --- a/yql/essentials/public/udf/udf_value_ut.cpp +++ b/yql/essentials/public/udf/udf_value_ut.cpp @@ -8,163 +8,163 @@ using namespace NYql::NUdf; Y_UNIT_TEST_SUITE(TUdfValue) { - Y_UNIT_TEST(TestOptional) { - TUnboxedValuePod foo((ui32) 42); - UNIT_ASSERT(foo); - UNIT_ASSERT(42 == foo.Get<ui32>()); - - auto optFoo = foo.MakeOptional(); - UNIT_ASSERT(optFoo); - UNIT_ASSERT(optFoo.HasValue()); - - auto bar = optFoo.GetOptionalValue(); - UNIT_ASSERT(42 == bar.Get<ui32>()); - } - - Y_UNIT_TEST(TestOptional2) { - auto valueOpt = TUnboxedValuePod((ui32) 42); - UNIT_ASSERT(valueOpt); - UNIT_ASSERT(valueOpt.HasValue()); - - auto value = valueOpt.GetOptionalValue(); - UNIT_ASSERT(42 == value.Get<ui32>()); - } - - Y_UNIT_TEST(TestEmptyOptional) { - auto optEmpty = TUnboxedValuePod(); - UNIT_ASSERT(!optEmpty); - UNIT_ASSERT(!optEmpty.HasValue()); - - auto optOptEmpty = optEmpty.MakeOptional(); - UNIT_ASSERT(optOptEmpty); - UNIT_ASSERT(!optOptEmpty.HasValue()); - - auto optOptOptEmpty = optOptEmpty.MakeOptional(); - UNIT_ASSERT(optOptOptEmpty); - UNIT_ASSERT(!optOptOptEmpty.HasValue()); - - auto v = optOptEmpty.GetOptionalValue(); - UNIT_ASSERT(!v); - } - - Y_UNIT_TEST(TestVariant) { - TUnboxedValuePod foo((ui64) 42); - UNIT_ASSERT(foo); - - UNIT_ASSERT(!foo.TryMakeVariant(63)); - - UNIT_ASSERT(foo.TryMakeVariant(62)); - - UNIT_ASSERT(!foo.TryMakeVariant(0)); - - UNIT_ASSERT(62 == foo.GetVariantIndex()); - UNIT_ASSERT(42 == foo.Get<ui64>()); - } - - Y_UNIT_TEST(TestEmptyInVariant) { - TUnboxedValuePod foo; - UNIT_ASSERT(!foo); - UNIT_ASSERT(!foo.HasValue()); - - UNIT_ASSERT(foo.TryMakeVariant(0)); - UNIT_ASSERT(foo); - UNIT_ASSERT(!foo.HasValue()); - - UNIT_ASSERT(0 == foo.GetVariantIndex()); - - const auto opt = foo.MakeOptional(); - UNIT_ASSERT(!std::memcmp(&opt, &foo, sizeof(opt))); - - const auto bar = opt.GetOptionalValue(); - UNIT_ASSERT(!std::memcmp(&opt, &bar, sizeof(bar))); - } - - Y_UNIT_TEST(TestInvalid) { - TUnboxedValuePod foo; - UNIT_ASSERT(!foo.IsInvalid()); - - UNIT_ASSERT(!TUnboxedValuePod::Void().IsInvalid()); - UNIT_ASSERT(!TUnboxedValuePod::Zero().IsInvalid()); - UNIT_ASSERT(!TUnboxedValuePod::Embedded(TStringRef("abc")).IsInvalid()); - - auto bad = TUnboxedValuePod::Invalid(); - UNIT_ASSERT(bad.IsInvalid()); - } - - Y_UNIT_TEST(TestDump) { - NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__); - UNIT_ASSERT_STRINGS_EQUAL(TStringBuilder() << TUnboxedValuePod(), "Empty, count: 0"); - UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod().MakeOptional()), "Empty, count: 1"); - UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod::Invalid()), "Empty, count: -1"); - UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod::Void()), - "Embedded, size: 0, buffer: \"\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\""); - UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod::Embedded("foo")), - "Embedded, size: 3, buffer: \"foo\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\""); - UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod(ui32(258))), - "Embedded, size: 0, buffer: \"\\2\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\""); - TString pattern = "VERY VERY LONG STRING"; - TStringValue str(pattern.size()); - memcpy(str.Data(), pattern.data(), pattern.size()); - TUnboxedValue strVal(TUnboxedValuePod(std::move(str))); - UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << strVal), - "String, size: 21, offset: 0, buffer: \"VERY VERY LONG STRING\""); - - TString objStr = (TStringBuilder() << TUnboxedValue(TUnboxedValuePod(IBoxedValuePtr(new TBoxedValue())))); - UNIT_ASSERT(objStr.StartsWith("Boxed, pointer:")); - } - - Y_UNIT_TEST(LockMethodsTable) { +Y_UNIT_TEST(TestOptional) { + TUnboxedValuePod foo((ui32)42); + UNIT_ASSERT(foo); + UNIT_ASSERT(42 == foo.Get<ui32>()); + + auto optFoo = foo.MakeOptional(); + UNIT_ASSERT(optFoo); + UNIT_ASSERT(optFoo.HasValue()); + + auto bar = optFoo.GetOptionalValue(); + UNIT_ASSERT(42 == bar.Get<ui32>()); +} + +Y_UNIT_TEST(TestOptional2) { + auto valueOpt = TUnboxedValuePod((ui32)42); + UNIT_ASSERT(valueOpt); + UNIT_ASSERT(valueOpt.HasValue()); + + auto value = valueOpt.GetOptionalValue(); + UNIT_ASSERT(42 == value.Get<ui32>()); +} + +Y_UNIT_TEST(TestEmptyOptional) { + auto optEmpty = TUnboxedValuePod(); + UNIT_ASSERT(!optEmpty); + UNIT_ASSERT(!optEmpty.HasValue()); + + auto optOptEmpty = optEmpty.MakeOptional(); + UNIT_ASSERT(optOptEmpty); + UNIT_ASSERT(!optOptEmpty.HasValue()); + + auto optOptOptEmpty = optOptEmpty.MakeOptional(); + UNIT_ASSERT(optOptOptEmpty); + UNIT_ASSERT(!optOptOptEmpty.HasValue()); + + auto v = optOptEmpty.GetOptionalValue(); + UNIT_ASSERT(!v); +} + +Y_UNIT_TEST(TestVariant) { + TUnboxedValuePod foo((ui64)42); + UNIT_ASSERT(foo); + + UNIT_ASSERT(!foo.TryMakeVariant(63)); + + UNIT_ASSERT(foo.TryMakeVariant(62)); + + UNIT_ASSERT(!foo.TryMakeVariant(0)); + + UNIT_ASSERT(62 == foo.GetVariantIndex()); + UNIT_ASSERT(42 == foo.Get<ui64>()); +} + +Y_UNIT_TEST(TestEmptyInVariant) { + TUnboxedValuePod foo; + UNIT_ASSERT(!foo); + UNIT_ASSERT(!foo.HasValue()); + + UNIT_ASSERT(foo.TryMakeVariant(0)); + UNIT_ASSERT(foo); + UNIT_ASSERT(!foo.HasValue()); + + UNIT_ASSERT(0 == foo.GetVariantIndex()); + + const auto opt = foo.MakeOptional(); + UNIT_ASSERT(!std::memcmp(&opt, &foo, sizeof(opt))); + + const auto bar = opt.GetOptionalValue(); + UNIT_ASSERT(!std::memcmp(&opt, &bar, sizeof(bar))); +} + +Y_UNIT_TEST(TestInvalid) { + TUnboxedValuePod foo; + UNIT_ASSERT(!foo.IsInvalid()); + + UNIT_ASSERT(!TUnboxedValuePod::Void().IsInvalid()); + UNIT_ASSERT(!TUnboxedValuePod::Zero().IsInvalid()); + UNIT_ASSERT(!TUnboxedValuePod::Embedded(TStringRef("abc")).IsInvalid()); + + auto bad = TUnboxedValuePod::Invalid(); + UNIT_ASSERT(bad.IsInvalid()); +} + +Y_UNIT_TEST(TestDump) { + NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__); + UNIT_ASSERT_STRINGS_EQUAL(TStringBuilder() << TUnboxedValuePod(), "Empty, count: 0"); + UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod().MakeOptional()), "Empty, count: 1"); + UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod::Invalid()), "Empty, count: -1"); + UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod::Void()), + "Embedded, size: 0, buffer: \"\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\""); + UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod::Embedded("foo")), + "Embedded, size: 3, buffer: \"foo\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\""); + UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << TUnboxedValuePod(ui32(258))), + "Embedded, size: 0, buffer: \"\\2\\1\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\""); + TString pattern = "VERY VERY LONG STRING"; + TStringValue str(pattern.size()); + memcpy(str.Data(), pattern.data(), pattern.size()); + TUnboxedValue strVal(TUnboxedValuePod(std::move(str))); + UNIT_ASSERT_STRINGS_EQUAL((TStringBuilder() << strVal), + "String, size: 21, offset: 0, buffer: \"VERY VERY LONG STRING\""); + + TString objStr = (TStringBuilder() << TUnboxedValue(TUnboxedValuePod(IBoxedValuePtr(new TBoxedValue())))); + UNIT_ASSERT(objStr.StartsWith("Boxed, pointer:")); +} + +Y_UNIT_TEST(LockMethodsTable) { #define METHOD_INDEX(name) NYql::GetMethodPtrIndex(TBoxedValueAccessor::GetMethodPtr(TBoxedValueAccessor::EMethod::name)) - UNIT_ASSERT_VALUES_EQUAL(2, METHOD_INDEX(HasFastListLength)); - UNIT_ASSERT_VALUES_EQUAL(3, METHOD_INDEX(GetListLength)); - UNIT_ASSERT_VALUES_EQUAL(4, METHOD_INDEX(GetEstimatedListLength)); - UNIT_ASSERT_VALUES_EQUAL(5, METHOD_INDEX(GetListIterator)); - UNIT_ASSERT_VALUES_EQUAL(6, METHOD_INDEX(GetListRepresentation)); - UNIT_ASSERT_VALUES_EQUAL(7, METHOD_INDEX(ReverseListImpl)); - UNIT_ASSERT_VALUES_EQUAL(8, METHOD_INDEX(SkipListImpl)); - UNIT_ASSERT_VALUES_EQUAL(9, METHOD_INDEX(TakeListImpl)); - UNIT_ASSERT_VALUES_EQUAL(10, METHOD_INDEX(ToIndexDictImpl)); - UNIT_ASSERT_VALUES_EQUAL(11, METHOD_INDEX(GetDictLength)); - UNIT_ASSERT_VALUES_EQUAL(12, METHOD_INDEX(GetDictIterator)); - UNIT_ASSERT_VALUES_EQUAL(13, METHOD_INDEX(GetKeysIterator)); - UNIT_ASSERT_VALUES_EQUAL(14, METHOD_INDEX(GetPayloadsIterator)); - UNIT_ASSERT_VALUES_EQUAL(15, METHOD_INDEX(Contains)); - UNIT_ASSERT_VALUES_EQUAL(16, METHOD_INDEX(Lookup)); - UNIT_ASSERT_VALUES_EQUAL(17, METHOD_INDEX(GetElement)); - UNIT_ASSERT_VALUES_EQUAL(18, METHOD_INDEX(GetElements)); - UNIT_ASSERT_VALUES_EQUAL(19, METHOD_INDEX(Run)); - UNIT_ASSERT_VALUES_EQUAL(20, METHOD_INDEX(GetResourceTag)); - UNIT_ASSERT_VALUES_EQUAL(21, METHOD_INDEX(GetResource)); - UNIT_ASSERT_VALUES_EQUAL(22, METHOD_INDEX(HasListItems)); - UNIT_ASSERT_VALUES_EQUAL(23, METHOD_INDEX(HasDictItems)); - UNIT_ASSERT_VALUES_EQUAL(24, METHOD_INDEX(GetVariantIndex)); - UNIT_ASSERT_VALUES_EQUAL(25, METHOD_INDEX(GetVariantItem)); - UNIT_ASSERT_VALUES_EQUAL(26, METHOD_INDEX(Fetch)); - UNIT_ASSERT_VALUES_EQUAL(27, METHOD_INDEX(Skip)); - UNIT_ASSERT_VALUES_EQUAL(28, METHOD_INDEX(Next)); - UNIT_ASSERT_VALUES_EQUAL(29, METHOD_INDEX(NextPair)); - UNIT_ASSERT_VALUES_EQUAL(30, METHOD_INDEX(Apply)); + UNIT_ASSERT_VALUES_EQUAL(2, METHOD_INDEX(HasFastListLength)); + UNIT_ASSERT_VALUES_EQUAL(3, METHOD_INDEX(GetListLength)); + UNIT_ASSERT_VALUES_EQUAL(4, METHOD_INDEX(GetEstimatedListLength)); + UNIT_ASSERT_VALUES_EQUAL(5, METHOD_INDEX(GetListIterator)); + UNIT_ASSERT_VALUES_EQUAL(6, METHOD_INDEX(GetListRepresentation)); + UNIT_ASSERT_VALUES_EQUAL(7, METHOD_INDEX(ReverseListImpl)); + UNIT_ASSERT_VALUES_EQUAL(8, METHOD_INDEX(SkipListImpl)); + UNIT_ASSERT_VALUES_EQUAL(9, METHOD_INDEX(TakeListImpl)); + UNIT_ASSERT_VALUES_EQUAL(10, METHOD_INDEX(ToIndexDictImpl)); + UNIT_ASSERT_VALUES_EQUAL(11, METHOD_INDEX(GetDictLength)); + UNIT_ASSERT_VALUES_EQUAL(12, METHOD_INDEX(GetDictIterator)); + UNIT_ASSERT_VALUES_EQUAL(13, METHOD_INDEX(GetKeysIterator)); + UNIT_ASSERT_VALUES_EQUAL(14, METHOD_INDEX(GetPayloadsIterator)); + UNIT_ASSERT_VALUES_EQUAL(15, METHOD_INDEX(Contains)); + UNIT_ASSERT_VALUES_EQUAL(16, METHOD_INDEX(Lookup)); + UNIT_ASSERT_VALUES_EQUAL(17, METHOD_INDEX(GetElement)); + UNIT_ASSERT_VALUES_EQUAL(18, METHOD_INDEX(GetElements)); + UNIT_ASSERT_VALUES_EQUAL(19, METHOD_INDEX(Run)); + UNIT_ASSERT_VALUES_EQUAL(20, METHOD_INDEX(GetResourceTag)); + UNIT_ASSERT_VALUES_EQUAL(21, METHOD_INDEX(GetResource)); + UNIT_ASSERT_VALUES_EQUAL(22, METHOD_INDEX(HasListItems)); + UNIT_ASSERT_VALUES_EQUAL(23, METHOD_INDEX(HasDictItems)); + UNIT_ASSERT_VALUES_EQUAL(24, METHOD_INDEX(GetVariantIndex)); + UNIT_ASSERT_VALUES_EQUAL(25, METHOD_INDEX(GetVariantItem)); + UNIT_ASSERT_VALUES_EQUAL(26, METHOD_INDEX(Fetch)); + UNIT_ASSERT_VALUES_EQUAL(27, METHOD_INDEX(Skip)); + UNIT_ASSERT_VALUES_EQUAL(28, METHOD_INDEX(Next)); + UNIT_ASSERT_VALUES_EQUAL(29, METHOD_INDEX(NextPair)); + UNIT_ASSERT_VALUES_EQUAL(30, METHOD_INDEX(Apply)); #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 3) - UNIT_ASSERT_VALUES_EQUAL(31, METHOD_INDEX(GetTraverseCount)); - UNIT_ASSERT_VALUES_EQUAL(32, METHOD_INDEX(GetTraverseItem)); - UNIT_ASSERT_VALUES_EQUAL(33, METHOD_INDEX(Save)); - UNIT_ASSERT_VALUES_EQUAL(34, METHOD_INDEX(Load)); + UNIT_ASSERT_VALUES_EQUAL(31, METHOD_INDEX(GetTraverseCount)); + UNIT_ASSERT_VALUES_EQUAL(32, METHOD_INDEX(GetTraverseItem)); + UNIT_ASSERT_VALUES_EQUAL(33, METHOD_INDEX(Save)); + UNIT_ASSERT_VALUES_EQUAL(34, METHOD_INDEX(Load)); #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 11) - UNIT_ASSERT_VALUES_EQUAL(35, METHOD_INDEX(Push)); + UNIT_ASSERT_VALUES_EQUAL(35, METHOD_INDEX(Push)); #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 12) - UNIT_ASSERT_VALUES_EQUAL(36, METHOD_INDEX(IsSortedDict)); + UNIT_ASSERT_VALUES_EQUAL(36, METHOD_INDEX(IsSortedDict)); #endif #if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 19) - UNIT_ASSERT_VALUES_EQUAL(37, METHOD_INDEX(Unused1)); - UNIT_ASSERT_VALUES_EQUAL(38, METHOD_INDEX(Unused2)); - UNIT_ASSERT_VALUES_EQUAL(39, METHOD_INDEX(Unused3)); - UNIT_ASSERT_VALUES_EQUAL(40, METHOD_INDEX(Unused4)); - UNIT_ASSERT_VALUES_EQUAL(41, METHOD_INDEX(Unused5)); - UNIT_ASSERT_VALUES_EQUAL(42, METHOD_INDEX(Unused6)); + UNIT_ASSERT_VALUES_EQUAL(37, METHOD_INDEX(Unused1)); + UNIT_ASSERT_VALUES_EQUAL(38, METHOD_INDEX(Unused2)); + UNIT_ASSERT_VALUES_EQUAL(39, METHOD_INDEX(Unused3)); + UNIT_ASSERT_VALUES_EQUAL(40, METHOD_INDEX(Unused4)); + UNIT_ASSERT_VALUES_EQUAL(41, METHOD_INDEX(Unused5)); + UNIT_ASSERT_VALUES_EQUAL(42, METHOD_INDEX(Unused6)); #endif #undef METHOD_INDEX - } } +} // Y_UNIT_TEST_SUITE(TUdfValue) diff --git a/yql/essentials/public/udf/udf_version.cpp b/yql/essentials/public/udf/udf_version.cpp index f900f99b4b7..28a7a679824 100644 --- a/yql/essentials/public/udf/udf_version.cpp +++ b/yql/essentials/public/udf/udf_version.cpp @@ -2,7 +2,6 @@ #include <util/string/builder.h> - namespace NYql { namespace NUdf { diff --git a/yql/essentials/public/udf/udf_version.h b/yql/essentials/public/udf/udf_version.h index 5853d62a0ce..5f6e6ffd0bb 100644 --- a/yql/essentials/public/udf/udf_version.h +++ b/yql/essentials/public/udf/udf_version.h @@ -11,13 +11,13 @@ namespace NUdf { #define CURRENT_UDF_ABI_VERSION_PATCH 0 #ifdef USE_CURRENT_UDF_ABI_VERSION -#define UDF_ABI_VERSION_MAJOR CURRENT_UDF_ABI_VERSION_MAJOR -#define UDF_ABI_VERSION_MINOR CURRENT_UDF_ABI_VERSION_MINOR -#define UDF_ABI_VERSION_PATCH CURRENT_UDF_ABI_VERSION_PATCH + #define UDF_ABI_VERSION_MAJOR CURRENT_UDF_ABI_VERSION_MAJOR + #define UDF_ABI_VERSION_MINOR CURRENT_UDF_ABI_VERSION_MINOR + #define UDF_ABI_VERSION_PATCH CURRENT_UDF_ABI_VERSION_PATCH #else -#if !defined(UDF_ABI_VERSION_MAJOR) || !defined(UDF_ABI_VERSION_MINOR) || !defined(UDF_ABI_VERSION_PATCH) -#error Please use UDF_ABI_VERSION macro to define ABI version -#endif + #if !defined(UDF_ABI_VERSION_MAJOR) || !defined(UDF_ABI_VERSION_MINOR) || !defined(UDF_ABI_VERSION_PATCH) + #error Please use UDF_ABI_VERSION macro to define ABI version + #endif #endif inline const char* CurrentAbiVersionStr() @@ -25,9 +25,7 @@ inline const char* CurrentAbiVersionStr() #define STR(s) #s #define XSTR(s) STR(s) - return XSTR(UDF_ABI_VERSION_MAJOR) "." - XSTR(UDF_ABI_VERSION_MINOR) "." - XSTR(UDF_ABI_VERSION_PATCH); + return XSTR(UDF_ABI_VERSION_MAJOR) "." XSTR(UDF_ABI_VERSION_MINOR) "." XSTR(UDF_ABI_VERSION_PATCH); #undef STR #undef XSTR @@ -37,9 +35,8 @@ inline const char* CurrentAbiVersionStr() #define UDF_ABI_COMPATIBILITY_VERSION_CURRENT UDF_ABI_COMPATIBILITY_VERSION(UDF_ABI_VERSION_MAJOR, UDF_ABI_VERSION_MINOR) static_assert(UDF_ABI_COMPATIBILITY_VERSION_CURRENT <= - UDF_ABI_COMPATIBILITY_VERSION(CURRENT_UDF_ABI_VERSION_MAJOR, CURRENT_UDF_ABI_VERSION_MINOR), - "UDF ABI version " Y_STRINGIZE(UDF_ABI_VERSION_MAJOR) "." Y_STRINGIZE(UDF_ABI_VERSION_MINOR) - " is above " Y_STRINGIZE(CURRENT_UDF_ABI_VERSION_MAJOR) "." Y_STRINGIZE(CURRENT_UDF_ABI_VERSION_MINOR)); + UDF_ABI_COMPATIBILITY_VERSION(CURRENT_UDF_ABI_VERSION_MAJOR, CURRENT_UDF_ABI_VERSION_MINOR), + "UDF ABI version " Y_STRINGIZE(UDF_ABI_VERSION_MAJOR) "." Y_STRINGIZE(UDF_ABI_VERSION_MINOR) " is above " Y_STRINGIZE(CURRENT_UDF_ABI_VERSION_MAJOR) "." Y_STRINGIZE(CURRENT_UDF_ABI_VERSION_MINOR)); constexpr ui32 MakeAbiVersion(ui8 major, ui8 minor, ui8 patch) { @@ -65,12 +62,14 @@ constexpr bool IsAbiCompatible(ui32 version) { // backward compatibility in greater minor versions of host return version / 10000 == UDF_ABI_VERSION_MAJOR && - (version / 100) % 100 <= UDF_ABI_VERSION_MINOR; + (version / 100) % 100 <= UDF_ABI_VERSION_MINOR; } TString AbiVersionToStr(ui32 version); -} // namspace NUdf -} // namspace NYql +} // namespace NUdf +} // namespace NYql -namespace NKikimr { namespace NUdf = ::NYql::NUdf; } +namespace NKikimr { +namespace NUdf = ::NYql::NUdf; +} // namespace NKikimr diff --git a/yql/essentials/public/udf/ut/ya.make b/yql/essentials/public/udf/ut/ya.make index f4aa2a5a4e4..e7fd58890c7 100644 --- a/yql/essentials/public/udf/ut/ya.make +++ b/yql/essentials/public/udf/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(yql/essentials/public/udf) +ENABLE(YQL_STYLE_CPP) + SRCS( udf_counter_ut.cpp udf_value_ut.cpp diff --git a/yql/essentials/public/udf/ya.make b/yql/essentials/public/udf/ya.make index b6ae76163d2..439e5e7eb18 100644 --- a/yql/essentials/public/udf/ya.make +++ b/yql/essentials/public/udf/ya.make @@ -1,5 +1,7 @@ LIBRARY() +ENABLE(YQL_STYLE_CPP) + SRCS( udf_allocator.cpp udf_allocator.h |