diff options
author | YDBot <ydbot@ydb.tech> | 2025-07-30 00:52:30 +0000 |
---|---|---|
committer | YDBot <ydbot@ydb.tech> | 2025-07-30 00:52:30 +0000 |
commit | aec6b4d77493bc833e78f820c3cb5632a20db790 (patch) | |
tree | 3fb1829cfae953b525c3488e0e6d6e7f207d4773 | |
parent | 80bf0b792ecdd91b5386bcb559bc40d3abf7a225 (diff) | |
parent | 94ab428afaf835a102a527535e6428f55180e070 (diff) | |
download | ydb-aec6b4d77493bc833e78f820c3cb5632a20db790.tar.gz |
Sync branches 250730-0051
23 files changed, 191 insertions, 106 deletions
diff --git a/build/conf/compilers/gnu_compiler.conf b/build/conf/compilers/gnu_compiler.conf index dd6c2351b6a..dc2ba68a520 100644 --- a/build/conf/compilers/gnu_compiler.conf +++ b/build/conf/compilers/gnu_compiler.conf @@ -80,6 +80,48 @@ when ($CLANG18 == "yes") { } } +when ($CLANG20 == "yes") { + CFLAGS+=-Wno-array-parameter -Wno-deprecate-lax-vec-conv-all -Wno-unqualified-std-cast-call -Wno-unused-but-set-parameter -Wno-implicit-function-declaration -Wno-int-conversion -Wno-incompatible-function-pointer-types -Wno-address-of-packed-member + CFLAGS+=-Wno-deprecated-this-capture -Wno-missing-designated-field-initializers \ + -Wno-format -Wno-vla-cxx-extension -Wno-invalid-offsetof \ + # warnings improved in clang20 + CFLAGS+=-Wno-ignored-qualifiers \ + -Wno-implicit-int \ + -Wno-array-bounds \ + -Wno-deprecated-literal-operator \ + -Wno-unused-parameter \ + -Wno-dangling-gsl \ + -Wno-macro-redefined \ + -Wno-extra-qualification \ + -Wno-return-stack-address \ + -Wno-shorten-64-to-32 \ + -Wno-nontrivial-memcall \ + -Wno-tautological-constant-out-of-range-compare \ + -Wno-unused-but-set-variable \ + -Wno-explicit-specialization-storage-class \ + -Wno-deprecated-declarations \ + -Wno-dangling-assignment \ + -Wno-cast-function-type-mismatch \ + -Wno-delete-incomplete \ + -Wno-unused-private-field \ + -Wno-unused-value \ + -Wno-sign-compare \ + -Wno-braced-scalar-init \ + -Wno-main \ + -Wno-pointer-bool-conversion + + # new warnings for clang20 + CFLAGS+=-Wno-missing-template-arg-list-after-template-kw \ + -Wno-strict-primary-template-shadow \ + -Wno-extraneous-template-head \ + -Wno-dangling-assignment-gsl \ + -Wno-alias-template-in-declaration-name + + when ($MAPSMOBI_BUILD_TARGET == "yes") { + CFLAGS+=-Wno-deprecated-declarations + } +} + when ($MSAN_TRACK_ORIGIN == "yes") { CFLAGS+=-fsanitize-memory-track-origins=2 } diff --git a/build/conf/coverage_full_instrumentation.conf b/build/conf/coverage_full_instrumentation.conf index de478eb8f8e..207cb51eb49 100644 --- a/build/conf/coverage_full_instrumentation.conf +++ b/build/conf/coverage_full_instrumentation.conf @@ -1,12 +1,18 @@ USE_SCU_VALUE="yes" +_DISABLE_CLANG_COVERAGE_CFLAGS_BASE= +_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC= +_DISABLE_CLANG_COVERAGE_CFLAGS=$_DISABLE_CLANG_COVERAGE_CFLAGS_BASE $_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC + macro POPULATE_CPP_COVERAGE_FLAGS() { when ($CLANG_COVERAGE && $CLANG_COVERAGE != "no") { CFLAGS+=-fprofile-instr-generate -fcoverage-mapping -DCLANG_COVERAGE LDFLAGS+=-fprofile-instr-generate -fcoverage-mapping + _DISABLE_CLANG_COVERAGE_CFLAGS_BASE=-fno-profile-instr-generate -fno-coverage-mapping -UCLANG_COVERAGE when ($CLANG_MCDC_COVERAGE == "yes") { CFLAGS+=-fcoverage-mcdc LDFLAGS+=-fcoverage-mcdc + _DISABLE_CLANG_COVERAGE_CFLAGS_MCDC=-fno-coverage-mcdc } } } diff --git a/build/conf/coverage_selective_instrumentation.conf b/build/conf/coverage_selective_instrumentation.conf index eda61fa65ed..bbf9d6e2fd3 100644 --- a/build/conf/coverage_selective_instrumentation.conf +++ b/build/conf/coverage_selective_instrumentation.conf @@ -1,5 +1,9 @@ USE_SCU_VALUE="no" +_DISABLE_CLANG_COVERAGE_CFLAGS_BASE= +_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC= +_DISABLE_CLANG_COVERAGE_CFLAGS=$_DISABLE_CLANG_COVERAGE_CFLAGS_BASE $_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC + macro POPULATE_CPP_COVERAGE_FLAGS() { SET_CPP_COVERAGE_FLAGS() } @@ -11,10 +15,12 @@ macro _SETUP_CLANG_COVERAGE() { SET(_LD_USE_STDLIB) ENABLE(NEED_PROFILE_RUNTIME) CFLAGS(-fprofile-instr-generate -fcoverage-mapping -DCLANG_COVERAGE) + SET(_DISABLE_CLANG_COVERAGE_CFLAGS_BASE -fno-profile-instr-generate -fno-coverage-mapping -UCLANG_COVERAGE) SET_APPEND(LDFLAGS -fprofile-instr-generate -fcoverage-mapping) when ($CLANG_MCDC_COVERAGE == "yes") { CFLAGS+=-fcoverage-mcdc LDFLAGS+=-fcoverage-mcdc + _DISABLE_CLANG_COVERAGE_CFLAGS_MCDC=-fno-coverage-mcdc } PEERDIR(library/cpp/testing/dump_clang_coverage) } diff --git a/build/platform/lld/ya.make b/build/platform/lld/ya.make index b6f67da9bf4..8f006168e81 100644 --- a/build/platform/lld/ya.make +++ b/build/platform/lld/ya.make @@ -13,6 +13,10 @@ ELSEIF (LLD_VERSION == 18) DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld18.json) ELSEIF (LLD_VERSION == 16) DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld16.json) +ELSEIF (LLD_VERSION == 14) + # Allow empty lld 14 for android +ELSE() + MESSAGE(FATAL_ERROR "Unsupported LLD version ${LLD_VERSION} was required") ENDIF() IF (OS_ANDROID) diff --git a/build/ymake.core.conf b/build/ymake.core.conf index bcbe08c53b0..8da1d2fa303 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -17,7 +17,7 @@ CYTHON_FAKEID=16618405 JAVA_FAKEID=9293095 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 -DOCS_FAKEID=0 +DOCS_FAKEID=1 # Change of this value will invalidate some parts of configure cache # but will not affect builds anyhow (except tests referring build/ directory) @@ -3436,10 +3436,10 @@ macro _SRC_c_nodeps(SRC, OUTFILE, INC...) { # generated sources ## tag:src-processing when ($CLANG == "yes") { - _LANG_CFLAGS_RL=-Wno-implicit-fallthrough -fno-profile-instr-generate -fno-coverage-mapping - _LANG_CFLAGS_SWG=-Wno-deprecated-declarations -fno-profile-instr-generate -fno-coverage-mapping - _LANG_CFLAGS_LEX=-Wno-unused-variable -fno-profile-instr-generate -fno-coverage-mapping - _LANG_CFLAGS_BISON=-Wno-unused-but-set-variable -Wno-deprecated-copy -fno-profile-instr-generate -fno-coverage-mapping + _LANG_CFLAGS_RL=-Wno-implicit-fallthrough $_DISABLE_CLANG_COVERAGE_CFLAGS + _LANG_CFLAGS_SWG=-Wno-deprecated-declarations $_DISABLE_CLANG_COVERAGE_CFLAGS + _LANG_CFLAGS_LEX=-Wno-unused-variable $_DISABLE_CLANG_COVERAGE_CFLAGS + _LANG_CFLAGS_BISON=-Wno-unused-but-set-variable -Wno-deprecated-copy $_DISABLE_CLANG_COVERAGE_CFLAGS } otherwise { _LANG_CFLAGS_RL=-Wno-implicit-fallthrough diff --git a/yql/essentials/core/common_opt/yql_co_pgselect.cpp b/yql/essentials/core/common_opt/yql_co_pgselect.cpp index da8bf3fe534..db7b6d31676 100644 --- a/yql/essentials/core/common_opt/yql_co_pgselect.cpp +++ b/yql/essentials/core/common_opt/yql_co_pgselect.cpp @@ -1765,31 +1765,25 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle } TExprNode::TPtr left, right; - if (!IsEquality(andTerm, left, right)) { + if (!IsMemberEquality(andTerm, predicate->Head().Head(), left, right)) { bad = true; break; } bool leftOnLeft; - if (left->IsCallable("Member") && &left->Head() == &predicate->Head().Head()) { + { auto inputPtr = memberToInput.FindPtr(left->Child(1)->Content()); YQL_ENSURE(inputPtr); leftOnLeft = leftIdxs.contains(*inputPtr); (leftOnLeft ? leftColumns : rightColumns).push_back(left->ChildPtr(1)); - } else { - bad = true; - break; } bool rightOnRight; - if (right->IsCallable("Member") && &right->Head() == &predicate->Head().Head()) { + { auto inputPtr = memberToInput.FindPtr(right->Child(1)->Content()); YQL_ENSURE(inputPtr); rightOnRight = rightIdxs.contains(*inputPtr); (rightOnRight ? rightColumns : leftColumns).push_back(right->ChildPtr(1)); - } else { - bad = true; - break; } if (leftOnLeft != rightOnRight) { diff --git a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp index 4953b4c4ee6..69435bad401 100644 --- a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp +++ b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp @@ -955,62 +955,48 @@ private: }; TExprNode::TPtr DecayCrossJoinIntoInner(TExprNode::TPtr equiJoin, const TExprNode::TPtr& predicate, - const TJoinLabels& labels, ui32 index1, ui32 index2, const TExprNode& row, const THashMap<TString, TString>& backRenameMap, - const TParentsMap& parentsMap, TExprContext& ctx, bool rotateJoinTree) { - YQL_ENSURE(index1 != index2); + const TJoinLabels& labels, const TExprNode& row, const THashMap<TString, TString>& backRenameMap, + TExprContext& ctx, bool rotateJoinTree) +{ TExprNode::TPtr left, right; - if (!IsEquality(predicate, left, right)) { + if (!IsMemberEquality(predicate, row, left, right)) { return equiJoin; } - TSet<ui32> leftInputs, rightInputs; - TSet<TStringBuf> usedFields; - GatherJoinInputs(left, row, parentsMap, backRenameMap, labels, leftInputs, usedFields); - GatherJoinInputs(right, row, parentsMap, backRenameMap, labels, rightInputs, usedFields); - bool good = false; - if (leftInputs.size() == 1 && rightInputs.size() == 1) { - if (*leftInputs.begin() == index1 && *rightInputs.begin() == index2) { - good = true; - } else if (*leftInputs.begin() == index2 && *rightInputs.begin() == index1) { - good = true; + TVector<ui32> inputs; + TVector<TStringBuf> columnLabels; + TVector<TStringBuf> columns; + for (auto member : { left, right }) { + // rename used fields + TStringBuf memberName(member->Child(1)->Content()); + if (auto renamed = backRenameMap.FindPtr(memberName)) { + memberName = *renamed; } + + TStringBuf label; + TStringBuf column; + SplitTableName(memberName, label, column); + TMaybe<ui32> maybeIndex = labels.FindInputIndex(label); + YQL_ENSURE(maybeIndex, "Unable to find input for label " << ToString(label).Quote()); + inputs.push_back(*maybeIndex); + columnLabels.push_back(label); + columns.push_back(column); } - if (!good) { + YQL_ENSURE(inputs.size() == 2 && inputs.size() == columnLabels.size() && columnLabels.size() == columns.size()); + if (inputs.front() == inputs.back()) { return equiJoin; } auto inputsCount = equiJoin->ChildrenSize() - 2; auto joinTree = equiJoin->Child(inputsCount); - if (!IsRequiredSide(joinTree, labels, index1).first || - !IsRequiredSide(joinTree, labels, index2).first) { - return equiJoin; - } - - TStringBuf label1, column1, label2, column2; - if (left->IsCallable("Member") && left->Child(0) == &row) { - auto x = left->Tail().Content(); - if (auto ptr = backRenameMap.FindPtr(x)) { - x = *ptr; - } - - SplitTableName(x, label1, column1); - } else { - return equiJoin; - } - - if (right->IsCallable("Member") && right->Child(0) == &row) { - auto x = right->Tail().Content(); - if (auto ptr = backRenameMap.FindPtr(x)) { - x = *ptr; - } - - SplitTableName(x, label2, column2); - } else { + if (!IsRequiredSide(joinTree, labels, inputs.front()).first || + !IsRequiredSide(joinTree, labels, inputs.back()).first) + { return equiJoin; } - TJoinTreeRebuilder rebuilder(joinTree, label1, column1, label2, column2, ctx, rotateJoinTree); + TJoinTreeRebuilder rebuilder(joinTree, columnLabels.front(), columns.front(), columnLabels.back(), columns.back(), ctx, rotateJoinTree); auto newJoinTree = rebuilder.Run(); return ctx.ChangeChild(*equiJoin, inputsCount, std::move(newJoinTree)); } @@ -1208,10 +1194,8 @@ TExprBase HandleEqualityFilterOverJoin(const TCoFlatMapBase& node, const TJoinLa THashSet<ui32> makeNoNullInputs; for (auto pred : andComponents) { TExprNode::TPtr left, right; - if (!IsEquality(pred, left, right) || - !left->IsCallable("Member") || left->Child(0) != &row || - !right->IsCallable("Member") || right->Child(0) != &row) - { + // TODO: handle case IsEquality() && !IsMemberEquality() + if (!IsMemberEquality(pred, row, left, right)) { rest.push_back(pred); continue; } @@ -1591,7 +1575,7 @@ TExprBase FlatMapOverEquiJoin( if (!IsEqualityFilterOverJoinEnabled(types) && inputs.size() == 2) { auto newJoin = DecayCrossJoinIntoInner(equiJoin.Ptr(), andTerm, - labels, *inputs.begin(), *(++inputs.begin()), row, backRenameMap, parentsMap, ctx, types->RotateJoinTree); + labels, row, backRenameMap, ctx, types->RotateJoinTree); if (newJoin != equiJoin.Ptr()) { YQL_CLOG(DEBUG, Core) << "DecayCrossJoinIntoInner"; ret = newJoin; diff --git a/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json index e774ea78084..3f699162782 100644 --- a/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json +++ b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json @@ -189,6 +189,11 @@ "Match": {"Type": "Callable", "Name": "DqCnUnionAll"} }, { + "Name": "TDqCnParallelUnionAll", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnParallelUnionAll"} + }, + { "Name": "TDqCnMap", "Base": "TDqConnection", "Match": {"Type": "Callable", "Name": "DqCnMap"} diff --git a/yql/essentials/core/yql_join.cpp b/yql/essentials/core/yql_join.cpp index c1a81834828..912b690aba1 100644 --- a/yql/essentials/core/yql_join.cpp +++ b/yql/essentials/core/yql_join.cpp @@ -2046,6 +2046,16 @@ bool IsEquality(TExprNode::TPtr predicate, TExprNode::TPtr& left, TExprNode::TPt return false; } +bool IsMemberEquality(const TExprNode::TPtr& predicate, const TExprNode& row, TExprNode::TPtr& leftMember, TExprNode::TPtr& rightMember) { + if (!IsEquality(predicate, leftMember, rightMember)) { + return false; + } + + return + leftMember->IsCallable("Member") && &leftMember->Head() == &row && + rightMember->IsCallable("Member") && &rightMember->Head() == &row; +} + void GatherJoinInputs(const TExprNode::TPtr& expr, const TExprNode& row, const TParentsMap& parentsMap, const THashMap<TString, TString>& backRenameMap, const TJoinLabels& labels, TSet<ui32>& inputs, TSet<TStringBuf>& usedFields) { diff --git a/yql/essentials/core/yql_join.h b/yql/essentials/core/yql_join.h index 1a135c9c32e..1185c9805fd 100644 --- a/yql/essentials/core/yql_join.h +++ b/yql/essentials/core/yql_join.h @@ -172,6 +172,7 @@ void GatherAndTerms(const TExprNode::TPtr& predicate, TExprNode::TListType& andT TExprNode::TPtr FuseAndTerms(TPositionHandle position, const TExprNode::TListType& andTerms, const TExprNode::TPtr& exclude, bool isPg, TExprContext& ctx); bool IsEquality(TExprNode::TPtr predicate, TExprNode::TPtr& left, TExprNode::TPtr& right); +bool IsMemberEquality(const TExprNode::TPtr& predicate, const TExprNode& row, TExprNode::TPtr& leftMember, TExprNode::TPtr& rightMember); void GatherJoinInputs(const TExprNode::TPtr& expr, const TExprNode& row, const TParentsMap& parentsMap, const THashMap<TString, TString>& backRenameMap, diff --git a/yql/essentials/docs/en/builtins/struct.md b/yql/essentials/docs/en/builtins/struct.md index 5376e96a6ae..31766cf9e95 100644 --- a/yql/essentials/docs/en/builtins/struct.md +++ b/yql/essentials/docs/en/builtins/struct.md @@ -316,7 +316,7 @@ SELECT ForceSpreadMembers([('a',1),('a',2),('c',100)],['a','b']); -- (a: 2, b: null) ``` -## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference +## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference {#struct-combining} Combine two structures using one of the four methods (using the provided lambda to merge fields with the same name): diff --git a/yql/essentials/docs/en/builtins/types.md b/yql/essentials/docs/en/builtins/types.md index b1e75055d86..566ee1de1f2 100644 --- a/yql/essentials/docs/en/builtins/types.md +++ b/yql/essentials/docs/en/builtins/types.md @@ -380,7 +380,7 @@ SELECT FormatType(ListTypeHandle( )); -- List<Bool> ``` -### EmptyListTypeHandle and EmptyDictTypeHandle +### EmptyListTypeHandle and EmptyDictTypeHandle {#empty-list-dict-type-handle} Constructs a handle for an empty list or dictionary. @@ -546,7 +546,7 @@ SELECT FormatType(VariantTypeHandle( )); -- Variant<Int32, String> ``` -### VoidTypeHandle and NullTypeHandle +### VoidTypeHandle and NullTypeHandle {#void-null-type-handle} Constructing a handle for Void and Null types, respectively. diff --git a/yql/essentials/docs/en/builtins/window.md b/yql/essentials/docs/en/builtins/window.md index c764a305049..8bc4b44ba6e 100644 --- a/yql/essentials/docs/en/builtins/window.md +++ b/yql/essentials/docs/en/builtins/window.md @@ -91,7 +91,7 @@ item odd lag1 ``` -## FIRST_VALUE / LAST_VALUE +## FIRST_VALUE / LAST_VALUE {#first-last-value} Access values from the first and last rows (using the `ORDER BY` clause for the window) of the [window frame](../syntax/window.md#frame). The only argument is the expression that you need to access. @@ -240,7 +240,7 @@ WINDOW w AS (ORDER BY key); ``` -## SessionState() {#session-state} +## SessionState {#session-state} A non-standard window function `SessionState()` (without arguments) lets you get the session calculation status from [SessionWindow](../syntax/group_by.md#session-window) for the current row. It's allowed only if `SessionWindow()` is present in the `PARTITION BY` section in the window definition. diff --git a/yql/essentials/docs/en/types/primitive.md b/yql/essentials/docs/en/types/primitive.md index 2d80ad4f057..f364de27541 100644 --- a/yql/essentials/docs/en/types/primitive.md +++ b/yql/essentials/docs/en/types/primitive.md @@ -51,7 +51,7 @@ To store numbers (JSON Number) in `JsonDocument`, as well as for arithmetic oper ## Date and time {#datetime} -### Basic types +### Basic types {#datetime-basic} Value range for all basic datetime types: From midnight<sup>1</sup> 01.01.1970 to midnight<sup>1</sup> 01.01.2106. Value range for basic interval type: from -136 years to +136 years. @@ -68,7 +68,7 @@ Value range for basic interval type: from -136 years to +136 years. <sup>1</sup> Midnight is the particular time moment with all _time_ components being zero. -### Extended types +### Extended types {#datetime-extended} Value range for all basic datetime types: From midnight<sup>1</sup> 01.01.144169 BC to midnight<sup>1</sup> 01.01.148107. Value range for basic interval type: from -292277 years to +292277 years. diff --git a/yql/essentials/docs/ru/builtins/struct.md b/yql/essentials/docs/ru/builtins/struct.md index fb897bd19ee..47bdb720c4d 100644 --- a/yql/essentials/docs/ru/builtins/struct.md +++ b/yql/essentials/docs/ru/builtins/struct.md @@ -426,7 +426,7 @@ SELECT ForceSpreadMembers([('a',1),('a',2),('c',100)],['a','b']); -- (a: 2, b: null) ``` -## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference +## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference {#struct-combining} Комбинируют две структуры одним из четырех способов, используя предоставленную функцию для слияния полей с одинаковыми именами: diff --git a/yql/essentials/docs/ru/builtins/types.md b/yql/essentials/docs/ru/builtins/types.md index 2d6eb6548d9..577dd4cbb66 100644 --- a/yql/essentials/docs/ru/builtins/types.md +++ b/yql/essentials/docs/ru/builtins/types.md @@ -598,7 +598,7 @@ SELECT FormatType(ListTypeHandle( )); -- List<Bool> ``` -### EmptyListTypeHandle и EmptyDictTypeHandle +### EmptyListTypeHandle и EmptyDictTypeHandle {#empty-list-dict-type-handle} #### Сигнатура @@ -838,7 +838,7 @@ SELECT FormatType(VariantTypeHandle( )); -- Variant<Int32, String> ``` -### VoidTypeHandle и NullTypeHandle +### VoidTypeHandle и NullTypeHandle {#void-null-type-handle} #### Сигнатура diff --git a/yql/essentials/docs/ru/builtins/window.md b/yql/essentials/docs/ru/builtins/window.md index fbaaa954872..45e8c9f285d 100644 --- a/yql/essentials/docs/ru/builtins/window.md +++ b/yql/essentials/docs/ru/builtins/window.md @@ -88,7 +88,7 @@ item odd lag1 ``` -## FIRST_VALUE / LAST_VALUE +## FIRST_VALUE / LAST_VALUE {#first-last-value} Доступ к значениям из первой и последней (в порядке `ORDER BY` на окне) строк [рамки окна](../syntax/window.md#frame). Единственный аргумент - выражение, к которому необходим доступ. @@ -232,7 +232,7 @@ WINDOW w AS (ORDER BY key); -## SessionState() {#session-state} +## SessionState {#session-state} Нестандартная оконная функция `SessionState()` (без аргументов) позволяет получить состояние расчета сессий из [SessionWindow](../syntax/group_by.md#session-window) для текущей строки. diff --git a/yql/essentials/docs/ru/syntax/lexer.md b/yql/essentials/docs/ru/syntax/lexer.md index 42959357707..4a0ef33b0fe 100644 --- a/yql/essentials/docs/ru/syntax/lexer.md +++ b/yql/essentials/docs/ru/syntax/lexer.md @@ -285,7 +285,7 @@ SELECT ; ``` -### Литерал массива +### Литерал массива {#array-literal} Для построения литерала массива используется функция `PgArray`: diff --git a/yql/essentials/docs/ru/types/primitive.md b/yql/essentials/docs/ru/types/primitive.md index 05c6158cfb6..c745534817d 100644 --- a/yql/essentials/docs/ru/types/primitive.md +++ b/yql/essentials/docs/ru/types/primitive.md @@ -48,7 +48,7 @@ ## Дата и время {#datetime} -### Базовые типы +### Базовые типы {#datetime-basic} Диапазон значений всех базовых временных типов – от полуночи<sup>1</sup> 01.01.1970 до полуночи<sup>1</sup> 01.01.2106. Диапазон значений базового типа интевала – от -136 лет до +136 лет. @@ -65,7 +65,7 @@ <sup>1</sup> Полночью считаем такой момент времени, в которой все компоненты _времени_ равны нулю. -### Расширенные типы +### Расширенные типы {#datetime-extended} Диапазон значений всех расширенных временных типов – от полуночи<sup>1</sup> 01.01.144169 до н.э. до полуночи<sup>1</sup> 01.01.148107. Диапазон значений расширенного типа интевала – от -292277 лет до +292277 лет. diff --git a/yql/essentials/docs/ru/udf/list/postgres.md b/yql/essentials/docs/ru/udf/list/postgres.md index b36e91ce3b5..d138307411e 100644 --- a/yql/essentials/docs/ru/udf/list/postgres.md +++ b/yql/essentials/docs/ru/udf/list/postgres.md @@ -59,7 +59,7 @@ SELECT ; ``` -### Литерал массива +### Литерал массива {#array-literal} Для построения литерала массива используется функция `PgArray`: @@ -287,7 +287,7 @@ FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a'p,'a,b'p,'a,b,c'p {% endnote %} -## Логические операции +## Логические операции {#logic-operations} Для выполнения логических операций используются функции `PgAnd`, `PgOr`, `PgNot`: diff --git a/yt/python/yt/yson/convert.py b/yt/python/yt/yson/convert.py index 947e708979e..23bd457e1a2 100644 --- a/yt/python/yt/yson/convert.py +++ b/yt/python/yt/yson/convert.py @@ -1,6 +1,6 @@ from .yson_types import ( YsonType, YsonString, YsonUnicode, YsonBoolean, YsonInt64, YsonUint64, YsonDouble, - YsonList, YsonMap, YsonEntity) + YsonList, YsonMap, YsonEntity, YsonStringProxy, get_bytes) from .common import YsonError try: @@ -97,7 +97,7 @@ def json_to_yson(json_tree, use_byte_strings=None): result = YsonString(value) elif value is False or value is True: result = YsonBoolean(value) - elif isinstance(value, integer_types): + elif isinstance(value, int): greater_than_max_int64 = value >= 2 ** 63 if greater_than_max_int64: result = YsonUint64(value) @@ -119,7 +119,7 @@ def json_to_yson(json_tree, use_byte_strings=None): return result -def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, annotate_with_types=False): +def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, annotate_with_types=False, use_byte_strings=False): should_annotate_with_types = ( annotate_with_types and not isinstance(yson_tree, list) and @@ -139,7 +139,7 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an return dict( ( encode_key(k), - _yson_to_json(v, print_attributes=print_attributes, annotate_with_types=annotate_with_types), + _yson_to_json(v, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings), ) for k, v in iteritems(d) ) @@ -156,7 +156,10 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an return "int64" elif isinstance(yson_tree, float): return "double" - elif isinstance(yson_tree, str) or isinstance(yson_tree, binary_type): + elif isinstance(yson_tree, text_type): + # TODO: "utf8" + return "string" + elif isinstance(yson_tree, binary_type): return "string" else: raise RuntimeError("Failed to perform yson to json conversion of {!r}, unknown type {!r} to annotate with types".format( @@ -173,7 +176,7 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an result = { "$attributes": process_dict(yson_tree.attributes), - "$value": _yson_to_json(yson_tree, print_attributes=print_attributes, attributes_printed=True, annotate_with_types=value_annotate_with_types), + "$value": _yson_to_json(yson_tree, print_attributes=print_attributes, attributes_printed=True, annotate_with_types=value_annotate_with_types, use_byte_strings=use_byte_strings), } if should_annotate_with_types: @@ -182,13 +185,22 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an return result if isinstance(yson_tree, list): - return [_yson_to_json(element, print_attributes=print_attributes, annotate_with_types=annotate_with_types) for element in yson_tree] + return [_yson_to_json(element, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings) for element in yson_tree] elif isinstance(yson_tree, dict): return process_dict(yson_tree) elif isinstance(yson_tree, YsonEntity): return None - elif isinstance(yson_tree, YsonString) or isinstance(yson_tree, binary_type): - return do_annotate_with_types(yson_tree.decode("utf-8")) + elif isinstance(yson_tree, YsonStringProxy): + tree_value = get_bytes(yson_tree) + if not use_byte_strings: + tree_value = tree_value.decode("utf-8") + return do_annotate_with_types(tree_value) + elif isinstance(yson_tree, binary_type) or isinstance(yson_tree, YsonString): + if use_byte_strings: + tree_value = get_bytes(yson_tree) + else: + tree_value = yson_tree.decode("utf-8") + return do_annotate_with_types(tree_value) elif isinstance(yson_tree, bool) or isinstance(yson_tree, YsonBoolean): return do_annotate_with_types(True if yson_tree else False) else: @@ -205,5 +217,5 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an return do_annotate_with_types(yson_tree) -def yson_to_json(yson_tree, print_attributes=True, annotate_with_types=False): - return _yson_to_json(yson_tree, print_attributes=print_attributes, annotate_with_types=annotate_with_types) +def yson_to_json(yson_tree, print_attributes=True, annotate_with_types=False, use_byte_strings=False): + return _yson_to_json(yson_tree, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings) diff --git a/yt/yt/core/concurrency/fair_throttler.cpp b/yt/yt/core/concurrency/fair_throttler.cpp index 1d3f1edeb41..34559da8474 100644 --- a/yt/yt/core/concurrency/fair_throttler.cpp +++ b/yt/yt/core/concurrency/fair_throttler.cpp @@ -345,17 +345,17 @@ public: TFairThrottlerConfigPtr config) : Logger(logger) , SharedBucket_(sharedBucket) - , Value_(profiler.Counter("/value")) - , Released_(profiler.Counter("/released")) - , WaitTime_(profiler.Timer("/wait_time")) - , Quota_(config->BucketAccumulationTicks, profiler.Gauge("/quota")) - , DistributionPeriod_(config->DistributionPeriod) + , Profiler_(profiler) + , Value_(Profiler_.Counter("/value")) + , Released_(Profiler_.Counter("/released")) + , WaitTime_(Profiler_.Timer("/wait_time")) + , Quota_(config->BucketAccumulationTicks, Profiler_.Gauge("/quota")) { - profiler.AddFuncGauge("/queue_size", MakeStrong(this), [this] { + Profiler_.AddFuncGauge("/queue_size", MakeStrong(this), [this] { return GetQueueTotalAmount(); }); - profiler.AddFuncGauge("/throttled", MakeStrong(this), [this] { + Profiler_.AddFuncGauge("/throttled", MakeStrong(this), [this] { return IsOverdraft(); }); } @@ -568,14 +568,30 @@ public: } } - void SetDistributionPeriod(TDuration distributionPeriod) + void Update( + const TFairThrottlerConfigPtr& config, + const TFairThrottlerBucketConfigPtr& bucketConfig) { - DistributionPeriod_.store(distributionPeriod); - } + Limited_ = bucketConfig->Limit || bucketConfig->RelativeLimit; + DistributionPeriod_.store(config->DistributionPeriod); - void SetLimited(bool limited) - { - Limited_ = limited; + if (const auto& limit = bucketConfig->GetLimit(config->TotalLimit)) { + if (!Limit_) { + Limit_ = Profiler_.Gauge("/limit"); + } + Limit_->Update(limit.value()); + } else { + Limit_ = std::nullopt; + } + + if (const auto& guarantee = bucketConfig->GetGuarantee(config->TotalLimit)) { + if (!Guarantee_) { + Guarantee_ = Profiler_.Gauge("/guarantee"); + } + Guarantee_->Update(guarantee.value()); + } else { + Guarantee_ = std::nullopt; + } } bool IsLimited() const @@ -588,9 +604,12 @@ private: TSharedBucketPtr SharedBucket_; + NProfiling::TProfiler Profiler_; NProfiling::TCounter Value_; NProfiling::TCounter Released_; NProfiling::TEventTimer WaitTime_; + std::optional<NProfiling::TGauge> Limit_ = std::nullopt; + std::optional<NProfiling::TGauge> Guarantee_ = std::nullopt; TLeakyCounter Quota_; std::atomic<i64> EstimatedLimit_ = 0; @@ -599,7 +618,7 @@ private: std::atomic<bool> Limited_ = false; - std::atomic<TDuration> DistributionPeriod_; + std::atomic<TDuration> DistributionPeriod_{}; YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, Lock_); std::deque<TBucketThrottleRequestPtr> Queue_; @@ -655,10 +674,10 @@ IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler( auto guard = Guard(Lock_); if (auto it = Buckets_.find(name); it != Buckets_.end()) { - it->second.Throttler->SetLimited(config->Limit || config->RelativeLimit); + it->second.Throttler->Update(Config_, config); it->second.Config = std::move(config); - it->second.Throttler->SetDistributionPeriod(Config_->DistributionPeriod); + return it->second.Throttler; } @@ -668,7 +687,7 @@ IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler( SharedBucket_, Config_); - throttler->SetLimited(config->Limit || config->RelativeLimit); + throttler->Update(Config_, config); IIpcBucketPtr state; if (Ipc_) { diff --git a/yt/yt/core/misc/protobuf_helpers.cpp b/yt/yt/core/misc/protobuf_helpers.cpp index 2ed1448c919..62c0050cc2e 100644 --- a/yt/yt/core/misc/protobuf_helpers.cpp +++ b/yt/yt/core/misc/protobuf_helpers.cpp @@ -182,7 +182,9 @@ bool TryDeserializeProtoWithEnvelope( return false; } - const auto* fixedHeader = reinterpret_cast<const TEnvelopeFixedHeader*>(data.Begin()); + TEnvelopeFixedHeader fixedHeaderStorage; + ::memcpy(&fixedHeaderStorage, data.Begin(), sizeof(fixedHeaderStorage)); + const auto* fixedHeader = &fixedHeaderStorage; const char* sourceHeader = data.Begin() + sizeof(TEnvelopeFixedHeader); if (fixedHeader->EnvelopeSize + sizeof(*fixedHeader) > data.Size()) { return false; |