aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYDBot <ydbot@ydb.tech>2025-07-30 00:52:30 +0000
committerYDBot <ydbot@ydb.tech>2025-07-30 00:52:30 +0000
commitaec6b4d77493bc833e78f820c3cb5632a20db790 (patch)
tree3fb1829cfae953b525c3488e0e6d6e7f207d4773
parent80bf0b792ecdd91b5386bcb559bc40d3abf7a225 (diff)
parent94ab428afaf835a102a527535e6428f55180e070 (diff)
downloadydb-aec6b4d77493bc833e78f820c3cb5632a20db790.tar.gz
Sync branches 250730-0051
-rw-r--r--build/conf/compilers/gnu_compiler.conf42
-rw-r--r--build/conf/coverage_full_instrumentation.conf6
-rw-r--r--build/conf/coverage_selective_instrumentation.conf6
-rw-r--r--build/platform/lld/ya.make4
-rw-r--r--build/ymake.core.conf10
-rw-r--r--yql/essentials/core/common_opt/yql_co_pgselect.cpp12
-rw-r--r--yql/essentials/core/common_opt/yql_flatmap_over_join.cpp76
-rw-r--r--yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json5
-rw-r--r--yql/essentials/core/yql_join.cpp10
-rw-r--r--yql/essentials/core/yql_join.h1
-rw-r--r--yql/essentials/docs/en/builtins/struct.md2
-rw-r--r--yql/essentials/docs/en/builtins/types.md4
-rw-r--r--yql/essentials/docs/en/builtins/window.md4
-rw-r--r--yql/essentials/docs/en/types/primitive.md4
-rw-r--r--yql/essentials/docs/ru/builtins/struct.md2
-rw-r--r--yql/essentials/docs/ru/builtins/types.md4
-rw-r--r--yql/essentials/docs/ru/builtins/window.md4
-rw-r--r--yql/essentials/docs/ru/syntax/lexer.md2
-rw-r--r--yql/essentials/docs/ru/types/primitive.md4
-rw-r--r--yql/essentials/docs/ru/udf/list/postgres.md4
-rw-r--r--yt/python/yt/yson/convert.py34
-rw-r--r--yt/yt/core/concurrency/fair_throttler.cpp53
-rw-r--r--yt/yt/core/misc/protobuf_helpers.cpp4
23 files changed, 191 insertions, 106 deletions
diff --git a/build/conf/compilers/gnu_compiler.conf b/build/conf/compilers/gnu_compiler.conf
index dd6c2351b6a..dc2ba68a520 100644
--- a/build/conf/compilers/gnu_compiler.conf
+++ b/build/conf/compilers/gnu_compiler.conf
@@ -80,6 +80,48 @@ when ($CLANG18 == "yes") {
}
}
+when ($CLANG20 == "yes") {
+ CFLAGS+=-Wno-array-parameter -Wno-deprecate-lax-vec-conv-all -Wno-unqualified-std-cast-call -Wno-unused-but-set-parameter -Wno-implicit-function-declaration -Wno-int-conversion -Wno-incompatible-function-pointer-types -Wno-address-of-packed-member
+ CFLAGS+=-Wno-deprecated-this-capture -Wno-missing-designated-field-initializers \
+ -Wno-format -Wno-vla-cxx-extension -Wno-invalid-offsetof \
+ # warnings improved in clang20
+ CFLAGS+=-Wno-ignored-qualifiers \
+ -Wno-implicit-int \
+ -Wno-array-bounds \
+ -Wno-deprecated-literal-operator \
+ -Wno-unused-parameter \
+ -Wno-dangling-gsl \
+ -Wno-macro-redefined \
+ -Wno-extra-qualification \
+ -Wno-return-stack-address \
+ -Wno-shorten-64-to-32 \
+ -Wno-nontrivial-memcall \
+ -Wno-tautological-constant-out-of-range-compare \
+ -Wno-unused-but-set-variable \
+ -Wno-explicit-specialization-storage-class \
+ -Wno-deprecated-declarations \
+ -Wno-dangling-assignment \
+ -Wno-cast-function-type-mismatch \
+ -Wno-delete-incomplete \
+ -Wno-unused-private-field \
+ -Wno-unused-value \
+ -Wno-sign-compare \
+ -Wno-braced-scalar-init \
+ -Wno-main \
+ -Wno-pointer-bool-conversion
+
+ # new warnings for clang20
+ CFLAGS+=-Wno-missing-template-arg-list-after-template-kw \
+ -Wno-strict-primary-template-shadow \
+ -Wno-extraneous-template-head \
+ -Wno-dangling-assignment-gsl \
+ -Wno-alias-template-in-declaration-name
+
+ when ($MAPSMOBI_BUILD_TARGET == "yes") {
+ CFLAGS+=-Wno-deprecated-declarations
+ }
+}
+
when ($MSAN_TRACK_ORIGIN == "yes") {
CFLAGS+=-fsanitize-memory-track-origins=2
}
diff --git a/build/conf/coverage_full_instrumentation.conf b/build/conf/coverage_full_instrumentation.conf
index de478eb8f8e..207cb51eb49 100644
--- a/build/conf/coverage_full_instrumentation.conf
+++ b/build/conf/coverage_full_instrumentation.conf
@@ -1,12 +1,18 @@
USE_SCU_VALUE="yes"
+_DISABLE_CLANG_COVERAGE_CFLAGS_BASE=
+_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC=
+_DISABLE_CLANG_COVERAGE_CFLAGS=$_DISABLE_CLANG_COVERAGE_CFLAGS_BASE $_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC
+
macro POPULATE_CPP_COVERAGE_FLAGS() {
when ($CLANG_COVERAGE && $CLANG_COVERAGE != "no") {
CFLAGS+=-fprofile-instr-generate -fcoverage-mapping -DCLANG_COVERAGE
LDFLAGS+=-fprofile-instr-generate -fcoverage-mapping
+ _DISABLE_CLANG_COVERAGE_CFLAGS_BASE=-fno-profile-instr-generate -fno-coverage-mapping -UCLANG_COVERAGE
when ($CLANG_MCDC_COVERAGE == "yes") {
CFLAGS+=-fcoverage-mcdc
LDFLAGS+=-fcoverage-mcdc
+ _DISABLE_CLANG_COVERAGE_CFLAGS_MCDC=-fno-coverage-mcdc
}
}
}
diff --git a/build/conf/coverage_selective_instrumentation.conf b/build/conf/coverage_selective_instrumentation.conf
index eda61fa65ed..bbf9d6e2fd3 100644
--- a/build/conf/coverage_selective_instrumentation.conf
+++ b/build/conf/coverage_selective_instrumentation.conf
@@ -1,5 +1,9 @@
USE_SCU_VALUE="no"
+_DISABLE_CLANG_COVERAGE_CFLAGS_BASE=
+_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC=
+_DISABLE_CLANG_COVERAGE_CFLAGS=$_DISABLE_CLANG_COVERAGE_CFLAGS_BASE $_DISABLE_CLANG_COVERAGE_CFLAGS_MCDC
+
macro POPULATE_CPP_COVERAGE_FLAGS() {
SET_CPP_COVERAGE_FLAGS()
}
@@ -11,10 +15,12 @@ macro _SETUP_CLANG_COVERAGE() {
SET(_LD_USE_STDLIB)
ENABLE(NEED_PROFILE_RUNTIME)
CFLAGS(-fprofile-instr-generate -fcoverage-mapping -DCLANG_COVERAGE)
+ SET(_DISABLE_CLANG_COVERAGE_CFLAGS_BASE -fno-profile-instr-generate -fno-coverage-mapping -UCLANG_COVERAGE)
SET_APPEND(LDFLAGS -fprofile-instr-generate -fcoverage-mapping)
when ($CLANG_MCDC_COVERAGE == "yes") {
CFLAGS+=-fcoverage-mcdc
LDFLAGS+=-fcoverage-mcdc
+ _DISABLE_CLANG_COVERAGE_CFLAGS_MCDC=-fno-coverage-mcdc
}
PEERDIR(library/cpp/testing/dump_clang_coverage)
}
diff --git a/build/platform/lld/ya.make b/build/platform/lld/ya.make
index b6f67da9bf4..8f006168e81 100644
--- a/build/platform/lld/ya.make
+++ b/build/platform/lld/ya.make
@@ -13,6 +13,10 @@ ELSEIF (LLD_VERSION == 18)
DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld18.json)
ELSEIF (LLD_VERSION == 16)
DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(LLD_ROOT lld16.json)
+ELSEIF (LLD_VERSION == 14)
+ # Allow empty lld 14 for android
+ELSE()
+ MESSAGE(FATAL_ERROR "Unsupported LLD version ${LLD_VERSION} was required")
ENDIF()
IF (OS_ANDROID)
diff --git a/build/ymake.core.conf b/build/ymake.core.conf
index bcbe08c53b0..8da1d2fa303 100644
--- a/build/ymake.core.conf
+++ b/build/ymake.core.conf
@@ -17,7 +17,7 @@ CYTHON_FAKEID=16618405
JAVA_FAKEID=9293095
PROTO_FAKEID=0
FBS_FAKEID=2024-03-13
-DOCS_FAKEID=0
+DOCS_FAKEID=1
# Change of this value will invalidate some parts of configure cache
# but will not affect builds anyhow (except tests referring build/ directory)
@@ -3436,10 +3436,10 @@ macro _SRC_c_nodeps(SRC, OUTFILE, INC...) {
# generated sources
## tag:src-processing
when ($CLANG == "yes") {
- _LANG_CFLAGS_RL=-Wno-implicit-fallthrough -fno-profile-instr-generate -fno-coverage-mapping
- _LANG_CFLAGS_SWG=-Wno-deprecated-declarations -fno-profile-instr-generate -fno-coverage-mapping
- _LANG_CFLAGS_LEX=-Wno-unused-variable -fno-profile-instr-generate -fno-coverage-mapping
- _LANG_CFLAGS_BISON=-Wno-unused-but-set-variable -Wno-deprecated-copy -fno-profile-instr-generate -fno-coverage-mapping
+ _LANG_CFLAGS_RL=-Wno-implicit-fallthrough $_DISABLE_CLANG_COVERAGE_CFLAGS
+ _LANG_CFLAGS_SWG=-Wno-deprecated-declarations $_DISABLE_CLANG_COVERAGE_CFLAGS
+ _LANG_CFLAGS_LEX=-Wno-unused-variable $_DISABLE_CLANG_COVERAGE_CFLAGS
+ _LANG_CFLAGS_BISON=-Wno-unused-but-set-variable -Wno-deprecated-copy $_DISABLE_CLANG_COVERAGE_CFLAGS
}
otherwise {
_LANG_CFLAGS_RL=-Wno-implicit-fallthrough
diff --git a/yql/essentials/core/common_opt/yql_co_pgselect.cpp b/yql/essentials/core/common_opt/yql_co_pgselect.cpp
index da8bf3fe534..db7b6d31676 100644
--- a/yql/essentials/core/common_opt/yql_co_pgselect.cpp
+++ b/yql/essentials/core/common_opt/yql_co_pgselect.cpp
@@ -1765,31 +1765,25 @@ std::tuple<TVector<ui32>, TExprNode::TListType> BuildJoinGroups(TPositionHandle
}
TExprNode::TPtr left, right;
- if (!IsEquality(andTerm, left, right)) {
+ if (!IsMemberEquality(andTerm, predicate->Head().Head(), left, right)) {
bad = true;
break;
}
bool leftOnLeft;
- if (left->IsCallable("Member") && &left->Head() == &predicate->Head().Head()) {
+ {
auto inputPtr = memberToInput.FindPtr(left->Child(1)->Content());
YQL_ENSURE(inputPtr);
leftOnLeft = leftIdxs.contains(*inputPtr);
(leftOnLeft ? leftColumns : rightColumns).push_back(left->ChildPtr(1));
- } else {
- bad = true;
- break;
}
bool rightOnRight;
- if (right->IsCallable("Member") && &right->Head() == &predicate->Head().Head()) {
+ {
auto inputPtr = memberToInput.FindPtr(right->Child(1)->Content());
YQL_ENSURE(inputPtr);
rightOnRight = rightIdxs.contains(*inputPtr);
(rightOnRight ? rightColumns : leftColumns).push_back(right->ChildPtr(1));
- } else {
- bad = true;
- break;
}
if (leftOnLeft != rightOnRight) {
diff --git a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp
index 4953b4c4ee6..69435bad401 100644
--- a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp
+++ b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp
@@ -955,62 +955,48 @@ private:
};
TExprNode::TPtr DecayCrossJoinIntoInner(TExprNode::TPtr equiJoin, const TExprNode::TPtr& predicate,
- const TJoinLabels& labels, ui32 index1, ui32 index2, const TExprNode& row, const THashMap<TString, TString>& backRenameMap,
- const TParentsMap& parentsMap, TExprContext& ctx, bool rotateJoinTree) {
- YQL_ENSURE(index1 != index2);
+ const TJoinLabels& labels, const TExprNode& row, const THashMap<TString, TString>& backRenameMap,
+ TExprContext& ctx, bool rotateJoinTree)
+{
TExprNode::TPtr left, right;
- if (!IsEquality(predicate, left, right)) {
+ if (!IsMemberEquality(predicate, row, left, right)) {
return equiJoin;
}
- TSet<ui32> leftInputs, rightInputs;
- TSet<TStringBuf> usedFields;
- GatherJoinInputs(left, row, parentsMap, backRenameMap, labels, leftInputs, usedFields);
- GatherJoinInputs(right, row, parentsMap, backRenameMap, labels, rightInputs, usedFields);
- bool good = false;
- if (leftInputs.size() == 1 && rightInputs.size() == 1) {
- if (*leftInputs.begin() == index1 && *rightInputs.begin() == index2) {
- good = true;
- } else if (*leftInputs.begin() == index2 && *rightInputs.begin() == index1) {
- good = true;
+ TVector<ui32> inputs;
+ TVector<TStringBuf> columnLabels;
+ TVector<TStringBuf> columns;
+ for (auto member : { left, right }) {
+ // rename used fields
+ TStringBuf memberName(member->Child(1)->Content());
+ if (auto renamed = backRenameMap.FindPtr(memberName)) {
+ memberName = *renamed;
}
+
+ TStringBuf label;
+ TStringBuf column;
+ SplitTableName(memberName, label, column);
+ TMaybe<ui32> maybeIndex = labels.FindInputIndex(label);
+ YQL_ENSURE(maybeIndex, "Unable to find input for label " << ToString(label).Quote());
+ inputs.push_back(*maybeIndex);
+ columnLabels.push_back(label);
+ columns.push_back(column);
}
- if (!good) {
+ YQL_ENSURE(inputs.size() == 2 && inputs.size() == columnLabels.size() && columnLabels.size() == columns.size());
+ if (inputs.front() == inputs.back()) {
return equiJoin;
}
auto inputsCount = equiJoin->ChildrenSize() - 2;
auto joinTree = equiJoin->Child(inputsCount);
- if (!IsRequiredSide(joinTree, labels, index1).first ||
- !IsRequiredSide(joinTree, labels, index2).first) {
- return equiJoin;
- }
-
- TStringBuf label1, column1, label2, column2;
- if (left->IsCallable("Member") && left->Child(0) == &row) {
- auto x = left->Tail().Content();
- if (auto ptr = backRenameMap.FindPtr(x)) {
- x = *ptr;
- }
-
- SplitTableName(x, label1, column1);
- } else {
- return equiJoin;
- }
-
- if (right->IsCallable("Member") && right->Child(0) == &row) {
- auto x = right->Tail().Content();
- if (auto ptr = backRenameMap.FindPtr(x)) {
- x = *ptr;
- }
-
- SplitTableName(x, label2, column2);
- } else {
+ if (!IsRequiredSide(joinTree, labels, inputs.front()).first ||
+ !IsRequiredSide(joinTree, labels, inputs.back()).first)
+ {
return equiJoin;
}
- TJoinTreeRebuilder rebuilder(joinTree, label1, column1, label2, column2, ctx, rotateJoinTree);
+ TJoinTreeRebuilder rebuilder(joinTree, columnLabels.front(), columns.front(), columnLabels.back(), columns.back(), ctx, rotateJoinTree);
auto newJoinTree = rebuilder.Run();
return ctx.ChangeChild(*equiJoin, inputsCount, std::move(newJoinTree));
}
@@ -1208,10 +1194,8 @@ TExprBase HandleEqualityFilterOverJoin(const TCoFlatMapBase& node, const TJoinLa
THashSet<ui32> makeNoNullInputs;
for (auto pred : andComponents) {
TExprNode::TPtr left, right;
- if (!IsEquality(pred, left, right) ||
- !left->IsCallable("Member") || left->Child(0) != &row ||
- !right->IsCallable("Member") || right->Child(0) != &row)
- {
+ // TODO: handle case IsEquality() && !IsMemberEquality()
+ if (!IsMemberEquality(pred, row, left, right)) {
rest.push_back(pred);
continue;
}
@@ -1591,7 +1575,7 @@ TExprBase FlatMapOverEquiJoin(
if (!IsEqualityFilterOverJoinEnabled(types) && inputs.size() == 2) {
auto newJoin = DecayCrossJoinIntoInner(equiJoin.Ptr(), andTerm,
- labels, *inputs.begin(), *(++inputs.begin()), row, backRenameMap, parentsMap, ctx, types->RotateJoinTree);
+ labels, row, backRenameMap, ctx, types->RotateJoinTree);
if (newJoin != equiJoin.Ptr()) {
YQL_CLOG(DEBUG, Core) << "DecayCrossJoinIntoInner";
ret = newJoin;
diff --git a/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json
index e774ea78084..3f699162782 100644
--- a/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json
+++ b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json
@@ -189,6 +189,11 @@
"Match": {"Type": "Callable", "Name": "DqCnUnionAll"}
},
{
+ "Name": "TDqCnParallelUnionAll",
+ "Base": "TDqConnection",
+ "Match": {"Type": "Callable", "Name": "DqCnParallelUnionAll"}
+ },
+ {
"Name": "TDqCnMap",
"Base": "TDqConnection",
"Match": {"Type": "Callable", "Name": "DqCnMap"}
diff --git a/yql/essentials/core/yql_join.cpp b/yql/essentials/core/yql_join.cpp
index c1a81834828..912b690aba1 100644
--- a/yql/essentials/core/yql_join.cpp
+++ b/yql/essentials/core/yql_join.cpp
@@ -2046,6 +2046,16 @@ bool IsEquality(TExprNode::TPtr predicate, TExprNode::TPtr& left, TExprNode::TPt
return false;
}
+bool IsMemberEquality(const TExprNode::TPtr& predicate, const TExprNode& row, TExprNode::TPtr& leftMember, TExprNode::TPtr& rightMember) {
+ if (!IsEquality(predicate, leftMember, rightMember)) {
+ return false;
+ }
+
+ return
+ leftMember->IsCallable("Member") && &leftMember->Head() == &row &&
+ rightMember->IsCallable("Member") && &rightMember->Head() == &row;
+}
+
void GatherJoinInputs(const TExprNode::TPtr& expr, const TExprNode& row,
const TParentsMap& parentsMap, const THashMap<TString, TString>& backRenameMap,
const TJoinLabels& labels, TSet<ui32>& inputs, TSet<TStringBuf>& usedFields) {
diff --git a/yql/essentials/core/yql_join.h b/yql/essentials/core/yql_join.h
index 1a135c9c32e..1185c9805fd 100644
--- a/yql/essentials/core/yql_join.h
+++ b/yql/essentials/core/yql_join.h
@@ -172,6 +172,7 @@ void GatherAndTerms(const TExprNode::TPtr& predicate, TExprNode::TListType& andT
TExprNode::TPtr FuseAndTerms(TPositionHandle position, const TExprNode::TListType& andTerms, const TExprNode::TPtr& exclude, bool isPg, TExprContext& ctx);
bool IsEquality(TExprNode::TPtr predicate, TExprNode::TPtr& left, TExprNode::TPtr& right);
+bool IsMemberEquality(const TExprNode::TPtr& predicate, const TExprNode& row, TExprNode::TPtr& leftMember, TExprNode::TPtr& rightMember);
void GatherJoinInputs(const TExprNode::TPtr& expr, const TExprNode& row,
const TParentsMap& parentsMap, const THashMap<TString, TString>& backRenameMap,
diff --git a/yql/essentials/docs/en/builtins/struct.md b/yql/essentials/docs/en/builtins/struct.md
index 5376e96a6ae..31766cf9e95 100644
--- a/yql/essentials/docs/en/builtins/struct.md
+++ b/yql/essentials/docs/en/builtins/struct.md
@@ -316,7 +316,7 @@ SELECT
ForceSpreadMembers([('a',1),('a',2),('c',100)],['a','b']); -- (a: 2, b: null)
```
-## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference
+## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference {#struct-combining}
Combine two structures using one of the four methods (using the provided lambda to merge fields with the same name):
diff --git a/yql/essentials/docs/en/builtins/types.md b/yql/essentials/docs/en/builtins/types.md
index b1e75055d86..566ee1de1f2 100644
--- a/yql/essentials/docs/en/builtins/types.md
+++ b/yql/essentials/docs/en/builtins/types.md
@@ -380,7 +380,7 @@ SELECT FormatType(ListTypeHandle(
)); -- List<Bool>
```
-### EmptyListTypeHandle and EmptyDictTypeHandle
+### EmptyListTypeHandle and EmptyDictTypeHandle {#empty-list-dict-type-handle}
Constructs a handle for an empty list or dictionary.
@@ -546,7 +546,7 @@ SELECT FormatType(VariantTypeHandle(
)); -- Variant<Int32, String>
```
-### VoidTypeHandle and NullTypeHandle
+### VoidTypeHandle and NullTypeHandle {#void-null-type-handle}
Constructing a handle for Void and Null types, respectively.
diff --git a/yql/essentials/docs/en/builtins/window.md b/yql/essentials/docs/en/builtins/window.md
index c764a305049..8bc4b44ba6e 100644
--- a/yql/essentials/docs/en/builtins/window.md
+++ b/yql/essentials/docs/en/builtins/window.md
@@ -91,7 +91,7 @@ item odd lag1
```
-## FIRST_VALUE / LAST_VALUE
+## FIRST_VALUE / LAST_VALUE {#first-last-value}
Access values from the first and last rows (using the `ORDER BY` clause for the window) of the [window frame](../syntax/window.md#frame). The only argument is the expression that you need to access.
@@ -240,7 +240,7 @@ WINDOW w AS (ORDER BY key);
```
-## SessionState() {#session-state}
+## SessionState {#session-state}
A non-standard window function `SessionState()` (without arguments) lets you get the session calculation status from [SessionWindow](../syntax/group_by.md#session-window) for the current row.
It's allowed only if `SessionWindow()` is present in the `PARTITION BY` section in the window definition.
diff --git a/yql/essentials/docs/en/types/primitive.md b/yql/essentials/docs/en/types/primitive.md
index 2d80ad4f057..f364de27541 100644
--- a/yql/essentials/docs/en/types/primitive.md
+++ b/yql/essentials/docs/en/types/primitive.md
@@ -51,7 +51,7 @@ To store numbers (JSON Number) in `JsonDocument`, as well as for arithmetic oper
## Date and time {#datetime}
-### Basic types
+### Basic types {#datetime-basic}
Value range for all basic datetime types: From midnight<sup>1</sup> 01.01.1970 to midnight<sup>1</sup> 01.01.2106.
Value range for basic interval type: from -136 years to +136 years.
@@ -68,7 +68,7 @@ Value range for basic interval type: from -136 years to +136 years.
<sup>1</sup> Midnight is the particular time moment with all _time_ components being zero.
-### Extended types
+### Extended types {#datetime-extended}
Value range for all basic datetime types: From midnight<sup>1</sup> 01.01.144169 BC to midnight<sup>1</sup> 01.01.148107.
Value range for basic interval type: from -292277 years to +292277 years.
diff --git a/yql/essentials/docs/ru/builtins/struct.md b/yql/essentials/docs/ru/builtins/struct.md
index fb897bd19ee..47bdb720c4d 100644
--- a/yql/essentials/docs/ru/builtins/struct.md
+++ b/yql/essentials/docs/ru/builtins/struct.md
@@ -426,7 +426,7 @@ SELECT
ForceSpreadMembers([('a',1),('a',2),('c',100)],['a','b']); -- (a: 2, b: null)
```
-## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference
+## StructUnion, StructIntersection, StructDifference, StructSymmetricDifference {#struct-combining}
Комбинируют две структуры одним из четырех способов, используя предоставленную функцию для слияния полей с одинаковыми именами:
diff --git a/yql/essentials/docs/ru/builtins/types.md b/yql/essentials/docs/ru/builtins/types.md
index 2d6eb6548d9..577dd4cbb66 100644
--- a/yql/essentials/docs/ru/builtins/types.md
+++ b/yql/essentials/docs/ru/builtins/types.md
@@ -598,7 +598,7 @@ SELECT FormatType(ListTypeHandle(
)); -- List<Bool>
```
-### EmptyListTypeHandle и EmptyDictTypeHandle
+### EmptyListTypeHandle и EmptyDictTypeHandle {#empty-list-dict-type-handle}
#### Сигнатура
@@ -838,7 +838,7 @@ SELECT FormatType(VariantTypeHandle(
)); -- Variant<Int32, String>
```
-### VoidTypeHandle и NullTypeHandle
+### VoidTypeHandle и NullTypeHandle {#void-null-type-handle}
#### Сигнатура
diff --git a/yql/essentials/docs/ru/builtins/window.md b/yql/essentials/docs/ru/builtins/window.md
index fbaaa954872..45e8c9f285d 100644
--- a/yql/essentials/docs/ru/builtins/window.md
+++ b/yql/essentials/docs/ru/builtins/window.md
@@ -88,7 +88,7 @@ item odd lag1
```
-## FIRST_VALUE / LAST_VALUE
+## FIRST_VALUE / LAST_VALUE {#first-last-value}
Доступ к значениям из первой и последней (в порядке `ORDER BY` на окне) строк [рамки окна](../syntax/window.md#frame). Единственный аргумент - выражение, к которому необходим доступ.
@@ -232,7 +232,7 @@ WINDOW w AS (ORDER BY key);
-## SessionState() {#session-state}
+## SessionState {#session-state}
Нестандартная оконная функция `SessionState()` (без аргументов) позволяет получить состояние расчета сессий из [SessionWindow](../syntax/group_by.md#session-window) для текущей строки.
diff --git a/yql/essentials/docs/ru/syntax/lexer.md b/yql/essentials/docs/ru/syntax/lexer.md
index 42959357707..4a0ef33b0fe 100644
--- a/yql/essentials/docs/ru/syntax/lexer.md
+++ b/yql/essentials/docs/ru/syntax/lexer.md
@@ -285,7 +285,7 @@ SELECT
;
```
-### Литерал массива
+### Литерал массива {#array-literal}
Для построения литерала массива используется функция `PgArray`:
diff --git a/yql/essentials/docs/ru/types/primitive.md b/yql/essentials/docs/ru/types/primitive.md
index 05c6158cfb6..c745534817d 100644
--- a/yql/essentials/docs/ru/types/primitive.md
+++ b/yql/essentials/docs/ru/types/primitive.md
@@ -48,7 +48,7 @@
## Дата и время {#datetime}
-### Базовые типы
+### Базовые типы {#datetime-basic}
Диапазон значений всех базовых временных типов – от полуночи<sup>1</sup> 01.01.1970 до полуночи<sup>1</sup> 01.01.2106.
Диапазон значений базового типа интевала – от -136 лет до +136 лет.
@@ -65,7 +65,7 @@
<sup>1</sup> Полночью считаем такой момент времени, в которой все компоненты _времени_ равны нулю.
-### Расширенные типы
+### Расширенные типы {#datetime-extended}
Диапазон значений всех расширенных временных типов – от полуночи<sup>1</sup> 01.01.144169 до н.э. до полуночи<sup>1</sup> 01.01.148107.
Диапазон значений расширенного типа интевала – от -292277 лет до +292277 лет.
diff --git a/yql/essentials/docs/ru/udf/list/postgres.md b/yql/essentials/docs/ru/udf/list/postgres.md
index b36e91ce3b5..d138307411e 100644
--- a/yql/essentials/docs/ru/udf/list/postgres.md
+++ b/yql/essentials/docs/ru/udf/list/postgres.md
@@ -59,7 +59,7 @@ SELECT
;
```
-### Литерал массива
+### Литерал массива {#array-literal}
Для построения литерала массива используется функция `PgArray`:
@@ -287,7 +287,7 @@ FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a'p,'a,b'p,'a,b,c'p
{% endnote %}
-## Логические операции
+## Логические операции {#logic-operations}
Для выполнения логических операций используются функции `PgAnd`, `PgOr`, `PgNot`:
diff --git a/yt/python/yt/yson/convert.py b/yt/python/yt/yson/convert.py
index 947e708979e..23bd457e1a2 100644
--- a/yt/python/yt/yson/convert.py
+++ b/yt/python/yt/yson/convert.py
@@ -1,6 +1,6 @@
from .yson_types import (
YsonType, YsonString, YsonUnicode, YsonBoolean, YsonInt64, YsonUint64, YsonDouble,
- YsonList, YsonMap, YsonEntity)
+ YsonList, YsonMap, YsonEntity, YsonStringProxy, get_bytes)
from .common import YsonError
try:
@@ -97,7 +97,7 @@ def json_to_yson(json_tree, use_byte_strings=None):
result = YsonString(value)
elif value is False or value is True:
result = YsonBoolean(value)
- elif isinstance(value, integer_types):
+ elif isinstance(value, int):
greater_than_max_int64 = value >= 2 ** 63
if greater_than_max_int64:
result = YsonUint64(value)
@@ -119,7 +119,7 @@ def json_to_yson(json_tree, use_byte_strings=None):
return result
-def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, annotate_with_types=False):
+def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, annotate_with_types=False, use_byte_strings=False):
should_annotate_with_types = (
annotate_with_types and
not isinstance(yson_tree, list) and
@@ -139,7 +139,7 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an
return dict(
(
encode_key(k),
- _yson_to_json(v, print_attributes=print_attributes, annotate_with_types=annotate_with_types),
+ _yson_to_json(v, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings),
) for k, v in iteritems(d)
)
@@ -156,7 +156,10 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an
return "int64"
elif isinstance(yson_tree, float):
return "double"
- elif isinstance(yson_tree, str) or isinstance(yson_tree, binary_type):
+ elif isinstance(yson_tree, text_type):
+ # TODO: "utf8"
+ return "string"
+ elif isinstance(yson_tree, binary_type):
return "string"
else:
raise RuntimeError("Failed to perform yson to json conversion of {!r}, unknown type {!r} to annotate with types".format(
@@ -173,7 +176,7 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an
result = {
"$attributes": process_dict(yson_tree.attributes),
- "$value": _yson_to_json(yson_tree, print_attributes=print_attributes, attributes_printed=True, annotate_with_types=value_annotate_with_types),
+ "$value": _yson_to_json(yson_tree, print_attributes=print_attributes, attributes_printed=True, annotate_with_types=value_annotate_with_types, use_byte_strings=use_byte_strings),
}
if should_annotate_with_types:
@@ -182,13 +185,22 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an
return result
if isinstance(yson_tree, list):
- return [_yson_to_json(element, print_attributes=print_attributes, annotate_with_types=annotate_with_types) for element in yson_tree]
+ return [_yson_to_json(element, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings) for element in yson_tree]
elif isinstance(yson_tree, dict):
return process_dict(yson_tree)
elif isinstance(yson_tree, YsonEntity):
return None
- elif isinstance(yson_tree, YsonString) or isinstance(yson_tree, binary_type):
- return do_annotate_with_types(yson_tree.decode("utf-8"))
+ elif isinstance(yson_tree, YsonStringProxy):
+ tree_value = get_bytes(yson_tree)
+ if not use_byte_strings:
+ tree_value = tree_value.decode("utf-8")
+ return do_annotate_with_types(tree_value)
+ elif isinstance(yson_tree, binary_type) or isinstance(yson_tree, YsonString):
+ if use_byte_strings:
+ tree_value = get_bytes(yson_tree)
+ else:
+ tree_value = yson_tree.decode("utf-8")
+ return do_annotate_with_types(tree_value)
elif isinstance(yson_tree, bool) or isinstance(yson_tree, YsonBoolean):
return do_annotate_with_types(True if yson_tree else False)
else:
@@ -205,5 +217,5 @@ def _yson_to_json(yson_tree, print_attributes=True, attributes_printed=False, an
return do_annotate_with_types(yson_tree)
-def yson_to_json(yson_tree, print_attributes=True, annotate_with_types=False):
- return _yson_to_json(yson_tree, print_attributes=print_attributes, annotate_with_types=annotate_with_types)
+def yson_to_json(yson_tree, print_attributes=True, annotate_with_types=False, use_byte_strings=False):
+ return _yson_to_json(yson_tree, print_attributes=print_attributes, annotate_with_types=annotate_with_types, use_byte_strings=use_byte_strings)
diff --git a/yt/yt/core/concurrency/fair_throttler.cpp b/yt/yt/core/concurrency/fair_throttler.cpp
index 1d3f1edeb41..34559da8474 100644
--- a/yt/yt/core/concurrency/fair_throttler.cpp
+++ b/yt/yt/core/concurrency/fair_throttler.cpp
@@ -345,17 +345,17 @@ public:
TFairThrottlerConfigPtr config)
: Logger(logger)
, SharedBucket_(sharedBucket)
- , Value_(profiler.Counter("/value"))
- , Released_(profiler.Counter("/released"))
- , WaitTime_(profiler.Timer("/wait_time"))
- , Quota_(config->BucketAccumulationTicks, profiler.Gauge("/quota"))
- , DistributionPeriod_(config->DistributionPeriod)
+ , Profiler_(profiler)
+ , Value_(Profiler_.Counter("/value"))
+ , Released_(Profiler_.Counter("/released"))
+ , WaitTime_(Profiler_.Timer("/wait_time"))
+ , Quota_(config->BucketAccumulationTicks, Profiler_.Gauge("/quota"))
{
- profiler.AddFuncGauge("/queue_size", MakeStrong(this), [this] {
+ Profiler_.AddFuncGauge("/queue_size", MakeStrong(this), [this] {
return GetQueueTotalAmount();
});
- profiler.AddFuncGauge("/throttled", MakeStrong(this), [this] {
+ Profiler_.AddFuncGauge("/throttled", MakeStrong(this), [this] {
return IsOverdraft();
});
}
@@ -568,14 +568,30 @@ public:
}
}
- void SetDistributionPeriod(TDuration distributionPeriod)
+ void Update(
+ const TFairThrottlerConfigPtr& config,
+ const TFairThrottlerBucketConfigPtr& bucketConfig)
{
- DistributionPeriod_.store(distributionPeriod);
- }
+ Limited_ = bucketConfig->Limit || bucketConfig->RelativeLimit;
+ DistributionPeriod_.store(config->DistributionPeriod);
- void SetLimited(bool limited)
- {
- Limited_ = limited;
+ if (const auto& limit = bucketConfig->GetLimit(config->TotalLimit)) {
+ if (!Limit_) {
+ Limit_ = Profiler_.Gauge("/limit");
+ }
+ Limit_->Update(limit.value());
+ } else {
+ Limit_ = std::nullopt;
+ }
+
+ if (const auto& guarantee = bucketConfig->GetGuarantee(config->TotalLimit)) {
+ if (!Guarantee_) {
+ Guarantee_ = Profiler_.Gauge("/guarantee");
+ }
+ Guarantee_->Update(guarantee.value());
+ } else {
+ Guarantee_ = std::nullopt;
+ }
}
bool IsLimited() const
@@ -588,9 +604,12 @@ private:
TSharedBucketPtr SharedBucket_;
+ NProfiling::TProfiler Profiler_;
NProfiling::TCounter Value_;
NProfiling::TCounter Released_;
NProfiling::TEventTimer WaitTime_;
+ std::optional<NProfiling::TGauge> Limit_ = std::nullopt;
+ std::optional<NProfiling::TGauge> Guarantee_ = std::nullopt;
TLeakyCounter Quota_;
std::atomic<i64> EstimatedLimit_ = 0;
@@ -599,7 +618,7 @@ private:
std::atomic<bool> Limited_ = false;
- std::atomic<TDuration> DistributionPeriod_;
+ std::atomic<TDuration> DistributionPeriod_{};
YT_DECLARE_SPIN_LOCK(NThreading::TSpinLock, Lock_);
std::deque<TBucketThrottleRequestPtr> Queue_;
@@ -655,10 +674,10 @@ IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler(
auto guard = Guard(Lock_);
if (auto it = Buckets_.find(name); it != Buckets_.end()) {
- it->second.Throttler->SetLimited(config->Limit || config->RelativeLimit);
+ it->second.Throttler->Update(Config_, config);
it->second.Config = std::move(config);
- it->second.Throttler->SetDistributionPeriod(Config_->DistributionPeriod);
+
return it->second.Throttler;
}
@@ -668,7 +687,7 @@ IThroughputThrottlerPtr TFairThrottler::CreateBucketThrottler(
SharedBucket_,
Config_);
- throttler->SetLimited(config->Limit || config->RelativeLimit);
+ throttler->Update(Config_, config);
IIpcBucketPtr state;
if (Ipc_) {
diff --git a/yt/yt/core/misc/protobuf_helpers.cpp b/yt/yt/core/misc/protobuf_helpers.cpp
index 2ed1448c919..62c0050cc2e 100644
--- a/yt/yt/core/misc/protobuf_helpers.cpp
+++ b/yt/yt/core/misc/protobuf_helpers.cpp
@@ -182,7 +182,9 @@ bool TryDeserializeProtoWithEnvelope(
return false;
}
- const auto* fixedHeader = reinterpret_cast<const TEnvelopeFixedHeader*>(data.Begin());
+ TEnvelopeFixedHeader fixedHeaderStorage;
+ ::memcpy(&fixedHeaderStorage, data.Begin(), sizeof(fixedHeaderStorage));
+ const auto* fixedHeader = &fixedHeaderStorage;
const char* sourceHeader = data.Begin() + sizeof(TEnvelopeFixedHeader);
if (fixedHeader->EnvelopeSize + sizeof(*fixedHeader) > data.Size()) {
return false;