diff options
author | Maxim Yurchuk <maxim-yurchuk@ydb.tech> | 2024-11-30 22:18:35 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-11-30 22:18:35 +0000 |
commit | b2a5d3a720980a7304cdf814082ebed47bbf1191 (patch) | |
tree | 7ee20fb54151a1468759da4feee3de61f013ff7d /yql | |
parent | 7466d62733bffe5cb040f37b21c5a9a4ad174353 (diff) | |
parent | b4cba2872f6348770b561394bef1668f54abfe27 (diff) | |
download | ydb-b2a5d3a720980a7304cdf814082ebed47bbf1191.tar.gz |
Merge pull request #12156 from ydb-platform/mergelibs-241130-1815
Library import 241130-1815
Diffstat (limited to 'yql')
50 files changed, 692 insertions, 88 deletions
diff --git a/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h new file mode 100644 index 0000000000..c6a9ce8119 --- /dev/null +++ b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.h @@ -0,0 +1,71 @@ +#pragma once + +#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.gen.h> + +#include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> + +namespace NYql::NNodes { + +#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.decl.inl.h> + +namespace NDq { +struct TTopSortSettings { + static inline const TString AscendingSort = "Asc"; + static inline const TString DescendingSort = "Desc"; +}; + +class TDqConnection : public NGenerated::TDqConnectionStub<TExprBase, TCallable, TDqOutput> { +public: + explicit TDqConnection(const TExprNode* node) + : TDqConnectionStub(node) {} + + explicit TDqConnection(const TExprNode::TPtr& node) + : TDqConnectionStub(node) {} + + static bool Match(const TExprNode* node) { + if (!node) { + return false; + } + + if (!node->IsCallable()) { + return false; + } + + if (node->ChildrenSize() < 1) { + return false; + } + + return TDqOutput::Match(node->Child(0)); + } +}; + +class TDqOutputAnnotationBase : public NGenerated::TDqOutputAnnotationBaseStub<TExprBase, TCallable, TCoAtom> { +public: + explicit TDqOutputAnnotationBase(const TExprNode* node) + : TDqOutputAnnotationBaseStub(node) {} + + explicit TDqOutputAnnotationBase(const TExprNode::TPtr& node) + : TDqOutputAnnotationBaseStub(node) {} + + static bool Match(const TExprNode* node) { + if (!node) { + return false; + } + + if (!node->IsCallable()) { + return false; + } + + if (node->ChildrenSize() < 2) { + return false; + } + + return TCoAtom::Match(node->Child(0)) + && TCallable::Match(node->Child(1)); + } +}; +} // namespace NDq + +#include <yql/essentials/core/dq_expr_nodes/dq_expr_nodes.defs.inl.h> + +} // namespace NYql::NNodes diff --git a/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json new file mode 100644 index 0000000000..e9efb8c204 --- /dev/null +++ b/yql/essentials/core/dq_expr_nodes/dq_expr_nodes.json @@ -0,0 +1,316 @@ +{ + "NodeRootType": "TExprBase", + "NodeBuilderBase": "TNodeBuilderBase", + "ListBuilderBase": "TListBuilderBase", + "FreeArgCallableBase": "TFreeArgCallable", + "FreeArgBuilderBase": "TFreeArgCallableBuilderBase", + "Nodes": [ + { + "Name": "TDqJoinKeyTuple", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "LeftLabel", "Type": "TCoAtom"}, + {"Index": 1, "Name": "LeftColumn", "Type": "TCoAtom"}, + {"Index": 2, "Name": "RightLabel", "Type": "TCoAtom"}, + {"Index": 3, "Name": "RightColumn", "Type": "TCoAtom"} + ] + }, + { + "Name": "TDqJoinKeyTupleList", + "ListBase": "TDqJoinKeyTuple" + }, + { + "Name": "TDqJoinBase", + "Base": "TCallable", + "Match": {"Type": "CallableBase"}, + "Builder": {"Generate": "None"}, + "Children": [ + {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"}, + {"Index": 2, "Name": "LeftLabel", "Type": "TExprBase", + "NB": "Atom - if left is real table, Void - otherwise"}, + {"Index": 1, "Name": "RightInput", "Type": "TExprBase"}, + {"Index": 3, "Name": "RightLabel", "Type": "TExprBase", + "NB": "Atom - if right is real table, Void - otherwise"}, + {"Index": 4, "Name": "JoinType", "Type": "TCoAtom"}, + {"Index": 5, "Name": "JoinKeys", "Type": "TDqJoinKeyTupleList"}, + {"Index": 6, "Name": "LeftJoinKeyNames", "Type": "TCoAtomList"}, + {"Index": 7, "Name": "RightJoinKeyNames", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TDqJoin", + "Base": "TDqJoinBase", + "Match": {"Type": "Callable", "Name": "DqJoin"}, + "Children": [ + {"Index": 8, "Name": "JoinAlgo", "Type": "TCoAtom"}, + {"Index": 9, "Name": "Flags", "Type": "TCoAtomList", "Optional": true} + ] + }, + { + "Name": "TDqPhyGraceJoin", + "Base": "TDqJoinBase", + "Match": {"Type": "Callable", "Name": "DqPhyGraceJoin"}, + "Children": [ + {"Index": 8, "Name": "Flags", "Type": "TCoAtomList", "Optional": true} + ] + }, + { + "Name": "TDqPhyMapJoin", + "Base": "TDqJoinBase", + "Match": {"Type": "Callable", "Name": "DqPhyMapJoin"} + }, + { + "Name": "TDqPhyCrossJoin", + "Base": "TDqJoinBase", + "Match": {"Type": "Callable", "Name": "DqPhyCrossJoin"} + }, + { + "Name": "TDqPhyJoinDict", + "Base": "TDqJoinBase", + "Match": {"Type": "Callable", "Name": "DqPhyJoinDict"} + }, + { + "Name": "TDqSource", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqSource"}, + "Children": [ + {"Index": 0, "Name": "DataSource", "Type": "TCallable"}, + {"Index": 1, "Name": "Settings", "Type": "TExprBase"} + ] + }, + { + "Name": "TDqOutputAnnotationBase", + "Base": "TCallable", + "Definition": "Custom", + "Builder": {"Generate": "None"}, + "Children": [ + {"Index": 0, "Name": "Index", "Type": "TCoAtom"}, + {"Index": 1, "Name": "DataSink", "Type": "TCallable"} + ] + }, + { + "Name": "TDqTransform", + "Base": "TDqOutputAnnotationBase", + "Match": {"Type": "Callable", "Name": "DqTransform"}, + "Children": [ + {"Index": 2, "Name": "Type", "Type": "TCoAtom"}, + {"Index": 3, "Name": "InputType", "Type": "TExprBase"}, + {"Index": 4, "Name": "OutputType", "Type": "TExprBase"}, + {"Index": 5, "Name": "Settings", "Type": "TCallable"} + ] + }, + { + "Name": "TDqSink", + "Base": "TDqOutputAnnotationBase", + "Match": {"Type": "Callable", "Name": "DqSink"}, + "Children": [ + {"Index": 2, "Name": "Settings", "Type": "TCallable"} + ] + }, + { + "Name": "TDqStageOutputsList", + "ListBase": "TDqOutputAnnotationBase" + }, + { + "Name": "TDqStageBase", + "Base": "TCallable", + "Match": {"Type": "CallableBase"}, + "Builder": {"Generate": "None"}, + "Children": [ + {"Index": 0, "Name": "Inputs", "Type": "TExprList"}, + {"Index": 1, "Name": "Program", "Type": "TCoLambda"}, + {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"}, + {"Index": 3, "Name": "Outputs", "Type": "TDqStageOutputsList", "Optional": true} + ] + }, + { + "Name": "TDqStage", + "Base": "TDqStageBase", + "Match": {"Type": "Callable", "Name": "DqStage"} + }, + { + "Name": "TDqPhyStage", + "Base": "TDqStageBase", + "Match": {"Type": "Callable", "Name": "DqPhyStage"} + }, + { + "Name": "TDqStageList", + "ListBase": "TDqStageBase" + }, + { + "Name": "TDqPhyStageList", + "ListBase": "TDqPhyStage" + }, + { + "Name": "TDqOutput", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "TDqOutput"}, + "Children": [ + {"Index": 0, "Name": "Stage", "Type": "TDqStageBase"}, + {"Index": 1, "Name": "Index", "Type": "TCoAtom"} + ] + }, + { + "Name": "TDqConnection", + "Base": "TCallable", + "Definition": "Custom", + "Builder": {"Generate": "None"}, + "Children": [ + {"Index": 0, "Name": "Output", "Type": "TDqOutput"} + ] + }, + { + "Name": "TDqCnHashShuffle", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnHashShuffle"}, + "Children": [ + {"Index": 1, "Name": "KeyColumns", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TDqCnBroadcast", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnBroadcast"} + }, + { + "Name": "TDqCnUnionAll", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnUnionAll"} + }, + { + "Name": "TDqCnMap", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnMap"} + }, + { + "Name": "TDqCnStreamLookup", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnStreamLookup"}, + "Children": [ + {"Index": 1, "Name": "LeftLabel", "Type": "TCoAtom"}, + {"Index": 2, "Name": "RightInput", "Type": "TExprBase"}, + {"Index": 3, "Name": "RightLabel", "Type": "TCoAtom"}, + {"Index": 4, "Name": "JoinType", "Type": "TCoAtom"}, + {"Index": 5, "Name": "JoinKeys", "Type": "TDqJoinKeyTupleList"}, + {"Index": 6, "Name": "LeftJoinKeyNames", "Type": "TCoAtomList"}, + {"Index": 7, "Name": "RightJoinKeyNames", "Type": "TCoAtomList"}, + {"Index": 8, "Name": "TTL", "Type": "TCoAtom"}, + {"Index": 9, "Name": "MaxDelayedRows", "Type": "TCoAtom"}, + {"Index": 10, "Name": "MaxCachedRows", "Type": "TCoAtom"} + ] + }, + { + "Name": "TDqCnResult", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnResult"}, + "Children": [ + {"Index": 1, "Name": "ColumnHints", "Type": "TCoAtomList"} + ] + }, + { + "Name": "TDqCnValue", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnValue"} + }, + { + "Name": "TDqSortColumn", + "Base": "TExprBase", + "Match": {"Type": "Tuple"}, + "Children": [ + {"Index": 0, "Name": "Column", "Type": "TCoAtom"}, + {"Index": 1, "Name": "SortDirection", "Type": "TCoAtom"} + ] + }, + { + "Name": "TDqSortColumnList", + "ListBase": "TDqSortColumn" + }, + { + "Name": "TDqCnMerge", + "Base": "TDqConnection", + "Match": {"Type": "Callable", "Name": "DqCnMerge"}, + "Children": [ + {"Index": 1, "Name": "SortColumns", "Type": "TDqSortColumnList"} + ] + }, + { + "Name": "TDqReplicate", + "Base": "TFreeArgCallable", + "Match": {"Type": "Callable", "Name": "DqReplicate"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"} + ] + }, + { + "Name": "TDqQuery", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqQuery!"}, + "Children": [ + {"Index": 0, "Name": "World", "Type": "TExprBase"}, + {"Index": 1, "Name": "SinkStages", "Type": "TDqStageList"} + ] + }, + { + "Name": "TDqPrecompute", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqPrecompute"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"} + ] + }, + { + "Name": "TDqPhyPrecompute", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqPhyPrecompute"}, + "Children": [ + {"Index": 0, "Name": "Connection", "Type": "TDqConnection"} + ] + }, + { + "Name": "TDqSqlExternalFunction", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "SqlExternalFunction"}, + "Children": [ + {"Index": 0, "Name": "TransformType", "Type": "TExprBase"}, + {"Index": 1, "Name": "TransformName", "Type": "TExprBase"}, + {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList"} + ] + }, + { + "Name": "TDqPhyLength", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqPhyLength"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"}, + {"Index": 1, "Name": "Name", "Type": "TCoAtom"} + ] + }, + { + "Name": "TDqReadWrapBase", + "Base": "TExprBase", + "Match": {"Type": "CallableBase"}, + "Builder": {"Generate": "None"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"}, + {"Index": 1, "Name": "Flags", "Type": "TCoAtomList"}, + {"Index": 2, "Name": "Token", "Type": "TCoSecureParam", "Optional": true} + ] + }, + { + "Name": "TDqReadWrap", + "Base": "TDqReadWrapBase", + "Match": {"Type": "Callable", "Name": "DqReadWrap"} + }, + { + "Name": "TDqReadWideWrap", + "Base": "TDqReadWrapBase", + "Match": {"Type": "Callable", "Name": "DqReadWideWrap"} + }, + { + "Name": "TDqReadBlockWideWrap", + "Base": "TDqReadWrapBase", + "Match": {"Type": "Callable", "Name": "DqReadBlockWideWrap"} + } + ] +} diff --git a/yql/essentials/core/dq_expr_nodes/ya.make b/yql/essentials/core/dq_expr_nodes/ya.make new file mode 100644 index 0000000000..e1acd2003e --- /dev/null +++ b/yql/essentials/core/dq_expr_nodes/ya.make @@ -0,0 +1,53 @@ +LIBRARY() + +SRCS( + dq_expr_nodes.h +) + +PEERDIR( + yql/essentials/core/expr_nodes +) + +SRCDIR( + yql/essentials/core/expr_nodes_gen +) + +IF(EXPORT_CMAKE) + RUN_PYTHON3( + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py + yql_expr_nodes_gen.jnj + dq_expr_nodes.json + dq_expr_nodes.gen.h + dq_expr_nodes.decl.inl.h + dq_expr_nodes.defs.inl.h + NDq + IN yql_expr_nodes_gen.jnj + IN dq_expr_nodes.json + OUT dq_expr_nodes.gen.h + OUT dq_expr_nodes.decl.inl.h + OUT dq_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h + ) +ELSE() + RUN_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + dq_expr_nodes.json + dq_expr_nodes.gen.h + dq_expr_nodes.decl.inl.h + dq_expr_nodes.defs.inl.h + NDq + IN yql_expr_nodes_gen.jnj + IN dq_expr_nodes.json + OUT dq_expr_nodes.gen.h + OUT dq_expr_nodes.decl.inl.h + OUT dq_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h + ) +ENDIF() + +END() diff --git a/yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h b/yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h new file mode 100644 index 0000000000..8851127f83 --- /dev/null +++ b/yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.gen.h> + +#include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> + +namespace NYql::NNodes { +#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.decl.inl.h> +#include <yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.defs.inl.h> +} diff --git a/yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.json b/yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.json new file mode 100644 index 0000000000..7ee283859f --- /dev/null +++ b/yql/essentials/core/dqs_expr_nodes/dqs_expr_nodes.json @@ -0,0 +1,50 @@ +{ + "NodeRootType": "TExprBase", + "NodeBuilderBase": "TNodeBuilderBase", + "ListBuilderBase": "TListBuilderBase", + "FreeArgCallableBase": "TFreeArgCallable", + "FreeArgBuilderBase": "TFreeArgCallableBuilderBase", + "Nodes": [ + { + "Name": "TDqWrite", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "DqWrite"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"}, + {"Index": 1, "Name": "Provider", "Type": "TCoAtom"}, + {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList", "Optional": true} + ] + }, + { + "Name": "TDqSourceWrapBase", + "Base": "TExprBase", + "Match": {"Type": "CallableBase"}, + "Children": [ + {"Index": 0, "Name": "Input", "Type": "TExprBase"}, + {"Index": 1, "Name": "DataSource", "Type": "TCoDataSource"}, + {"Index": 2, "Name": "RowType", "Type": "TExprBase"}, + {"Index": 3, "Name": "Settings", "Type": "TExprBase", "Optional": true} + ] + }, + { + "Name": "TDqSourceWrap", + "Base": "TDqSourceWrapBase", + "Match": {"Type": "Callable", "Name": "DqSourceWrap"} + }, + { + "Name": "TDqSourceWideWrap", + "Base": "TDqSourceWrapBase", + "Match": {"Type": "Callable", "Name": "DqSourceWideWrap"} + }, + { + "Name": "TDqSourceWideBlockWrap", + "Base": "TDqSourceWrapBase", + "Match": {"Type": "Callable", "Name": "DqSourceWideBlockWrap"} + }, + { + "Name": "TDqLookupSourceWrap", + "Base": "TDqSourceWrapBase", + "Match": {"Type": "Callable", "Name": "TDqLookupSourceWrap"} + } + ] +} diff --git a/yql/essentials/core/dqs_expr_nodes/ya.make b/yql/essentials/core/dqs_expr_nodes/ya.make new file mode 100644 index 0000000000..35519a2d71 --- /dev/null +++ b/yql/essentials/core/dqs_expr_nodes/ya.make @@ -0,0 +1,53 @@ +LIBRARY() + +SRCS( + dqs_expr_nodes.h +) + +PEERDIR( + yql/essentials/core/expr_nodes +) + +SRCDIR( + yql/essentials/core/expr_nodes_gen +) + +IF(EXPORT_CMAKE) + RUN_PYTHON3( + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/gen/__main__.py + yql_expr_nodes_gen.jnj + dqs_expr_nodes.json + dqs_expr_nodes.gen.h + dqs_expr_nodes.decl.inl.h + dqs_expr_nodes.defs.inl.h + NDq + IN yql_expr_nodes_gen.jnj + IN dqs_expr_nodes.json + OUT dqs_expr_nodes.gen.h + OUT dqs_expr_nodes.decl.inl.h + OUT dqs_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h + ) +ELSE() + RUN_PROGRAM( + yql/essentials/core/expr_nodes_gen/gen + yql_expr_nodes_gen.jnj + dqs_expr_nodes.json + dqs_expr_nodes.gen.h + dqs_expr_nodes.decl.inl.h + dqs_expr_nodes.defs.inl.h + NDq + IN yql_expr_nodes_gen.jnj + IN dqs_expr_nodes.json + OUT dqs_expr_nodes.gen.h + OUT dqs_expr_nodes.decl.inl.h + OUT dqs_expr_nodes.defs.inl.h + OUTPUT_INCLUDES + ${ARCADIA_ROOT}/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.h + ${ARCADIA_ROOT}/util/generic/hash_set.h + ) +ENDIF() + +END() diff --git a/yql/essentials/core/expr_nodes_gen/gen/__main__.py b/yql/essentials/core/expr_nodes_gen/gen/__main__.py index b381e64b5d..39e182e194 100755 --- a/yql/essentials/core/expr_nodes_gen/gen/__main__.py +++ b/yql/essentials/core/expr_nodes_gen/gen/__main__.py @@ -12,8 +12,13 @@ jsonFile = sys.argv[2] headerOutFile = sys.argv[3] declOutFile = sys.argv[4] defsOutFile = sys.argv[5] +nodeNspace = sys.argv[6] if len(sys.argv) >= 7 else "" +nodeNspacePrefix = nodeNspace + "::" if nodeNspace else "" + +os.environ['NODES_NAMESPACE'] = nodeNspace env = Environment(loader=FileSystemLoader(templateDir)) +env.globals['ENV'] = os.getenv template = env.get_template(templateFilename) json_data = open(jsonFile) @@ -24,8 +29,10 @@ json_data.close() for node in model["Nodes"]: aux = node["aux"] = {} + aux["qName"] = nodeNspacePrefix + node["Name"] aux["stubName"] = node["Name"] + "Stub" aux["stubMaybeName"] = node["Name"] + "MaybeStub" + aux["qStubMaybeName"] = nodeNspacePrefix + node["Name"] + "MaybeStub" aux["stubBuilderName"] = node["Name"] + "BuilderStub" aux["stubBuilderAliasName"] = node["Name"] + "Builder" @@ -109,6 +116,11 @@ for node in model["Nodes"]: aux["generateBuilderStub"] = node["Builder"]["Generate"] != "None" and node["Builder"]["Kind"] != "List" aux["generateBuilder"] = node["Builder"]["Generate"] == "Auto" + if isListBuilder(node): + if node["Builder"]["ListItemType"] in nodesMap: + aux["qListItemType"] = nodeNspacePrefix + node["Builder"]["ListItemType"] + else: + aux["qListItemType"] = node["Builder"]["ListItemType"] # Get all children allChildren = [] @@ -174,6 +186,7 @@ for node in model["Nodes"]: for child in node["Children"]: addUsages(child["Type"]) aux["usages"] = usages + aux["qUsages"] = [name if name not in nodesMap else nodeNspacePrefix + name for name in usages] aux["typenames"] = declarations usages = [] diff --git a/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj b/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj index 94225db9ea..7d074a070d 100644 --- a/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj +++ b/yql/essentials/core/expr_nodes_gen/yql_expr_nodes_gen.jnj @@ -1,5 +1,17 @@ // Auto-generated by {{ generator }}, do not modify. +{% set nNspace = ENV("NODES_NAMESPACE") %} + +{%- if nNspace != "" %} + +{% set nPrefix = nNspace + "::" %} +{% set nNspaceOpen = "namespace " + nNspace + " {" %} +{% set nNspaceClose = "} // namespace " + nNspace %} + +{% else %} +{% set nPrefix, nNspaceOpen, nNspaceClose = "", "", "" %} +{%- endif %} + {% if genType == "Header" %} #pragma once @@ -8,6 +20,7 @@ namespace NYql { namespace NNodes { +{{ nNspaceOpen }} namespace NGenerated { {% for node in nodes -%} @@ -243,10 +256,12 @@ public: {% endfor %} } // namespace NGenerated +{{ nNspaceClose }} } // namespace NNodes } // namespace NYql {% elif genType == "Declarations" -%} +{{ nNspaceOpen }} {% for node in nodes %} {%- if node.Definition == "Custom" %} class {{ node.Name }}; @@ -254,28 +269,32 @@ class {{ node.Name }}; using {{ node.Name }} = NGenerated::{{ node.aux.stubName }}<{{ node.aux.usages | join(", ") }}>; {% endif %} {%- endfor %} +{{ nNspaceClose }} {% elif genType == "Definitions" -%} +{{ nNspaceOpen }} {% for node in nodes %} static_assert(std::is_constructible<{{ node.Name }}, const TExprNode*>::value, "{{ node.Name }} isn't defined correctly."); static_assert(std::is_constructible<{{ node.Name }}, const TExprNode::TPtr&>::value, "{{ node.Name }} isn't defined correctly."); {% endfor %} +{{ nNspaceClose }} {% for node in nodes %} template<> -class TMaybeNode<{{ node.Name }}> : public NGenerated:: - {{ node.aux.stubMaybeName }}<TMaybeNode, {{ node.Name }}, {{ node.aux.usages | join(", ") }}> +class TMaybeNode<{{ node.aux.qName }}> : public {{ nPrefix }}NGenerated:: + {{ node.aux.stubMaybeName }}<TMaybeNode, {{ node.aux.qName }}, {{ node.aux.qUsages | join(", ") }}> { public: TMaybeNode(const TExprNode* node = nullptr) : {{ node.aux.stubMaybeName }}(node) {} TMaybeNode(const TExprNode::TPtr& node) : {{ node.aux.stubMaybeName }}(node) {} - TMaybeNode(const {{ node.Name }}& node) : {{ node.aux.stubMaybeName }}(node.template Maybe<{{ node.Name }}>()) {} + TMaybeNode(const {{ node.aux.qName }}& node) : {{ node.aux.stubMaybeName }}(node.template Maybe<{{ node.aux.qName }}>()) {} }; {% endfor %} +{{ nNspaceOpen }} namespace NGenerated { {% for node in nodes %} {%- if node.aux.generateBuilderStub %} @@ -293,34 +312,35 @@ using {{ node.aux.stubBuilderAliasName }} = {% endif %} {%- endfor %} } // namespace NGenerated +{{ nNspaceClose }} {% for node in nodes %} {%- if node.aux.generateBuilder %} {% if node.Builder.Kind == "Node" or node.Builder.Kind == "FreeArg" %} template<typename TParent> -class TNodeBuilder<TParent, {{ node.Name }}> : public NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent> +class TNodeBuilder<TParent, {{ node.aux.qName }}> : public {{ nPrefix }}NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent> { private: - typedef typename NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>::BuildFuncType BuildFuncType; - typedef typename NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>::GetArgFuncType GetArgFuncType; + typedef typename {{ nPrefix }}NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>::BuildFuncType BuildFuncType; + typedef typename {{ nPrefix }}NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>::GetArgFuncType GetArgFuncType; public: TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) - : NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>(ctx, pos, buildFunc, getArgFunc) {} + : {{ nPrefix }}NGenerated::{{ node.aux.stubBuilderAliasName }}<TParent>(ctx, pos, buildFunc, getArgFunc) {} {% elif node.Builder.Kind == "List" %} template<typename TParent> -class TNodeBuilder<TParent, {{ node.Name }}> : public TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}> +class TNodeBuilder<TParent, {{ node.aux.qName }}> : public TListBuilderBase<TParent, {{ node.aux.qName }}, {{ node.aux.qListItemType }}> { private: - typedef typename TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}>::BuildFuncType BuildFuncType; - typedef typename TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}>::GetArgFuncType GetArgFuncType; + typedef typename TListBuilderBase<TParent, {{ node.aux.qName }}, {{ node.aux.qListItemType }}>::BuildFuncType BuildFuncType; + typedef typename TListBuilderBase<TParent, {{ node.aux.qName }}, {{ node.aux.qListItemType }}>::GetArgFuncType GetArgFuncType; public: TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc) - : TListBuilderBase<TParent, {{ node.Name }}, {{ node.Builder.ListItemType }}>(ctx, pos, buildFunc, getArgFunc) {} + : TListBuilderBase<TParent, {{ node.aux.qName }}, {{ node.aux.qListItemType }}>(ctx, pos, buildFunc, getArgFunc) {} {% else %} class UnknownBuilder { static_assert(false, "Unknown builder kind: {{ node.Builder.Kind }}."); {% endif %} - {{ node.Name }} DoBuild() { + {{ node.aux.qName }} DoBuild() { {%- if node.Match and (node.Match.Type == "Callable" or node.Match.Type == "CallableBase") %} TExprNode::TListType argsList; {%- if node.Builder.Kind == "List" %} @@ -347,7 +367,7 @@ class UnknownBuilder { {%- else %} auto node = this->Ctx.NewCallable(this->Pos, "{{ node.Match.Name }}", std::move(argsList)); {%- endif %} - return {{ node.Name }}(node); + return {{ node.aux.qName }}(node); {%- elif node.Match and node.Match.Type == "Tuple"%} TExprNode::TListType tupleItems; {%- for child in node.aux.allChildren %} @@ -357,14 +377,14 @@ class UnknownBuilder { } {%- endfor %} auto node = this->Ctx.NewList(this->Pos, std::move(tupleItems)); - return {{ node.Name }}(node); + return {{ node.aux.qName }}(node); {%- elif node.ListBase and node.Builder.Kind == "List"%} TExprNode::TListType nodeChildren; for (auto child : this->Items) { nodeChildren.push_back(child.Ptr()); } auto node = this->Ctx.NewList(this->Pos, std::move(nodeChildren)); - return {{ node.Name }}(node); + return {{ node.aux.qName }}(node); {%- else %} static_assert(false, "Don't know how to build {{ node.Name }}."); {% endif %} diff --git a/yql/essentials/core/issue/protos/issue_id.proto b/yql/essentials/core/issue/protos/issue_id.proto index 975efe5e3b..688a471afb 100644 --- a/yql/essentials/core/issue/protos/issue_id.proto +++ b/yql/essentials/core/issue/protos/issue_id.proto @@ -195,6 +195,10 @@ message TIssuesIds { PG_NO_LOCKING_SUPPORT = 7000; PG_COMPAT = 7001; +// CBO + CBO_ENUM_LIMIT_REACHED = 8000; + CBO_MISSING_TABLE_STATS = 8001; + // range [200000, 399999) reserved for KiKiMR issue codes, do not use! } diff --git a/yql/essentials/core/issue/yql_issue.txt b/yql/essentials/core/issue/yql_issue.txt index 82a0a1eecd..148363532c 100644 --- a/yql/essentials/core/issue/yql_issue.txt +++ b/yql/essentials/core/issue/yql_issue.txt @@ -683,3 +683,11 @@ ids { code: CORE_TOP_UNSUPPORTED_BLOCK_CALLABLES severity: S_INFO } +ids { + code: CBO_ENUM_LIMIT_REACHED + severity: S_WARNING +} +ids { + code: CBO_MISSING_TABLE_STATS + severity: S_WARNING +} diff --git a/yql/essentials/core/ya.make b/yql/essentials/core/ya.make index 2e253709c2..100103b92f 100644 --- a/yql/essentials/core/ya.make +++ b/yql/essentials/core/ya.make @@ -98,7 +98,9 @@ END() RECURSE( cbo credentials + dq_expr_nodes dq_integration + dqs_expr_nodes file_storage issue minsketch diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp index 173f73b7be..27ac0acda8 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.cpp +++ b/yql/essentials/public/purecalc/common/worker_factory.cpp @@ -130,7 +130,7 @@ template <typename TBase> TExprNode::TPtr TWorkerFactory<TBase>::Compile( TStringBuf query, ETranslationMode mode, - IModuleResolver::TPtr moduleResolver, + IModuleResolver::TPtr factoryModuleResolver, ui16 syntaxVersion, const THashMap<TString, TString>& modules, const TInputSpecBase& inputSpec, @@ -145,6 +145,7 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( TTypeAnnotationContextPtr typeContext; + IModuleResolver::TPtr moduleResolver = factoryModuleResolver ? factoryModuleResolver->CreateMutableChild() : nullptr; typeContext = MakeIntrusive<TTypeAnnotationContext>(); typeContext->RandomProvider = CreateDefaultRandomProvider(); typeContext->TimeProvider = DeterministicTimeProviderSeed_ ? diff --git a/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.cfg b/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.cfg +++ b/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.yql b/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.yql index 887bad3350..d6e6d8fe10 100644 --- a/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.yql +++ b/yql/essentials/tests/s-expressions/suites/Filter/LMapCombineWithFilter.yql @@ -23,7 +23,7 @@ def MyFunc(list): return list @@)) -(let udf (ScriptUdf 'Python 'MyFunc udfType udfScript)) +(let udf (ScriptUdf 'Python3 'MyFunc udfType udfScript)) # filter keys less than '100' (let table (Filter table (lambda '(item) diff --git a/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.cfg b/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.cfg +++ b/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.yql b/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.yql index 68bb0ad6da..f5981f92ac 100644 --- a/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.yql +++ b/yql/essentials/tests/s-expressions/suites/Filter/LMapWithFilter.yql @@ -23,7 +23,7 @@ def MyFunc(list): return list @@)) -(let udf (ScriptUdf 'Python 'MyFunc udfType udfScript '('('cpu '"5.0") '('extraMem '"12345")))) +(let udf (ScriptUdf 'Python3 'MyFunc udfType udfScript '('('cpu '"5.0") '('extraMem '"12345")))) # filter keys less than '100' (let table (Filter table (lambda '(item) diff --git a/yql/essentials/tests/s-expressions/suites/ManyYamrOperations/Bug1465.cfg b/yql/essentials/tests/s-expressions/suites/ManyYamrOperations/Bug1465.cfg index ac8f608e62..f4ca3c4cf6 100644 --- a/yql/essentials/tests/s-expressions/suites/ManyYamrOperations/Bug1465.cfg +++ b/yql/essentials/tests/s-expressions/suites/ManyYamrOperations/Bug1465.cfg @@ -3,4 +3,4 @@ out out1 out1.txt out out2 out2.txt res result.txt udf url_udf -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg index 0bbf3ad9f0..62b2a5fe18 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg @@ -1,4 +1,5 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf +providers dummy diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql index cc5b60392b..3df99f0f47 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql @@ -25,7 +25,7 @@ (return select) ))) (let core (LMap inputRowsList (lambda '(x) (block '( - (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ @@ -33,7 +33,7 @@ def f(input,x): yield d @@))) '0)) (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) - (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (return (Apply (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ @@ -46,7 +46,7 @@ def f(input,x): (return select) ))) (let core (LMap inputRowsList (lambda '(x) (block '( - (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ @@ -54,7 +54,7 @@ def f(input,x): yield d @@))) '0)) (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) - (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (return (Apply (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg index 0bbf3ad9f0..62b2a5fe18 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg @@ -1,4 +1,5 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf +providers dummy diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql index 1f806dc3a3..3e8637fc54 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql @@ -23,7 +23,7 @@ (return select) ))) (let core (LMap inputRowsList (lambda '(x) (block '( - (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ @@ -31,7 +31,7 @@ def f(input,x): yield d @@))) '0)) (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) - (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (return (Apply (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg index 0bbf3ad9f0..62b2a5fe18 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg @@ -1,4 +1,5 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf +providers dummy diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql index ea138e481a..5656d60fc3 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql @@ -25,24 +25,24 @@ (return select) ))) (let core (LMap inputRowsList (lambda '(x) (block '( - (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ d["pass"] = x yield d - + def f2(a,b,c): return {"a":a,"b":b,"c":c} @@))) '0)) (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) - (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ + (return (Apply (ScriptUdf 'Python3 '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ d["pass"] = x yield d - + def f2(a,b,c): return {"a":a,"b":b,"c":c} @@)) inputRowsList (Int32 '"1"))) @@ -52,13 +52,13 @@ def f2(a,b,c): (return select) ))) (let core (FlatMap core (lambda '(row) (block '( - (let res (Apply (ScriptUdf 'Python '"f2" (CallableType '() '((StructType '('"a" (DataType 'Uint32)) '('"b" (DataType 'Uint64)) '('"c" (DataType 'Int32)))) '((DataType 'Uint32) 'skey) '((DataType 'Uint64) 'cnt) '((DataType 'Int32) 'pass)) (String '@@ + (let res (Apply (ScriptUdf 'Python3 '"f2" (CallableType '() '((StructType '('"a" (DataType 'Uint32)) '('"b" (DataType 'Uint64)) '('"c" (DataType 'Int32)))) '((DataType 'Uint32) 'skey) '((DataType 'Uint64) 'cnt) '((DataType 'Int32) 'pass)) (String '@@ def f(input,x): for i in list(input): d = i.__dict__ d["pass"] = x yield d - + def f2(a,b,c): return {"a":a,"b":b,"c":c} @@)) (Member row '"skey") (Member row '"cnt") (Member row '"pass"))) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg index bc0dfe8e18..f3abc496dd 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg @@ -1,4 +1,4 @@ in Input SomeKeys.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql index a245de4f18..7b776d078b 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql @@ -8,7 +8,7 @@ def MyTuple(one, two): return (one, two) @@)) - (let udf (ScriptUdf 'Python 'MyTuple udfType udfScript)) + (let udf (ScriptUdf 'Python3 'MyTuple udfType udfScript)) (let mr_source (DataSource 'yt 'plato)) (let x (Read! world mr_source (Key '('table (String 'Input))) (Void) '())) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg index bc0dfe8e18..f3abc496dd 100644 --- a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg @@ -1,4 +1,4 @@ in Input SomeKeys.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.cfg b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.yql b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.yql index e179419a5c..6e29b890b9 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapMany.yql @@ -42,7 +42,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf0) (Int32 '"1") (Int32 '"2") (Int32 '"3"))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf0) (Int32 '"1") (Int32 '"2") (Int32 '"3"))))) (let res (AsList res)) (return res) ))))) @@ -62,7 +62,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf1) ("Just" (Int32 '"1")) (Int32 '"2") (Int32 '"3"))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf1) ("Just" (Int32 '"1")) (Int32 '"2") (Int32 '"3"))))) (let res (AsList res)) (return res) ))))) @@ -82,7 +82,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf2) ("Just" (Int32 '"1")) ("Just" (Int32 '"2")) (Int32 '"3"))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf2) ("Just" (Int32 '"1")) ("Just" (Int32 '"2")) (Int32 '"3"))))) (let res (AsList res)) (return res) ))))) @@ -102,7 +102,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf3) ("Just" (Int32 '"1")) ("Just" (Int32 '"2")) ("Just" (Int32 '"3")))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf3) ("Just" (Int32 '"1")) ("Just" (Int32 '"2")) ("Just" (Int32 '"3")))))) (let res (AsList res)) (return res) ))))) @@ -122,7 +122,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf4) (Int32 '"4") (Int32 '"5") (Int32 '"6"))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf4) (Int32 '"4") (Int32 '"5") (Int32 '"6"))))) (let res (AsList res)) (return res) ))))) @@ -142,7 +142,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf5) ("Just" (Int32 '"4")) (Int32 '"5") (Int32 '"6"))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf5) ("Just" (Int32 '"4")) (Int32 '"5") (Int32 '"6"))))) (let res (AsList res)) (return res) ))))) @@ -162,7 +162,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf6) ("Just" (Int32 '"4")) ("Just" (Int32 '"5")) (Int32 '"6"))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf6) ("Just" (Int32 '"4")) ("Just" (Int32 '"5")) (Int32 '"6"))))) (let res (AsList res)) (return res) ))))) @@ -182,7 +182,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf7) ("Just" (Int32 '"4")) ("Just" (Int32 '"5")) ("Just" (Int32 '"6")))))) + (let res (AsStruct '('"column0" ("Apply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1) '((DataType 'Int32) '"" '1)) scriptudf7) ("Just" (Int32 '"4")) ("Just" (Int32 '"5")) ("Just" (Int32 '"6")))))) (let res (AsList res)) (return res) ))))) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.cfg b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.yql b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.yql index 2225ad9b7e..64c30fb142 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/AutoMapManyNamed.yql @@ -58,7 +58,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf0) '() (AsStruct '('x (Int32 '"1")) '('y (Int32 '"2")) '('z (Int32 '"3"))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf0) '() (AsStruct '('x (Int32 '"1")) '('y (Int32 '"2")) '('z (Int32 '"3"))))))) (let res (AsList res)) (return res) ))))) @@ -78,7 +78,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf1) '() (AsStruct '('x ("Just" (Int32 '"1"))) '('y (Int32 '"2")) '('z (Int32 '"3"))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf1) '() (AsStruct '('x ("Just" (Int32 '"1"))) '('y (Int32 '"2")) '('z (Int32 '"3"))))))) (let res (AsList res)) (return res) ))))) @@ -98,7 +98,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf2) '() (AsStruct '('x ("Just" (Int32 '"1"))) '('y ("Just" (Int32 '"2"))) '('z (Int32 '"3"))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf2) '() (AsStruct '('x ("Just" (Int32 '"1"))) '('y ("Just" (Int32 '"2"))) '('z (Int32 '"3"))))))) (let res (AsList res)) (return res) ))))) @@ -118,7 +118,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf3) '() (AsStruct '('x ("Just" (Int32 '"1"))) '('y ("Just" (Int32 '"2"))) '('z ("Just" (Int32 '"3")))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf3) '() (AsStruct '('x ("Just" (Int32 '"1"))) '('y ("Just" (Int32 '"2"))) '('z ("Just" (Int32 '"3")))))))) (let res (AsList res)) (return res) ))))) @@ -138,7 +138,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf4) '(("Just" (Int32 '"1"))) (AsStruct '('y ("Just" (Int32 '"2"))) '('z ("Just" (Int32 '"3")))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf4) '(("Just" (Int32 '"1"))) (AsStruct '('y ("Just" (Int32 '"2"))) '('z ("Just" (Int32 '"3")))))))) (let res (AsList res)) (return res) ))))) @@ -158,7 +158,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf5) '(("Just" (Int32 '"1")) ("Just" (Int32 '"2"))) (AsStruct '('z ("Just" (Int32 '"3")))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((DataType 'Int32)) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf5) '(("Just" (Int32 '"1")) ("Just" (Int32 '"2"))) (AsStruct '('z ("Just" (Int32 '"3")))))))) (let res (AsList res)) (return res) ))))) @@ -178,7 +178,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf6) '() (AsStruct '('x (Int32 '"4")) '('y (Int32 '"5")) '('z (Int32 '"6"))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf6) '() (AsStruct '('x (Int32 '"4")) '('y (Int32 '"5")) '('z (Int32 '"6"))))))) (let res (AsList res)) (return res) ))))) @@ -198,7 +198,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf7) '() (AsStruct '('x ("Just" (Int32 '"4"))) '('y (Int32 '"5")) '('z (Int32 '"6"))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf7) '() (AsStruct '('x ("Just" (Int32 '"4"))) '('y (Int32 '"5")) '('z (Int32 '"6"))))))) (let res (AsList res)) (return res) ))))) @@ -218,7 +218,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf8) '() (AsStruct '('x ("Just" (Int32 '"4"))) '('y ("Just" (Int32 '"5"))) '('z (Int32 '"6"))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf8) '() (AsStruct '('x ("Just" (Int32 '"4"))) '('y ("Just" (Int32 '"5"))) '('z (Int32 '"6"))))))) (let res (AsList res)) (return res) ))))) @@ -238,7 +238,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf9) '() (AsStruct '('x ("Just" (Int32 '"4"))) '('y ("Just" (Int32 '"5"))) '('z ("Just" (Int32 '"6")))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf9) '() (AsStruct '('x ("Just" (Int32 '"4"))) '('y ("Just" (Int32 '"5"))) '('z ("Just" (Int32 '"6")))))))) (let res (AsList res)) (return res) ))))) @@ -258,7 +258,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf10) '(("Just" (Int32 '"4"))) (AsStruct '('y ("Just" (Int32 '"5"))) '('z ("Just" (Int32 '"6")))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf10) '(("Just" (Int32 '"4"))) (AsStruct '('y ("Just" (Int32 '"5"))) '('z ("Just" (Int32 '"6")))))))) (let res (AsList res)) (return res) ))))) @@ -278,7 +278,7 @@ def f(x,y,z): (let select (block '( (let core (AsList (Uint32 '0))) (let core (FlatMap core (lambda '(row) (block '( - (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf11) '(("Just" (Int32 '"4")) ("Just" (Int32 '"5"))) (AsStruct '('z ("Just" (Int32 '"6")))))))) + (let res (AsStruct '('"column0" ("NamedApply" (ScriptUdf 'Python3 '"f" (CallableType '() '((OptionalType (DataType 'Int32))) '((DataType 'Int32) 'x '1) '((DataType 'Int32) 'y '1) '((DataType 'Int32) 'z '1)) scriptudf11) '(("Just" (Int32 '"4")) ("Just" (Int32 '"5"))) (AsStruct '('z ("Just" (Int32 '"6")))))))) (let res (AsList res)) (return res) ))))) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.yql index 62dc07db05..9029469690 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonAvg.yql @@ -32,9 +32,9 @@ def avg_result(avg): @@)) -(let avgCreate (ScriptUdf 'Python 'avg_create (CallableType '() '(rt)) udfScript)) -(let avgAdd (ScriptUdf 'Python 'avg_add (CallableType '() '(rt) '(rt) '(ui32)) udfScript)) -(let avgResult (ScriptUdf 'Python 'avg_result (CallableType '() '(dbl) '(rt)) udfScript)) +(let avgCreate (ScriptUdf 'Python3 'avg_create (CallableType '() '(rt)) udfScript)) +(let avgAdd (ScriptUdf 'Python3 'avg_add (CallableType '() '(rt) '(rt) '(ui32)) udfScript)) +(let avgResult (ScriptUdf 'Python3 'avg_result (CallableType '() '(dbl) '(rt)) udfScript)) # data (let x (Uint32 '10)) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.cfg index 0bffbd4f08..d24419e54e 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.cfg @@ -1,5 +1,5 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf udf simple_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.yql index 36a70677e4..29a7d26a0a 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsArg.yql @@ -14,9 +14,9 @@ (let udfType (CallableType '() '(str) '(funcType) '(ui64))) (let udfScript (String '@@ def new_string(func, x): - return func('x', x) + ':' + func('y', x) + return func(b'x', x) + b':' + func(b'y', x) @@)) -(let udf (ScriptUdf 'Python 'new_string udfType udfScript)) +(let udf (ScriptUdf 'Python3 'new_string udfType udfScript)) # call udf (let x (Uint64 '3)) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.yql index 7a7a6a995a..887526632b 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonCallableAsResult.yql @@ -19,7 +19,7 @@ def create_counter(start): return counter @@)) -(let udf (ScriptUdf 'Python 'create_counter udfType udfScript)) +(let udf (ScriptUdf 'Python3 'create_counter udfType udfScript)) # call udf (let counter (Apply udf (Uint64 '1))) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.yql index d225a14c1b..b6e6a00171 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonGenerator.yql @@ -8,11 +8,11 @@ (let udfScript (String '@@ def g0(): def gen(): - for x in xrange(1, 4): - yield long(x + 42) + for x in range(1, 4): + yield int(x + 42) return gen @@)) -(let udf (ScriptUdf 'Python 'g0 udfType udfScript)) +(let udf (ScriptUdf 'Python3 'g0 udfType udfScript)) # call udf (let result (Apply udf)) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.yql index 86a1c9a0bb..164f870ec8 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorExprWithClosure.yql @@ -8,10 +8,10 @@ (let udfScript (String '@@ def g(input): def gen(): - return (long(x + 42) for x in input) + return (int(x + 42) for x in input) return gen @@)) -(let udf (ScriptUdf 'Python 'g udfType udfScript)) +(let udf (ScriptUdf 'Python3 'g udfType udfScript)) # call udf (let result (Apply udf (AsList (Int64 '1) (Int64 '2) (Int64 '3)))) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.yql index 87799cbf69..2c9a022cae 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonGeneratorWithClosure.yql @@ -9,10 +9,10 @@ def h(input): def gen(): for x in input: - yield long(x + 42) + yield int(x + 42) return gen @@)) -(let udf (ScriptUdf 'Python 'h udfType udfScript)) +(let udf (ScriptUdf 'Python3 'h udfType udfScript)) # call udf (let result (Apply udf (AsList (Int64 '1) (Int64 '2) (Int64 '3)))) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.yql index f45e2d9157..1a7f7f9dbd 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonPartialCall.yql @@ -6,7 +6,7 @@ (let ui32 (DataType 'Uint32)) (let udfType (CallableType '('1) '(ui32) '(ui32) '((OptionalType ui32)))) (let udfScript (String '"def Add(x, y): return x + (y if y is not None else 1)")) -(let udf (ScriptUdf 'Python 'Add udfType udfScript)) +(let udf (ScriptUdf 'Python3 'Add udfType udfScript)) # call udf (let x (Uint32 '20)) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.yql index 38a484a4c3..89e0bd435d 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonStruct.yql @@ -18,7 +18,7 @@ def NewPerson(name, age): return Person(name, age) @@)) -(let udf (ScriptUdf 'Python 'NewPerson udfType udfScript)) +(let udf (ScriptUdf 'Python3 'NewPerson udfType udfScript)) # call udf (let result (Apply udf (String 'Jamel) (Uint32 '99))) diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.cfg b/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.cfg index 0bbf3ad9f0..aa782f323c 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.cfg +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.cfg @@ -1,4 +1,4 @@ in Input input.txt out Output output.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.yql b/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.yql index b240dc0f58..ef5f4e1549 100644 --- a/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.yql +++ b/yql/essentials/tests/s-expressions/suites/Udf/PythonSum.yql @@ -6,7 +6,7 @@ (let ui32 (DataType 'Uint32)) (let udfType (CallableType '() '(ui32) '(ui32) '(ui32))) (let udfScript (String '"def Sum(x, y): return x + y")) -(let udf (ScriptUdf 'Python 'Sum udfType udfScript)) +(let udf (ScriptUdf 'Python3 'Sum udfType udfScript)) # call udf (let x (Uint32 '10)) diff --git a/yql/essentials/tests/s-expressions/suites/View/ViewWithUdfProcess.cfg b/yql/essentials/tests/s-expressions/suites/View/ViewWithUdfProcess.cfg index b2849b4f98..4cc66ff555 100644 --- a/yql/essentials/tests/s-expressions/suites/View/ViewWithUdfProcess.cfg +++ b/yql/essentials/tests/s-expressions/suites/View/ViewWithUdfProcess.cfg @@ -1,3 +1,3 @@ in Input input6.txt res result.txt -udf python2_udf +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/View/input6.txt.attr b/yql/essentials/tests/s-expressions/suites/View/input6.txt.attr index 501fdca6db..7f425e31ea 100644 --- a/yql/essentials/tests/s-expressions/suites/View/input6.txt.attr +++ b/yql/essentials/tests/s-expressions/suites/View/input6.txt.attr @@ -1,5 +1,5 @@ { - "_yql_view_my"="$udfScript=\"def f(row):\n\treturn {'myvalue':b'm' + row.value}\n\";$udf=Python::f(Callable<(Struct<key:String,subkey:String,value:String>)->Struct<myvalue:String>>, $udfScript);PROCESS self USING $udf(TableRow());"; + "_yql_view_my"="$udfScript=\"def f(row):\n\treturn {'myvalue':b'm' + row.value}\n\";$udf=Python3::f(Callable<(Struct<key:String,subkey:String,value:String>)->Struct<myvalue:String>>, $udfScript);PROCESS self USING $udf(TableRow());"; "_yql_row_spec" = { "Type" = [ "StructType"; |