diff options
| author | Maxim Yurchuk <maxim-yurchuk@ydb.tech> | 2024-11-20 17:37:57 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-11-20 17:37:57 +0000 |
| commit | f76323e9b295c15751e51e3443aa47a36bee8023 (patch) | |
| tree | 4113c8cad473a33e0f746966e0cf087252fa1d7a /yql/essentials/tests/s-expressions/suites/Optimizers | |
| parent | 753ecb8d410a4cb459c26f3a0082fb2d1724fe63 (diff) | |
| parent | a7b9a6afea2a9d7a7bfac4c5eb4c1a8e60adb9e6 (diff) | |
| download | ydb-f76323e9b295c15751e51e3443aa47a36bee8023.tar.gz | |
Merge pull request #11788 from ydb-platform/mergelibs-241120-1113
Library import 241120-1113
Diffstat (limited to 'yql/essentials/tests/s-expressions/suites/Optimizers')
198 files changed, 4368 insertions, 0 deletions
diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/AggregateAfterUnion.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/AggregateAfterUnion.cfg new file mode 100644 index 0000000000..27b6ab6bec --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/AggregateAfterUnion.cfg @@ -0,0 +1,3 @@ +in Input1 inputUnion1.txt +in Input2 inputUnion2.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/AggregateAfterUnion.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/AggregateAfterUnion.yql new file mode 100644 index 0000000000..f3d4e5db52 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/AggregateAfterUnion.yql @@ -0,0 +1,26 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input1"))) (Void) '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input2"))) (Void) '())) + (let world (Left! x)) + (let table2 (Right! x)) + (let u (UnionAll table1 table2)) + (let output (Aggregate u '() '('('Min0 (Apply (bind aggregate_module '"min_traits_factory") (TypeOf u) (lambda '(row) (Member row '"age"))))))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidDuplicationOfJoinsBeforeAggr.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidDuplicationOfJoinsBeforeAggr.cfg new file mode 100644 index 0000000000..6f99339756 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidDuplicationOfJoinsBeforeAggr.cfg @@ -0,0 +1,2 @@ +in Input input.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidDuplicationOfJoinsBeforeAggr.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidDuplicationOfJoinsBeforeAggr.yql new file mode 100644 index 0000000000..0886d5c7a9 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidDuplicationOfJoinsBeforeAggr.yql @@ -0,0 +1,111 @@ +#USE plato; +# +#$sample = (SELECT a.key as key, a.value as value FROM Input as a Join Input as b ON a.key = b.key); +# +#select key, count(*) from $sample group by key; +#select value, count(*) from $sample group by value; + +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) '('"key" '"value") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (block '( + (let select (block '( + (let core (EquiJoin '(table0 '"a") '(table0 '"b") '('Inner '"a" '"b" '('"a" '"key") '('"b" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Member row '"a.key"))) + (let res (AddMember res '"value" (Member row '"a.value"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (block '( + (let Count2_create (lambda '(row) (Uint64 '1))) + (let Count2_update (lambda '(row state) (OptionalReduce state (Uint64 '1) (lambda '(a b) (+ a b))))) + (let Count2_save (lambda '(state) state)) + (let Count2_load (lambda '(item) item)) + (let Count2_merge (lambda '(a b) (OptionalReduce a b (lambda '(a b) (+ a b))))) + (let Count2_finish (lambda '(state) state)) + (return (Aggregate core '('"key") '('('Count2 (AggregationTraits (ListItemType (TypeOf core)) Count2_create Count2_update Count2_save Count2_load Count2_merge Count2_finish (Uint64 '0)))))) + ))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Member row '"key"))) + (let res (AddMember res '"column1" (Member row 'Count2))) + (let res (AsList res)) + (return res) + ))))) + (let core (Sort core (Bool 'true) (lambda '(row) (Member row '"key")))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"key" '"column1"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) '('"key" '"value") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (block '( + (let select (block '( + (let core (EquiJoin '(table0 '"a") '(table0 '"b") '('Inner '"a" '"b" '('"a" '"key") '('"b" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Member row '"a.key"))) + (let res (AddMember res '"value" (Member row '"a.value"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (block '( + (let Count4_create (lambda '(row) (Uint64 '1))) + (let Count4_update (lambda '(row state) (OptionalReduce state (Uint64 '1) (lambda '(a b) (+ a b))))) + (let Count4_save (lambda '(state) state)) + (let Count4_load (lambda '(item) item)) + (let Count4_merge (lambda '(a b) (OptionalReduce a b (lambda '(a b) (+ a b))))) + (let Count4_finish (lambda '(state) state)) + (return (Aggregate core '('"value") '('('Count4 (AggregationTraits (ListItemType (TypeOf core)) Count4_create Count4_update Count4_save Count4_load Count4_merge Count4_finish (Uint64 '0)))))) + ))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"value" (Member row '"value"))) + (let res (AddMember res '"column1" (Member row 'Count4))) + (let res (AsList res)) + (return res) + ))))) + (let core (Sort core (Bool 'true) (lambda '(row) (Member row '"value")))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"value" '"column1"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidFuseOfSameJoins.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidFuseOfSameJoins.cfg new file mode 100644 index 0000000000..6f99339756 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidFuseOfSameJoins.cfg @@ -0,0 +1,2 @@ +in Input input.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidFuseOfSameJoins.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidFuseOfSameJoins.yql new file mode 100644 index 0000000000..9918cc7770 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/AvoidFuseOfSameJoins.yql @@ -0,0 +1,87 @@ +#USE plato; +# +#$sample = (SELECT a.key as key, a.value as value FROM Input as a Join Input as b ON a.key = b.key); +# +#select * from $sample as x left join Input as y using (key); +#select * from $sample as x right join Input as y using (value); + +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) (Void) '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (EquiJoin '((block '( + (let select (block '( + (let core (EquiJoin '(table0 '"a") '(table0 '"b") '('Inner '"a" '"b" '('"a" '"key") '('"b" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Member row '"a.key"))) + (let res (AddMember res '"value" (Member row '"a.value"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) '"x") '(table0 '"y") '('Left '"x" '"y" '('"x" '"key") '('"y" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsList row)) + (return res) + ))))) + (return core) + ))) + (let select (Sort select '((Bool 'true) (Bool 'true) (Bool 'true)) (lambda '(row) '((Member row '"x.key") (Member row '"x.value") (Member row '"y.key"))))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) (Void) '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (EquiJoin '((block '( + (let select (block '( + (let core (EquiJoin '(table0 '"a") '(table0 '"b") '('Inner '"a" '"b" '('"a" '"key") '('"b" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Member row '"a.key"))) + (let res (AddMember res '"value" (Member row '"a.value"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) '"x") '(table0 '"y") '('Right '"x" '"y" '('"x" '"value") '('"y" '"value") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsList row)) + (return res) + ))))) + (return core) + ))) + (let select (Sort select '((Bool 'true) (Bool 'true) (Bool 'true)) (lambda '(row) '((Member row '"x.key") (Member row '"x.value") (Member row '"y.key"))))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/BinaryDataLiterals.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/BinaryDataLiterals.yql new file mode 100644 index 0000000000..fe707cfe07 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/BinaryDataLiterals.yql @@ -0,0 +1,25 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) + +(let data (Bool 'x"00")) +(let world (Write! world res_sink (Key) data '())) + +(let data (Int16 'x"FEFF")) +(let world (Write! world res_sink (Key) data '())) + +(let data (Double 'x"722078DC7BF042D3")) +(let world (Write! world res_sink (Key) data '())) + +(let data (Date 'x"0100")) +(let world (Write! world res_sink (Key) data '())) + +(let data (TzTimestamp 'x"0000F9BCF768AC000001")) +(let world (Write! world res_sink (Key) data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ChopperUseOnlyKeyInHandler.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/ChopperUseOnlyKeyInHandler.cfg new file mode 100644 index 0000000000..de97bde764 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ChopperUseOnlyKeyInHandler.cfg @@ -0,0 +1,4 @@ +in Input input4.txt +out Output output.txt +res result.txt + diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ChopperUseOnlyKeyInHandler.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ChopperUseOnlyKeyInHandler.yql new file mode 100644 index 0000000000..7c3615fa7f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ChopperUseOnlyKeyInHandler.yql @@ -0,0 +1,15 @@ +( +#comment +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table1 (Right! x)) +(let keySelector (lambda '(x) (Member x 'key))) +(let keySwitch (lambda '(key item) (IsKeySwitch key item (lambda '(k) k) keySelector))) +(let listHandler (lambda '(key stream) (ToStream (Just (AsStruct '('KEY key)))))) +(let reducedTable (Chopper (ToStream table1) keySelector keySwitch listHandler)) +(let mr_sink (DataSink 'yt (quote plato))) +(let world (Write! world mr_sink (Key '('table (String 'Output))) reducedTable '('('mode 'append)))) +(let world (Commit! world mr_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CloneStructure.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CloneStructure.yql new file mode 100644 index 0000000000..e2e95e299b --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CloneStructure.yql @@ -0,0 +1,23 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table2 (Map table1 (lambda '(x) (block '( + (let s (Struct)) + (let s (AddMember s 'value (Member x 'value))) + (let s (AddMember s 'subkey (Member x 'subkey))) + (let s (AddMember s 'key (Member x 'key))) + (return s) +))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceOverJust.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceOverJust.yql new file mode 100644 index 0000000000..5d034dfec2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceOverJust.yql @@ -0,0 +1,12 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (Coalesce (Just (Int32 '1)) (Int32 '2))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (Coalesce (Just (Int32 '1)) (Just (Int32 '2)))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceOverNothing.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceOverNothing.yql new file mode 100644 index 0000000000..efbc67ef1b --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceOverNothing.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (Coalesce (Nothing (OptionalType (DataType 'Int32))) (Int32 '2))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (Coalesce (Nothing (OptionalType (DataType 'Int32))) (Just (Int32 '2)))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (Coalesce (Nothing (OptionalType (DataType 'Int32))) (Just (Int32 '2)) (Int32 '3))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverAnd.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverAnd.yql new file mode 100644 index 0000000000..1d290a7ef7 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverAnd.yql @@ -0,0 +1,12 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let optBool1 (== (Int32 '0) (/ (Int32 '1) (Int32 '2)))) +(let optBool2 (== (Int32 '0) (/ (Int32 '4) (Int32 '5)))) +(let data (Coalesce (And optBool1 optBool2) (Bool 'false))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverNot.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverNot.yql new file mode 100644 index 0000000000..1d48368834 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverNot.yql @@ -0,0 +1,11 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let optBool (SafeCast (/ (Int32 '1) (Int32 '0)) (OptionalType (DataType 'Bool)))) +(let data (Coalesce (Not optBool) (Bool 'false))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverOr.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverOr.yql new file mode 100644 index 0000000000..9e382837fc --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CoalesceWithBoolOverOr.yql @@ -0,0 +1,12 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let optBool1 (== (Int32 '0) (/ (Int32 '3) (Int32 '2)))) +(let optBool2 (== (Int32 '0) (/ (Int32 '4) (Int32 '5)))) +(let data (Coalesce (Or optBool1 optBool2) (Bool 'false))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Condense1OneItemStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Condense1OneItemStruct.yql new file mode 100644 index 0000000000..ef949a7afb --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Condense1OneItemStruct.yql @@ -0,0 +1,25 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Condense1 list + (lambda '(item) + (AddMember + (Struct) + 'z + item)) + (lambda '(item state) (Bool 'false)) + (lambda '(item state) + (AddMember + (Struct) + 'z + (Concat (Member state 'z) item))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Condense1OneItemTuple.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Condense1OneItemTuple.yql new file mode 100644 index 0000000000..1044367186 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Condense1OneItemTuple.yql @@ -0,0 +1,20 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Condense1 list + (lambda '(item) + '(item)) + (lambda '(item state) (Bool 'false)) + (lambda '(item state) (block '( + (let x (Concat (Nth state '0) item)) + (return '(x))))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CondenseOneItemStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CondenseOneItemStruct.yql new file mode 100644 index 0000000000..1d24deecd5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CondenseOneItemStruct.yql @@ -0,0 +1,20 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Condense list (AddMember (Struct) 'z (String '"")) + (lambda '(item state) (Bool 'false)) + (lambda '(item state) + (AddMember + (Struct) + 'z + (Concat (Member state 'z) item))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CondenseOneItemTuple.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CondenseOneItemTuple.yql new file mode 100644 index 0000000000..d638ca1fef --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CondenseOneItemTuple.yql @@ -0,0 +1,19 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let s (String '"")) +(let data (Condense list '(s) + (lambda '(item state) (Bool 'false)) + (lambda '(item state) (block '( + (let state (Concat (Nth state '0) item)) + (return '(state))))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenCondense.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenCondense.yql new file mode 100644 index 0000000000..ca92c5573e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenCondense.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let list1 (AsList (String 'a) (String 'b) (String 'c))) +(let map (Map list1 (lambda '(x) (Uint32 '2)))) +(let data (Condense map (Uint32 '1) (lambda '(item state) (Bool 'false)) (lambda '(item state) (* item state)))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenCondense1.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenCondense1.yql new file mode 100644 index 0000000000..5d213f0f5a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenCondense1.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let list1 (AsList (String 'a) (String 'b) (String 'c))) +(let map (Map list1 (lambda '(x) (Uint32 '2)))) +(let data (Condense1 map (lambda '(item) item) (lambda '(item state) (Bool 'false)) (lambda '(item state) (* item state)))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenFold.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenFold.yql new file mode 100644 index 0000000000..06d8d1ee2e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenFold.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let list1 (AsList (String 'a) (String 'b) (String 'c))) +(let map (Map list1 (lambda '(x) (Uint32 '1)))) +(let data (Fold map (Uint32 '0) (lambda '(item state) (+ item state)))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenFold1.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenFold1.yql new file mode 100644 index 0000000000..dbb25f37ee --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ConstMapThenFold1.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let list1 (AsList (String 'a) (String 'b) (String 'c))) +(let map (Map list1 (lambda '(x) (Uint32 '1)))) +(let data (Fold1 map (lambda '(item) item) (lambda '(item state) (+ item state)))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ConsumeOrMoveForwardList.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ConsumeOrMoveForwardList.yql new file mode 100644 index 0000000000..37b6fe207e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ConsumeOrMoveForwardList.yql @@ -0,0 +1,30 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(import id_module '"/lib/yql/id.yql") +(let id (bind id_module 'id)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) (ForwardList (Iterator list (DependsOn (Int32 '0)))) '('('type)))) +(let world (Write! world res_sink (Key) (ForwardList (Apply id (Iterator list (DependsOn (Int32 '1))))) '('('type)))) +(let world (Write! world res_sink (Key) (Fold (ForwardList (Apply id (Iterator list (DependsOn (Int32 '2))))) (String '"") + (lambda '(item state) (Concat item state))) '('('type)))) +(let world (Write! world res_sink (Key) (Fold1 (ForwardList (Apply id (Iterator list (DependsOn (Int32 '3))))) + (lambda '(item) item) + (lambda '(item state) (Concat item state))) '('('type)))) +(let world (Write! world res_sink (Key) (FlatMap (ForwardList (Apply id (Iterator list (DependsOn (Int32 '4))))) + (lambda '(item) (AsList item item))) '('('type)))) +(let world (Write! world res_sink (Key) (FoldMap (ForwardList (Apply id (Iterator list (DependsOn (Int32 '5))))) (String '"") + (lambda '(item state) '((Concat item state) (Concat item state)))) '('('type)))) + +(let world (Write! world res_sink (Key) (Fold1Map (ForwardList (Apply id (Iterator list (DependsOn (Int32 '6))))) + (lambda '(item) '(item item)) + (lambda '(item state) '((Concat item state) (Concat item state)))) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) + +)
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinct.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinct.cfg new file mode 100644 index 0000000000..96364ca2c5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinct.cfg @@ -0,0 +1,3 @@ +in Input inputDistinctKey.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinct.yql new file mode 100644 index 0000000000..6d0405e562 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinct.yql @@ -0,0 +1,41 @@ +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) '('"key") '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let output (block '( + (let select (block '( + (let core table1) + (let core (block '( + (let Count0_create (lambda '(row) (AggrCountInit row))) + (let Count0_update (lambda '(row state) (AggrCountUpdate row state))) + (let Count0_save (lambda '(state) state)) + (let Count0_load (lambda '(item) item)) + (let Count0_merge (lambda '(a b) (OptionalReduce a b (lambda '(a b) (+ a b))))) + (let Count0_finish (lambda '(state) state)) + (return (Aggregate core '() '('('Count0 (AggregationTraits (StructMemberType (ListItemType (TypeOf core)) '"key") Count0_create Count0_update Count0_save Count0_load Count0_merge Count0_finish (Uint64 '0)) '"key")))) + ))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"column0" (Member row 'Count0))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"column0"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinctOpt.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinctOpt.cfg new file mode 100644 index 0000000000..96364ca2c5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinctOpt.cfg @@ -0,0 +1,3 @@ +in Input inputDistinctKey.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinctOpt.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinctOpt.yql new file mode 100644 index 0000000000..ebd34f15ed --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/CountDistinctOpt.yql @@ -0,0 +1,53 @@ +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) '('"key") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (block '( + (let select (block '( + (let core table0) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Cast (Member row '"key") 'Int32))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (block '( + (let Count1_create (lambda '(row) (AggrCountInit row))) + (let Count1_update (lambda '(row state) (AggrCountUpdate row state))) + (let Count1_save (lambda '(state) state)) + (let Count1_load (lambda '(item) item)) + (let Count1_merge (lambda '(a b) (OptionalReduce a b (lambda '(a b) (+ a b))))) + (let Count1_finish (lambda '(state) state)) + (return (Aggregate core '() '('('Count1 (AggregationTraits (StructMemberType (ListItemType (TypeOf core)) '"key") Count1_create Count1_update Count1_save Count1_load Count1_merge Count1_finish (Uint64 '0)) '"key")))) + ))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"column0" (Member row 'Count1))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"column0"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/DedupAndWithCommonPartsUnderOr.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupAndWithCommonPartsUnderOr.cfg new file mode 100644 index 0000000000..b8c8b850ac --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupAndWithCommonPartsUnderOr.cfg @@ -0,0 +1,3 @@ +in Input input4.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/DedupAndWithCommonPartsUnderOr.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupAndWithCommonPartsUnderOr.yql new file mode 100644 index 0000000000..d1a2b6f1e2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupAndWithCommonPartsUnderOr.yql @@ -0,0 +1,16 @@ +( +(let $1 (DataSink 'result)) +(let $2 (Write! world $1 (Key) (Unordered (EnsurePersistable (OrderedFlatMap (OrderedFilter (Right! (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) (Void) '())) (lambda '($3) (Coalesce + (Or + (Or + (And (== (Member $3 '"key") (String '"911")) (== (Member $3 '"value") (String '"kkk"))) + (And (And (== (String '"911") (Member $3 '"key")) (== (String '"xyz") (Member $3 '"value"))) (== (Member $3 '"subkey") (String '"3"))) + ) + (Or + (And (== (String '"911") (Member $3 '"key")) (== (String '"xyz") (Member $3 '"value"))) + (And (== (String '"911") (Member $3 '"key")) (== (String '"aaa") (Member $3 '"value"))) + ) + ) +(Bool 'false)))) (lambda '($4) (AsList $4))))) '('('type) '('autoref)))) +(return (CommitAll! (Commit! $2 $1))) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/DedupOrWithCommonPartsUnderAnd.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupOrWithCommonPartsUnderAnd.cfg new file mode 100644 index 0000000000..b8c8b850ac --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupOrWithCommonPartsUnderAnd.cfg @@ -0,0 +1,3 @@ +in Input input4.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/DedupOrWithCommonPartsUnderAnd.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupOrWithCommonPartsUnderAnd.yql new file mode 100644 index 0000000000..0271673f55 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/DedupOrWithCommonPartsUnderAnd.yql @@ -0,0 +1,16 @@ +( +(let $1 (DataSink 'result)) +(let $2 (Write! world $1 (Key) (Unordered (EnsurePersistable (OrderedFlatMap (OrderedFilter (Right! (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) (Void) '())) (lambda '($3) (Coalesce + (And + (And + (Or (== (Member $3 '"key") (String '"075")) (== (Member $3 '"subkey") (String '"4"))) + (Or (== (String '"075") (Member $3 '"key")) (== (String '"xyz") (Member $3 '"value"))) + ) + (Or + (Or (== (String '"075") (Member $3 '"key")) (== (Member $3 '"value") (String '"xyz"))) + (== (Member $3 '"key") (String '"200")) + ) + ) +(Bool 'false)))) (lambda '($4) (AsList $4))))) '('('type) '('autoref)))) +(return (CommitAll! (Commit! $2 $1))) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/DemuxOverExtend.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/DemuxOverExtend.yql new file mode 100644 index 0000000000..b8eba4c967 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/DemuxOverExtend.yql @@ -0,0 +1,57 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let mr_sink (DataSink 'yt 'plato)) + +(let x (Read! world mr_source (Key '('table (String 'Input))) (Void) '())) +(let world (Left! x)) +(let data (Right! x)) + +(let varTuple (VariantType (TupleType + (StructType + '('key (DataType 'String)) + '('subkey (DataType 'String)) + '('value (DataType 'String)) + ) + (StructType + '('key (DataType 'String)) + '('subkey (DataType 'String)) + '('value (DataType 'String)) + ) +))) + +(let varData1 (Map data (lambda '(item) (block '( + (let intValue (FromString (Member item 'key) 'Int32)) + (let res + (If (Coalesce (Equal (% intValue (Int32 '2)) (Int32 '0)) (Bool 'false)) + (Variant item '0 varTuple) + (Variant item '1 varTuple) + ) + ) + (return res) +))))) + +(let varData2 (Map data (lambda '(item) (block '( + (let intValue (FromString (Member item 'key) 'Int32)) + (let intValue (+ intValue (Int32 '1))) + (let res + (If (Coalesce (Equal (% intValue (Int32 '2)) (Int32 '0)) (Bool 'false)) + (Variant item '0 varTuple) + (Variant item '1 varTuple) + ) + ) + (return res) +))))) + +(let dataTuple (Demux (Extend varData1 varData2))) + +(let data1 (Nth dataTuple '0)) +(let data2 (Nth dataTuple '1)) + +(let data1 (Filter data1 (lambda '(item) (Less (Member item 'key) (String '200))))) + +(let world (Write! world mr_sink (Key '('table (String 'Output))) data1 '('('mode 'renew)))) +(let world (Write! world mr_sink (Key '('table (String 'Output))) data2 '('('mode 'append)))) + +(let world (Commit! world mr_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/DoubleSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/DoubleSort.yql new file mode 100644 index 0000000000..cd9fa2787f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/DoubleSort.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'c) (String 'e) (String 'a))) +(let data (Sort data (Bool 'true) (lambda '(x) x))) +(let data (Sort data (Bool 'true) (lambda '(x) x))) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivInsideNestedLambdaOuterDiffLevel.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivInsideNestedLambdaOuterDiffLevel.yql new file mode 100644 index 0000000000..d78e05c61f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivInsideNestedLambdaOuterDiffLevel.yql @@ -0,0 +1,35 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +#comment +(let res_sink (DataSink 'result)) +(let outLambda1 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let outLambda2 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let anotherLambda (lambda '(sep) (block '( + (let seps (AsList (Concat (String 'z) sep) (Concat (String 'y) sep))) + (let res1 (FlatMap seps outLambda1)) + (let res2 (FlatMap seps outLambda2)) + (return (Extend res1 res2)) +)))) + +(let predicate (== (Int32 '1) (Int32 '2))) +(let sep (If predicate (Just (String '_)) (Just (String '+)))) +(let data1 (IfPresent sep outLambda1 (AsList (String 'none)))) +(let data2 (IfPresent sep anotherLambda (AsList (String 'none)))) +(let world (Write! world res_sink (Key) '(data1 data2) '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivLambda.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivLambda.yql new file mode 100644 index 0000000000..792976bc37 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivLambda.yql @@ -0,0 +1,14 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'a) (String 'a))) +(let data1 (Map data (lambda '(x) (Concat x (Concat (String '_) (String '_)))))) +(let data2 (Map data (lambda '(y) (Concat y (Concat (String '_) (String '_)))))) +(let data (Extend data1 data2)) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOneLevel.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOneLevel.yql new file mode 100644 index 0000000000..2c5734f703 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOneLevel.yql @@ -0,0 +1,20 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let outLambda (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let data2 (Map data (lambda '(y) (Concat y sep)))) + (let res (Extend data1 data2)) + (return res) +)))) +(let predicate (== (Int32 '1) (Int32 '2))) +(let sep (If predicate (Just (String '_)) (Just (String '+)))) +(let data (IfPresent sep outLambda (AsList (String 'none)))) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOuterDiffLevel.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOuterDiffLevel.yql new file mode 100644 index 0000000000..caa3cca763 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOuterDiffLevel.yql @@ -0,0 +1,34 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let outLambda1 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let outLambda2 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let anotherLambda (lambda '(sep) (block '( + (let seps (AsList (Concat (String 'z) sep) (Concat (String 'y) sep))) + (let res (FlatMap seps outLambda2)) + (return res) +)))) + +(let predicate (== (Int32 '1) (Int32 '2))) +(let sep (If predicate (Just (String '_)) (Just (String '+)))) +(let data1 (IfPresent sep outLambda1 (AsList (String 'none)))) +(let data2 (IfPresent sep anotherLambda (AsList (String 'none)))) +(let world (Write! world res_sink (Key) '(data1 data2) '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOuterSameLevel.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOuterSameLevel.yql new file mode 100644 index 0000000000..810324b0dd --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNestedLambdaOuterSameLevel.yql @@ -0,0 +1,28 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let outLambda1 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let outLambda2 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let predicate (== (Int32 '1) (Int32 '2))) +(let sep (If predicate (Just (String '_)) (Just (String '+)))) +(let data1 (IfPresent sep outLambda1 (AsList (String 'none)))) +(let data2 (IfPresent sep outLambda2 (AsList (String 'none)))) +(let world (Write! world res_sink (Key) '(data1 data2) '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNodeInNestedLambda.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNodeInNestedLambda.yql new file mode 100644 index 0000000000..5da1b36ee1 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNodeInNestedLambda.yql @@ -0,0 +1,18 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let predicate (== (Int32 '1) (Int32 '2))) +(let outLambda (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat (Concat x sep) (ToString (Int32 '1)))))) + (return data1) +)))) +(let sep (If predicate (Just (String '_)) (Just (String '+)))) +(let data (IfPresent sep outLambda (AsList (String 'none)))) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNodesInLambdas.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNodesInLambdas.yql new file mode 100644 index 0000000000..3f76dd440f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/EquivNodesInLambdas.yql @@ -0,0 +1,14 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'a) (String 'a))) +(let data1 (Map data (lambda '(x) (Concat x (Concat (String 'l1) (String 'l2)))))) +(let data2 (Map data (lambda '(y) (Concat y (Concat (String 'l2) (String 'l1)))))) +(let data (Extend data1 data2)) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExpandAddRemoveMember.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExpandAddRemoveMember.yql new file mode 100644 index 0000000000..69ede42271 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExpandAddRemoveMember.yql @@ -0,0 +1,17 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let func (lambda '() (AsStruct '('a (String '1)) '('b (Int32 '2))))) +(let type (CallableType '() '((StructType '('a (DataType 'String)) '('b (DataType 'Int32)))))) +(let call (Callable type func)) +(let data (Apply call)) +(let data1 (AddMember data 'd (Bool 'True))) +(let data2 (ForceRemoveMember data 'a)) +(let data2 (ForceRemoveMember data2 'x)) +(let world (Write! world res_sink (Key) '(data1 data2) '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendAndFlatmapExchange.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendAndFlatmapExchange.yql new file mode 100644 index 0000000000..0ff086d5c7 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendAndFlatmapExchange.yql @@ -0,0 +1,29 @@ +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) (Void) '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let table2 (Skip table1 (Uint32 '1))) + (let duplicate (lambda '(row) (block '( + (let res (Struct)) + (let res (AsList row row)) + (return res) + )))) + (let ttable1 (FlatMap table1 duplicate)) + (let ttable2 (FlatMap table2 duplicate)) + (let extend (Extend ttable1 ttable2)) + (let output (Filter extend (lambda '(row) (< (Member row '"key") (String '500))))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendAndFlatmapOptimizeTablePath.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendAndFlatmapOptimizeTablePath.yql new file mode 100644 index 0000000000..ac8743eeca --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendAndFlatmapOptimizeTablePath.yql @@ -0,0 +1,16 @@ +( +(let mr_sync (DataSink 'result)) +(let mr_source (DataSource '"yt" '"plato")) +(let lambda (lambda '(item) (block '( + (return (AsList (AsStruct '('"path" (TablePath (DependsOn item)))))) +)))) +(let res (Extend + (FlatMap (Right! (Read! world mr_source (Key '('table (String '"Input"))) '() '())) lambda) + (FlatMap (Right! (Read! world mr_source (Key '('table (String '"Input"))) '() '())) lambda) + (FlatMap (Extend + (Right! (Read! world mr_source (Key '('table (String '"Input"))) '() '()))) (lambda '(item) (block '( + (return (AsList (AsStruct '('"path" (String '"Special"))))) +)))))) +(let world (ResWrite! world mr_sync (Key) res '('('type) '('autoref) '('columns '('"path"))))) +(return (Commit! (Commit! world mr_sync) (DataSink '"yt" '"plato"))) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendTableWithLiteral.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendTableWithLiteral.yql new file mode 100644 index 0000000000..296bdafd03 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendTableWithLiteral.yql @@ -0,0 +1,80 @@ +#use plato; +# +#SELECT * FROM Input +#UNION ALL +#SELECT 2 AS y; +# +#SELECT 1 AS x +#UNION ALL +#SELECT * FROM Input; + +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) (Void) '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (UnionAll (block '( + (let core table0) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsList row)) + (return res) + ))))) + (return core) + )) (block '( + (let core (AsList (Void))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"y" (Int64 '"2"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + )))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) (Void) '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (UnionAll (block '( + (let core (AsList (Void))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"x" (Int64 '"1"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + )) (block '( + (let core table0) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsList row)) + (return res) + ))))) + (return core) + )))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.cfg new file mode 100644 index 0000000000..80d5638bff --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.cfg @@ -0,0 +1,3 @@ +in Input SomeKeys.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.sql new file mode 100644 index 0000000000..371846c584 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.sql @@ -0,0 +1,13 @@ +USE plato; + +-- Should not generate single MrFill for all content +INSERT INTO Output +SELECT * from( + select key, sum(cast(subkey as Uint32)) as value + from Input group by key + + union all + + select "" as key, sum(cast(subkey as Uint32)) as value + from Input-- group by key +); diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.yql new file mode 100644 index 0000000000..2e266ddca2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithFill.yql @@ -0,0 +1,56 @@ +( +# Should not generate single MrFill for all content +(import aggregate_module '"/lib/yql/aggregate.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) (Void) '())) + (let world (Left! x)) + (let table2 (Right! x)) + (let values (block '( + (let select (block '( + (let core (UnionAll (block '( + (let select (block '( + (let core table2) + (let core (Aggregate core '('"key") '('('Sum0 (Apply (bind aggregate_module '"sum_traits_factory") (TypeOf core) (lambda '(row) (Cast (Member row '"subkey") 'Uint32))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"key" (Member row '"key")) '('"value" (Member row 'Sum0)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) (block '( + (let select (block '( + (let core table2) + (let core (Aggregate core '() '('('Sum1 (Apply (bind aggregate_module '"sum_traits_factory") (TypeOf core) (lambda '(row) (Cast (Member row '"subkey") 'Uint32))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"key" (String '"")) '('"value" (Member row 'Sum1)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsList row)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let sink (DataSink '"yt" '"plato")) + (let world (Write! world sink (Key '('table (String '"Output"))) values '('('mode 'append)))) + (return world) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithTablePath.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithTablePath.cfg new file mode 100644 index 0000000000..807ada95e2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithTablePath.cfg @@ -0,0 +1,3 @@ +in Input1 input.txt +in Input2 input2.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithTablePath.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithTablePath.yql new file mode 100644 index 0000000000..a61e2d4d2f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtendWithTablePath.yql @@ -0,0 +1,48 @@ +#USE plato; +#select table, count(*) from +# (select TablePath() as table from CONCAT([Input],[Input])) group by table; + +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input1"))) (Key '('table (String '"Input2")))) '() '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (block '( + (let select (block '( + (let core table0) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"table" (TablePath (DependsOn row))))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (Aggregate core '('"table") '('('Count1 (Apply (bind aggregate_module '"count_traits_factory") (TypeOf core) (lambda '(row) (Void))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"table" (Member row '"table")) '('"column1" (Member row 'Count1)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"table" '"column1"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOnMultiInput.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOnMultiInput.cfg new file mode 100644 index 0000000000..807ada95e2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOnMultiInput.cfg @@ -0,0 +1,3 @@ +in Input1 input.txt +in Input2 input2.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOnMultiInput.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOnMultiInput.yql new file mode 100644 index 0000000000..d775d38829 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOnMultiInput.yql @@ -0,0 +1,22 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x1 (Read! world mr_source (Key '('table (String 'Input1))) '('key 'subkey 'value) '())) +(let x2 (Read! world mr_source (Key '('table (String 'Input2))) '('key 'subkey 'value) '())) +(let world (Left! x1)) +(let table1 (RemoveSystemMembers (Right! x1))) +(let table2 (RemoveSystemMembers (Right! x2))) + +(let joinType '('LeftOnly '"t" '"d" '('"t" '"key") '('"d" '"key") '())) +(let $67 '('"rename" '"t.key" '"key")) +(let $68 '('"rename" '"t.subkey" '"subkey")) +(let $69 '('"rename" '"t.value" '"value")) +(let rename '($67 $68 $69)) + +(let join (EquiJoin '(table1 '"t") '((FlatMap (Extend (ExtractMembers table1 '('"key")) (ExtractMembers table2 '('"key"))) (lambda '($159) (Just (AsStruct '('"key" (Member $159 '"key")))))) '"d") joinType rename)) +(let out (Extend table1 table2 join)) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) out '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverEquiJoin.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverEquiJoin.cfg new file mode 100644 index 0000000000..6f99339756 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverEquiJoin.cfg @@ -0,0 +1,2 @@ +in Input input.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverEquiJoin.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverEquiJoin.yql new file mode 100644 index 0000000000..08083fbc54 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverEquiJoin.yql @@ -0,0 +1,12 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table (Right! x)) +(let join (EquiJoin '(table 'a) '(table 'b) '('Inner '"a" '"b" '('"a" '"key") '('"b" '"value" ) '()) '())) +(let join (ExtractMembers join '('a.key 'b.value))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) join '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverRead.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverRead.yql new file mode 100644 index 0000000000..e5bccc409e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractMembersOverRead.yql @@ -0,0 +1,17 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (ExtractMembers table1 '('key 'value))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table1 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractPredicateFromFlatmapOverListIf.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractPredicateFromFlatmapOverListIf.yql new file mode 100644 index 0000000000..0a63ce0c2c --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ExtractPredicateFromFlatmapOverListIf.yql @@ -0,0 +1,37 @@ +( +# read data from Input table +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)))) +(let str3 (AsStruct '('a (Uint32 '3)) '('b (String 'c)))) +(let list1 (AsList str1 str2)) +(let list2 (AsList str2 str1)) +(let list3 (AsList str1 str3)) + +(let cond (>= (RandomNumber) (Uint64 '0))) + +(let res (FlatMap (OptionalIf cond str1) (lambda '(r) (OptionalIf (>= (Member r 'b) (String 'a)) r)))) +(let world (Write! world res_sink (Key) res '())) + +(let res (FlatMap (OptionalIf cond str1) (lambda '(r) (AsList r r)))) +(let world (Write! world res_sink (Key) res '())) + +(let res (FlatMap (OptionalIf cond str1) (lambda '(r) (ToStream (Just r) (DependsOn r))))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let res (FlatMap (ListIf cond str1) (lambda '(r) (OptionalIf (>= (Member r 'b) (String 'a)) r)))) +(let world (Write! world res_sink (Key) res '())) + +(let res (FlatMap (ListIf cond str1) (lambda '(r) (AsList r r)))) +(let world (Write! world res_sink (Key) res '())) + +(let res (FlatMap (ListIf cond str1) (lambda '(r) (ToStream (Just r) (DependsOn r))))) +(let world (Write! world res_sink (Key) res '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FalattenByColumns.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FalattenByColumns.cfg new file mode 100644 index 0000000000..fa5e600b39 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FalattenByColumns.cfg @@ -0,0 +1,2 @@ +in Input typed_table.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FalattenByColumns.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FalattenByColumns.yql new file mode 100644 index 0000000000..4a52d5d457 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FalattenByColumns.yql @@ -0,0 +1,38 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")) '('view (String '"opt_struct")))) '('"b") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core table0) + (let core (FlatMap core (lambda '(row) (block '( + (let res (FlattenByColumns row '('b 'bb))) + (return res) + ))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"bb" (Member row '"bb"))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"bb"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapAfterCombineByKey.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapAfterCombineByKey.yql new file mode 100644 index 0000000000..abfdcb837a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapAfterCombineByKey.yql @@ -0,0 +1,19 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'a) (String 'a) (String 'b))) +(let preMap (lambda '(item) (Just item))) +(let keyExtractor (lambda '(item) item)) +(let init (lambda '(key item) item)) +(let update (lambda '(key item state) (Concat state item))) +(let finish (lambda '(key state) (AsList '(key state)))) +(let combined (CombineByKey data preMap keyExtractor init update finish)) +(let map (Map combined (lambda '(x) (Nth x '1)))) +(let result (Sort map (Bool 'true) (lambda '(x) x))) +(let world (Write! world res_sink (Key) result '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapAfterFilterNullMembers.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapAfterFilterNullMembers.yql new file mode 100644 index 0000000000..62ab06b3ec --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapAfterFilterNullMembers.yql @@ -0,0 +1,18 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList + (AsStruct '('data (Just (String 'a)))) + (AsStruct '('data (Nothing (OptionalType (DataType 'String))))) + (AsStruct '('data (Just (String 'b)))) +)) +(let filtered (FilterNullMembers data '('data))) +(let map (Map filtered (lambda '(x) '(x x)))) +(let result (Sort map (Bool 'true) (lambda '(x) (Member (Nth x '0) 'data)))) +(let world (Write! world res_sink (Key) result '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapBeforeCombineByKey.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapBeforeCombineByKey.yql new file mode 100644 index 0000000000..aa7b1d874a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapBeforeCombineByKey.yql @@ -0,0 +1,19 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'a) (String 'a) (String 'b) (String 'c))) +(let preMap (lambda '(item) (Just item))) +(let keyExtractor (lambda '(item) item)) +(let init (lambda '(key item) item)) +(let update (lambda '(key item state) (Concat state item))) +(let finish (lambda '(key state) (AsList '(key state)))) +(let map (Filter data (lambda '(x) (< x (String 'c))))) +(let combined (CombineByKey map preMap keyExtractor init update finish)) +(let result (Sort combined (Bool 'true) (lambda '(x) (Nth x '0)))) +(let world (Write! world res_sink (Key) result '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapToExtractMembers.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapToExtractMembers.yql new file mode 100644 index 0000000000..b0dfbcde05 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapToExtractMembers.yql @@ -0,0 +1,30 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) + +(let res1 (FlatMap list1 (lambda '(item) (block '( + (let ret (Just (AsStruct '('a (Member item 'a)) '('c (Member item 'c))))) + (return ret) +))))) + +(let res2 (FlatMap list1 (lambda '(item) (block '( + (let ret (Just (AsStruct '('b (Member item 'b)) '('k (Member item 'c))))) + (return ret) +))))) + +(let res3 (FlatMap list1 (lambda '(item) (block '( + (let ret (Just (AsStruct))) + (return ret) +))))) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) res1 '('('type)))) +(let world (Write! world res_sink (Key) res2 '('('type)))) +(let world (Write! world res_sink (Key) res3 '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapWithVariantItemAfterSwitch.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapWithVariantItemAfterSwitch.yql new file mode 100644 index 0000000000..855ec4454f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FlatMapWithVariantItemAfterSwitch.yql @@ -0,0 +1,25 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) +(let res_sink (DataSink 'result)) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType (TupleType structType structType structType)) +(let vt (VariantType tupleType)) +(let vlist (AsList + (Variant (AsStruct '('key (String 'aaa)) '('value (String '0))) '0 vt) + (Variant (AsStruct '('key (String 'bbb)) '('value (String '1))) '1 vt) + (Variant (AsStruct '('key (String 'ccc)) '('value (String '2))) '2 vt) +)) + +(let takeHandler (lambda '(x) (Take x (Uint64 '42)))) + +(let world (Write! world res_sink (Key) (Collect (Map (Switch (ToFlow vlist) '1 '('0) takeHandler '('1 '2) takeHandler) (lambda '(s) (VariantItem s)))) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Map (Switch (ToFlow vlist) '1 '('1) takeHandler '('2) takeHandler) (lambda '(s) (VariantItem s)))) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Map (Switch (ToFlow vlist) '1 '('0 '1) takeHandler '('1 '2) takeHandler) (lambda '(s) (VariantItem s)))) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (OrderedMap (Switch (ToFlow vlist) '1 '('0 '1 '2) takeHandler) (lambda '(s) (VariantItem s)))) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1ConstMax.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1ConstMax.yql new file mode 100644 index 0000000000..a32571a772 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1ConstMax.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list + (lambda '(item) (String 'z)) + (lambda '(item state) (Max state (String 'y))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1ConstMin.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1ConstMin.yql new file mode 100644 index 0000000000..f173ae96ab --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1ConstMin.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list + (lambda '(item) (String 'z)) + (lambda '(item state) (Min state (String 'y))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1EmptyStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1EmptyStruct.yql new file mode 100644 index 0000000000..a9b8d1e1ee --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1EmptyStruct.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list (lambda '(item) (Struct)) (lambda '(item state) state))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1EmptyTuple.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1EmptyTuple.yql new file mode 100644 index 0000000000..e6be656d48 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1EmptyTuple.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list (lambda '(item) '()) (lambda '(item state) state))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1IncrementAsLength.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1IncrementAsLength.yql new file mode 100644 index 0000000000..47cda00536 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1IncrementAsLength.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list + (lambda '(item) (Uint32 '1)) + (lambda '(item state) (+ state (Uint32 '1))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OneItemStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OneItemStruct.yql new file mode 100644 index 0000000000..5077ac27e4 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OneItemStruct.yql @@ -0,0 +1,24 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list + (lambda '(item) + (AddMember + (Struct) + 'z + item)) + (lambda '(item state) + (AddMember + (Struct) + 'z + (Concat (Member state 'z) item))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OneItemTuple.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OneItemTuple.yql new file mode 100644 index 0000000000..0f6615de39 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OneItemTuple.yql @@ -0,0 +1,19 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold1 list + (lambda '(item) + '(item)) + (lambda '(item state) (block '( + (let x (Concat (Nth state '0) item)) + (return '(x))))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OverOptional.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OverOptional.yql new file mode 100644 index 0000000000..c4ac775064 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Fold1OverOptional.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (Just (String 'a))) +(let data (Fold1 list +(lambda '(item) (Concat item (String 'b))) +(lambda '(item state) (Concat item state)))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldConstMax.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldConstMax.yql new file mode 100644 index 0000000000..cc2d3f5d62 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldConstMax.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold list + (String 'z) + (lambda '(item state) (Max state (String 'y))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldConstMin.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldConstMin.yql new file mode 100644 index 0000000000..d7f3bc98b5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldConstMin.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold list + (String 'z) + (lambda '(item state) (Min state (String 'y))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldEmptyStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldEmptyStruct.yql new file mode 100644 index 0000000000..c170ea2fc0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldEmptyStruct.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold list (Struct) (lambda '(item state) state))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldEmptyTuple.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldEmptyTuple.yql new file mode 100644 index 0000000000..73ac847631 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldEmptyTuple.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold list '() (lambda '(item state) state))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldIncrementAsLength.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldIncrementAsLength.yql new file mode 100644 index 0000000000..758cd43bf4 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldIncrementAsLength.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold list (Uint32 '0) (lambda '(item state) (+ state (Uint32 '1))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldOneItemStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldOneItemStruct.yql new file mode 100644 index 0000000000..12014ead02 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldOneItemStruct.yql @@ -0,0 +1,19 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let data (Fold list (AddMember (Struct) 'z (String '"")) + (lambda '(item state) + (AddMember + (Struct) + 'z + (Concat (Member state 'z) item))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FoldOneItemTuple.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldOneItemTuple.yql new file mode 100644 index 0000000000..ccada97695 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FoldOneItemTuple.yql @@ -0,0 +1,18 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let s (String '"")) +(let data (Fold list '(s) + (lambda '(item state) (block '( + (let state (Concat (Nth state '0) item)) + (return '(state))))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseExtractMembers.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseExtractMembers.yql new file mode 100644 index 0000000000..8d267ce890 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseExtractMembers.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let world (Write! world res_sink (Key) (ExtractMembers (ExtractMembers list1 '('a 'c)) '('a)) '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapWithExtractMembers.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapWithExtractMembers.yql new file mode 100644 index 0000000000..57c81db622 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapWithExtractMembers.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let world (Write! world res_sink (Key) (Map (ExtractMembers list1 '('a 'c)) (lambda '(row) '((+ (Member row 'a) (Uint32 '10)) (Member row 'c)))) '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapWithStreamLambda.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapWithStreamLambda.yql new file mode 100644 index 0000000000..c4585070bf --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapWithStreamLambda.yql @@ -0,0 +1,15 @@ +( +(let res_sink (DataSink 'result)) + +(let singleList (AsList (Int32 '1))) +(let optional (/ (Int32 '1) (Int32 '0))) +(let streamLambda (lambda '(x) (Iterator (AsList (Int32 '1) (Int32 '2))))) + +(let map1 (Unordered (FlatMap singleList streamLambda))) +(let map2 (Unordered (FlatMap (ToList optional) streamLambda))) + +(let world (Write! world res_sink (Key) map1 '('('type)))) +(let world (Write! world res_sink (Key) map2 '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapsWithDependsOn.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapsWithDependsOn.yql new file mode 100644 index 0000000000..0553caa8fd --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatMapsWithDependsOn.yql @@ -0,0 +1,35 @@ +( +#comment +(let mr_source (DataSource 'yt 'plato)) +(let res_sink (DataSink 'result)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) + +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table (Right! x)) + +(let inner1 (FlatMap table (lambda '(row) (OptionalIf (> (Member row 'key) (String '100)) (AsStruct '('key1 (Member row 'key))))))) + +(let inner2 (FlatMap table (lambda '(row) (Just (AsStruct '('key1 (Member row 'key))))))) + +(let outer (lambda '(input) (FlatMap input (lambda '(row) + (Just + (AsStruct + '('key (Member row 'key1)) + '('value (ToOptional (Collect (Condense + (Iterator list (DependsOn row)) + (String '"") + (lambda '(item state) (Bool 'False)) + (lambda '(item state) (Concat state item)) + )))) + ) + ) +)))) + + +(let world (Write! world res_sink (Key) (Apply outer inner1) '('('type)))) +(let world (Write! world res_sink (Key) (Apply outer inner2) '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatmapsWithInnerVisit.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatmapsWithInnerVisit.yql new file mode 100644 index 0000000000..9a0a42e1f3 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatmapsWithInnerVisit.yql @@ -0,0 +1,59 @@ +( +#comment +(let config (DataSource 'config)) +(let res_sink (DataSink 'result)) + +(let world (Configure! world config 'PureDataSource 'yt)) + +(let vt (VariantType (TupleType (DataType 'String) (DataType 'String) (DataType 'String)))) +(let v0 (Variant (String 'aaa) '0 vt)) +(let v1 (Variant (String 'bbb) '1 vt)) +(let v2 (Variant (String 'ccc) '2 vt)) +(let list (AsList v0 v1 v2)) + + +(let inner (lambda '(list) + (FlatMap list (lambda '(item) + (Visit item + '0 (lambda '(x) (OptionalIf (== x (String 'aaa)) (Variant (Concat (String '0-) x) '0 vt))) + '1 (lambda '(x) (Just (Variant (Concat (String '1-) x) '1 vt))) + (Just (Variant (String '2-) '2 vt)) + ) + )) +)) + +(let res (FlatMap (Apply inner (Iterator list (DependsOn (String 'a)))) (lambda '(item) + (Just (VariantItem item)) +))) +(let world (Write! world res_sink (Key) (Collect res) '('('type)))) + +(let res (FlatMap (Apply inner (Iterator list (DependsOn (String 'b)))) (lambda '(item) + (Just (Visit item + '0 (lambda '(x) (Variant x '0 vt)) + '1 (lambda '(x) (Variant x '1 vt)) + '2 (lambda '(x) (Variant x '1 vt)) + )) +))) +(let world (Write! world res_sink (Key) (Collect res) '('('type)))) + +(let res (FlatMap + (FlatMap list (lambda '(item) + (Visit item + '0 (lambda '(x) (ListIf (== x (String 'aaa)) (Variant (Concat (String '0-) x) '0 vt))) + '1 (lambda '(x) (AsList (Variant (Concat (String '1-) x) '1 vt) (Variant (Concat (String '2-) x) '2 vt))) + (AsList (Variant (String '2-) '2 vt)) + ) + )) + (lambda '(item) + (Just (Visit item + '0 (lambda '(x) (Variant x '0 vt)) + '1 (lambda '(x) (Variant x '1 vt)) + '2 (lambda '(x) (Variant x '1 vt)) + )) + ) +)) +(let world (Write! world res_sink (Key) (Collect res) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatmapsWithOptionalIf.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatmapsWithOptionalIf.yql new file mode 100644 index 0000000000..3a7375371f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseFlatmapsWithOptionalIf.yql @@ -0,0 +1,32 @@ +( +# read data from Input table +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)))) +(let list (AsList str1 str2)) + +(let res (FlatMap (FlatMap (Iterator list (DependsOn (String '1))) (lambda '(r) (OptionalIf (> (Member r 'a) (Uint32 '0)) r))) (lambda '(r) (ToStream (Just r))))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let res (FlatMap (FlatMap (Iterator list (DependsOn (String '2))) (lambda '(r) (OptionalIf (> (Member r 'a) (Uint32 '0)) r))) (lambda '(r) (AsList r r)))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let res (FlatMap (FlatMap (Iterator list (DependsOn (String '3))) (lambda '(r) (OptionalIf (> (Member r 'a) (Uint32 '0)) r))) (lambda '(r) (OptionalIf (> (Member r 'b) (String 'a)) r)))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let res (FlatMap (FlatMap (Iterator list (DependsOn (String '4))) (lambda '(r) (ListIf (> (Member r 'a) (Uint32 '0)) r))) (lambda '(r) (ToStream (Just r))))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let res (FlatMap (FlatMap (Iterator list (DependsOn (String '5))) (lambda '(r) (ListIf (> (Member r 'a) (Uint32 '0)) r))) (lambda '(r) (AsList r r)))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let res (FlatMap (FlatMap (Iterator list (DependsOn (String '6))) (lambda '(r) (ListIf (> (Member r 'a) (Uint32 '0)) r))) (lambda '(r) (OptionalIf (> (Member r 'b) (String 'a)) r)))) +(let world (Write! world res_sink (Key) (ForwardList res) '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg new file mode 100644 index 0000000000..0bbf3ad9f0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.cfg @@ -0,0 +1,4 @@ +in Input input.txt +out Output output.txt +res result.txt +udf python2_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.sql new file mode 100644 index 0000000000..0f5cd0b0c6 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.sql @@ -0,0 +1,31 @@ +USE plato; + +$a = (SELECT + skey,count(*) as cnt +FROM Input +GROUP BY Length(key) as skey); + +$udfScript = @@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@; + +$udf = Python::f(@@ +(Stream<Struct<skey:Uint32,cnt:Uint64>>,Int32) +-> +Stream<Struct<skey:Uint32,cnt:Uint64,pass:Int32>> +@@, $udfScript); + +$udf2 = Python::f(@@ +(Stream<Struct<skey:Uint32,cnt:Uint64,pass:Int32>>,Int32) +-> +Stream<Struct<skey:Uint32,cnt:Uint64,pass:Int32>> +@@, $udfScript); + + +$b = (PROCESS $a USING $udf($ROWS,1)); + +PROCESS $b USING $udf2($ROWS,2); diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql new file mode 100644 index 0000000000..cc5b60392b --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterLReduce.yql @@ -0,0 +1,82 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(import core_module '"/lib/yql/core.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) '('"key") '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let output (block '( + (let select (block '( + (let inputRowsList (block '( + (let select (block '( + (let inputRowsList (block '( + (let select (block '( + (let core table1) + (let core (FlatMap core (lambda '(row) (AsList (AsStruct '('"key" (Member row '"key")) '('"skey" ("Size" (Member row '"key")))))))) + (let core (Aggregate core '('"skey") '('('Count0 (Apply (bind aggregate_module '"count_traits_factory") (TypeOf core) (lambda '(row) (Void))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"skey" (Member row '"skey")) '('"cnt" (Member row 'Count0)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (LMap inputRowsList (lambda '(x) (block '( + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@))) '0)) + (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) + (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@)) inputRowsList (Int32 '"1"))) + ))))) + (return core) + ))) + (return select) + ))) + (let core (LMap inputRowsList (lambda '(x) (block '( + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@))) '0)) + (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) + (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@)) inputRowsList (Int32 '"2"))) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg new file mode 100644 index 0000000000..0bbf3ad9f0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.cfg @@ -0,0 +1,4 @@ +in Input input.txt +out Output output.txt +res result.txt +udf python2_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.sql new file mode 100644 index 0000000000..f0bd1ff3ba --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.sql @@ -0,0 +1,23 @@ +USE plato; + +$a = (SELECT + skey,count(*) as cnt +FROM Input +GROUP BY Length(key) as skey); + +$udfScript = @@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@; + +$udf = Python::f(@@ +(Stream<Struct<skey:Uint32,cnt:Uint64>>,Int32) +-> +Stream<Struct<skey:Uint32,cnt:Uint64,pass:Int32>> +@@, $udfScript); + + +PROCESS $a USING $udf($ROWS,1); diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql new file mode 100644 index 0000000000..1f806dc3a3 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseLMapAfterReduce.yql @@ -0,0 +1,59 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(import core_module '"/lib/yql/core.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) '('"key") '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let output (block '( + (let select (block '( + (let inputRowsList (block '( + (let select (block '( + (let core table1) + (let core (FlatMap core (lambda '(row) (AsList (AsStruct '('"key" (Member row '"key")) '('"skey" ("Size" (Member row '"key")))))))) + (let core (Aggregate core '('"skey") '('('Count0 (Apply (bind aggregate_module '"count_traits_factory") (TypeOf core) (lambda '(row) (Void))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"skey" (Member row '"skey")) '('"cnt" (Member row 'Count0)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (LMap inputRowsList (lambda '(x) (block '( + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@))) '0)) + (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) + (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d +@@)) inputRowsList (Int32 '"1"))) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg new file mode 100644 index 0000000000..0bbf3ad9f0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.cfg @@ -0,0 +1,4 @@ +in Input input.txt +out Output output.txt +res result.txt +udf python2_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.sql new file mode 100644 index 0000000000..c87931cafd --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.sql @@ -0,0 +1,32 @@ +USE plato; + +$a = (SELECT + skey,count(*) as cnt +FROM Input +GROUP BY Length(key) as skey); + +$udfScript = @@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d + +def f2(a,b,c): + return {"a":a,"b":b,"c":c} +@@; + +$udf = Python::f(@@ +(Stream<Struct<skey:Uint32,cnt:Uint64>>,Int32) +-> +Stream<Struct<skey:Uint32,cnt:Uint64,pass:Int32>> +@@, $udfScript); + +$udf2 = Python::f2(@@ +(skey:Uint32,cnt:Uint64,pass:Int32)->Struct<a:Uint32,b:Uint64,c:Int32> +@@, $udfScript); + + +$b = (PROCESS $a USING $udf($ROWS,1)); + +PROCESS $b USING $udf2(skey,cnt,pass); diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql new file mode 100644 index 0000000000..ea138e481a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMapAfterLReduce.yql @@ -0,0 +1,85 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(import core_module '"/lib/yql/core.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) '('"key") '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let output (block '( + (let select (block '( + (let core (block '( + (let select (block '( + (let inputRowsList (block '( + (let select (block '( + (let core table1) + (let core (FlatMap core (lambda '(row) (AsList (AsStruct '('"key" (Member row '"key")) '('"skey" ("Size" (Member row '"key")))))))) + (let core (Aggregate core '('"skey") '('('Count0 (Apply (bind aggregate_module '"count_traits_factory") (TypeOf core) (lambda '(row) (Void))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"skey" (Member row '"skey")) '('"cnt" (Member row 'Count0)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (LMap inputRowsList (lambda '(x) (block '( + (let inputType (CallableArgumentType (TypeOf (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d + +def f2(a,b,c): + return {"a":a,"b":b,"c":c} +@@))) '0)) + (let inputRowsList (MatchType inputType 'List (lambda '(y) (ForwardList x)) (lambda '(y) "x"))) + (return (Apply (ScriptUdf 'Python '"f" (CallableType '() '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"pass" (DataType 'Int32)) '('"skey" (DataType 'Uint32))))) '((StreamType (StructType '('"cnt" (DataType 'Uint64)) '('"skey" (DataType 'Uint32))))) '((DataType 'Int32))) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d + +def f2(a,b,c): + return {"a":a,"b":b,"c":c} +@@)) inputRowsList (Int32 '"1"))) + ))))) + (return core) + ))) + (return select) + ))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Apply (ScriptUdf 'Python '"f2" (CallableType '() '((StructType '('"a" (DataType 'Uint32)) '('"b" (DataType 'Uint64)) '('"c" (DataType 'Int32)))) '((DataType 'Uint32) 'skey) '((DataType 'Uint64) 'cnt) '((DataType 'Int32) 'pass)) (String '@@ +def f(input,x): + for i in list(input): + d = i.__dict__ + d["pass"] = x + yield d + +def f2(a,b,c): + return {"a":a,"b":b,"c":c} +@@)) (Member row '"skey") (Member row '"cnt") (Member row '"pass"))) + (let res (MatchType res 'Optional (lambda '(opt) (ToList opt)) (lambda '(val) (MatchType val 'List (lambda '(list) list) (lambda '(item) (AsList item)))))) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref)))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMultipliersFlatmaps.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMultipliersFlatmaps.yql new file mode 100644 index 0000000000..6241a465a7 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/FuseMultipliersFlatmaps.yql @@ -0,0 +1,20 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList + (AsStruct '('data (String 'a))) + (AsStruct '('data (String 'b))) + (AsStruct '('data (String 'c))) +)) + +(let one (OrderedFlatMap data (lambda '(x) (AsList x (AsStruct '('data (String 'z))))))) +(let two (OrderedFlatMap one (lambda '(x) (AsList (AsStruct '('data (String 'y))) x x)))) +(let three (OrderedFlatMap two (lambda '(x) (AsList x (AsStruct '('data (String 'x))) x)))) +(let world (Write! world res_sink (Key) three '())) +(let world (Commit! world res_sink)) +(return world) +) + diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/GroupBySingleKeyListUnused.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/GroupBySingleKeyListUnused.yql new file mode 100644 index 0000000000..877db04e8b --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/GroupBySingleKeyListUnused.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let keyExtractor (lambda '(x) (String '"@"))) +(let handler (lambda '(key list) (AsList (Uint32 '42)))) +(let data (GroupByKey list keyExtractor handler)) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/GroupBySingleKeyListUsed.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/GroupBySingleKeyListUsed.yql new file mode 100644 index 0000000000..40045830d0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/GroupBySingleKeyListUsed.yql @@ -0,0 +1,16 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list (AsList (String 'a) (String 'b) (String 'c))) +(let keyExtractor (lambda '(x) (String '"@"))) +(let handler (lambda '(key list) (AsList '(key (Length list))))) +(let data (GroupByKey list keyExtractor handler)) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/HasItemsAfterEnumerate.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/HasItemsAfterEnumerate.yql new file mode 100644 index 0000000000..eb78e634af --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/HasItemsAfterEnumerate.yql @@ -0,0 +1,12 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let data (HasItems (Enumerate (AsList (String 'a) (String 'b) (String 'c))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) data '())) + +(let world (Commit! world res_sink)) +(return world) +) + diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/IfOverNotPredicate.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/IfOverNotPredicate.yql new file mode 100644 index 0000000000..798f4b5825 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/IfOverNotPredicate.yql @@ -0,0 +1,11 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (If (Not (ListHas (AsList (Uint32 '1)) (Uint32 '1))) (String 'a) (String 'b))) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/InjectExtractMembersOverSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/InjectExtractMembersOverSort.yql new file mode 100644 index 0000000000..2346c6fbe8 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/InjectExtractMembersOverSort.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Sort list1 (Bool 'true) (lambda '(row) (Member row 'a)))) +(let list1 (ExtractMembers list1 '('b))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/InnerLambda.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/InnerLambda.yql new file mode 100644 index 0000000000..cefbe8d819 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/InnerLambda.yql @@ -0,0 +1,18 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let list1 (AsList (String 'a) (String 'b))) +(let list2 (AsList (String 'c) (String 'd))) +(let data (FlatMap list1 (lambda '(x) (block '( + (let myList (Map list2 (lambda '(y) (block '( + (return (Concat x y))))))) + (return myList) +))))) + +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/InputUniq.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/InputUniq.txt new file mode 100644 index 0000000000..95a39214c5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/InputUniq.txt @@ -0,0 +1,8 @@ +{"key"="023";"subkey"="3";"value"="aaa"}; +{"key"="037";"subkey"="5";"value"="ddd"}; +{"key"="075";"subkey"="1";"value"="abc"}; +{"key"="150";"subkey"="8";"value"="zzz"}; +{"key"="200";"subkey"="7";"value"="qqq"}; +{"key"="527";"subkey"="4";"value"="bbb"}; +{"key"="761";"subkey"="6";"value"="ccc"}; +{"key"="911";"subkey"="2";"value"="kkk"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/InputUniq.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/InputUniq.txt.attr new file mode 100644 index 0000000000..b10bf7c075 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/InputUniq.txt.attr @@ -0,0 +1,10 @@ +{ + "_yql_row_spec"={ + "UniqueKeys"=%true; + "SortMembers"=["key"]; + "SortedBy"=["key"]; + "SortDirections"=[1]; + "SortedByTypes"=[["DataType";"String"]]; + "Type"=["StructType";[["key";["DataType";"String"]];["subkey";["DataType";"String"]];["value";["DataType";"String"]]]] + } +} diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LMapWithEmptyInput.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/LMapWithEmptyInput.cfg new file mode 100644 index 0000000000..9d65859cb9 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LMapWithEmptyInput.cfg @@ -0,0 +1,3 @@ +out Output output.txt +res result.txt +udf python3_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LMapWithEmptyInput.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LMapWithEmptyInput.yql new file mode 100644 index 0000000000..5395d3e9f8 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LMapWithEmptyInput.yql @@ -0,0 +1,23 @@ +( +(let structType (StructType '('"key" (DataType 'String)) '('"subkey" (DataType 'String)) '('"value" (DataType 'String)))) +#comment +(let udfType (CallableType '() '((StreamType structType)) '((StreamType structType)))) +(let script '@@ +def f(input): + for i in input: + yield { + 'key': i.key, + 'subkey': i.subkey, + 'value': i.value + } +@@) +(let udf (ScriptUdf 'Python3 '"f" udfType (String script))) + +(let res (LMap (List (ListType structType)) (lambda '(stream) (block '( + (return (Apply udf stream)) +))))) +(let mr_sink (DataSink 'yt (quote plato))) +(let world (Write! world mr_sink (Key '('table (String 'Output))) res '('('mode 'append)))) +(let world (Commit! world mr_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterAsList.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterAsList.yql new file mode 100644 index 0000000000..1e36b20260 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterAsList.yql @@ -0,0 +1,13 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let data (Length (AsList (String 'a) (String 'b)))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterEnumerate.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterEnumerate.yql new file mode 100644 index 0000000000..3926c6303f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterEnumerate.yql @@ -0,0 +1,12 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let data (Length (Enumerate (AsList (String 'a) (String 'b) (String 'c))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) data '())) + +(let world (Commit! world res_sink)) +(return world) +) + diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterList.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterList.yql new file mode 100644 index 0000000000..89e18df910 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterList.yql @@ -0,0 +1,13 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let data (Length (List (ListType (DataType 'String)) (String 'a) (String 'b)))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + data '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterMap.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterMap.yql new file mode 100644 index 0000000000..90563d41a3 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthAfterMap.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Map table1 (lambda '(x) '(x (String 'qwe))))) +(let table1len (Length table1)) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table1len '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContent.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContent.cfg new file mode 100644 index 0000000000..bf54cfd5be --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContent.cfg @@ -0,0 +1,2 @@ +in Input input_tbl_content.txt +out Output output.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContent.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContent.yql new file mode 100644 index 0000000000..1a1056aa00 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContent.yql @@ -0,0 +1,16 @@ +( + #comment + (let mr_source (DataSource 'yt 'plato)) + (let r (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))) + (let input (Right! r)) + (let out (FlatMap input (lambda '(item) (block '( + (let len (Length input)) + (let key (Member item 'key)) + (let key (Coalesce (Cast key 'Uint64) (Uint64 '0))) + (return (ListIf (> key len) item)) + ))))) + (let mr_sink (DataSink 'yt 'plato)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append)))) + (let world (Commit! world mr_sink)) + (return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContentInsideMap.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContentInsideMap.cfg new file mode 100644 index 0000000000..bf54cfd5be --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContentInsideMap.cfg @@ -0,0 +1,2 @@ +in Input input_tbl_content.txt +out Output output.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContentInsideMap.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContentInsideMap.yql new file mode 100644 index 0000000000..ce4c782796 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/LengthOverMrTableContentInsideMap.yql @@ -0,0 +1,20 @@ +( + #comment + (let mr_source (DataSource 'yt 'plato)) + (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) + (let world (Left! x)) + (let table1 (Right! x)) + + (let r (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))) + + (let out (FlatMap table1 (lambda '(item) (block '( + (let len (Length (Right! r))) + (let key (Member item 'key)) + (let key (Coalesce (Cast key 'Uint64) (Uint64 '0))) + (return (ListIf (> key len) item)) + ))))) + (let mr_sink (DataSink 'yt 'plato)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append)))) + (let world (Commit! world mr_sink)) + (return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MapNextWithUnusedNext.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MapNextWithUnusedNext.yql new file mode 100644 index 0000000000..06f26563b3 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MapNextWithUnusedNext.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let lst (AsList (Int32 '0) (Int32 '1) (Int32 '2) (Int32 '3))) +(let lst (Iterator lst)) +(let lst (MapNext lst (lambda '(current next) (+ current (Int32 '1))))) +(let lst (ForwardList lst)) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) lst '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MapThenSortWithLimit.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/MapThenSortWithLimit.cfg new file mode 100644 index 0000000000..a3547c5df6 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MapThenSortWithLimit.cfg @@ -0,0 +1,3 @@ +in Input input2.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MapThenSortWithLimit.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MapThenSortWithLimit.yql new file mode 100644 index 0000000000..800f3c5a9e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MapThenSortWithLimit.yql @@ -0,0 +1,37 @@ +( +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (Key '('table (String '"Input"))) '('"value" '"key") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core table0) + (let core (Filter core (lambda '(row) (Coalesce ("<" (Member row '"value") (String '"f")) (Bool 'false))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (Struct)) + (let res (AddMember res '"key" (Member row '"key"))) + (let res (AsList res)) + (return res) +) +)))) + (return core) +) +)) + (let select (Sort select (Bool 'true) (lambda '(row) (Member row '"key")))) + (let select (Take select (Uint64 '"2"))) + (return select) +) +)) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"key"))))) + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return (Commit! world result_sink)) +) +)) + (return world) +) +)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MemberOverFlattenMembers.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MemberOverFlattenMembers.yql new file mode 100644 index 0000000000..6d4866a169 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MemberOverFlattenMembers.yql @@ -0,0 +1,17 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let myStruct1 (AsStruct '('a (String 'z)))) +(let myStruct2 (AsStruct '('b (String 'y)))) +(let flatten (FlattenMembers '('"first_" myStruct1) '('"second_" myStruct2))) +(let world (Write! world res_sink (Key) flatten '('('type)))) +(let world (Write! world res_sink (Key) (Member flatten 'first_a) '('('type)))) +(let add (AddMember flatten 'c (String 'x))) +(let world (Write! world res_sink (Key) add '('('type)))) +(let world (Write! world res_sink (Key) (Member add 'second_b) '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MemberOverJustOrNothingWithNulls.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MemberOverJustOrNothingWithNulls.yql new file mode 100644 index 0000000000..af913733b0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MemberOverJustOrNothingWithNulls.yql @@ -0,0 +1,13 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let myStruct (AsStruct '('a (String 'z)) '('b (Null)))) +(let typeOfMyStruct (TypeOf myStruct)) + +(let world (Write! world res_sink (Key) (Member (Just myStruct) 'b) '('('type)))) +(let world (Write! world res_sink (Key) (Member (Nothing (OptionalType typeOfMyStruct)) 'b) '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverConstSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverConstSort.yql new file mode 100644 index 0000000000..94385fb383 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverConstSort.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Sort list1 (Bool 'true) (lambda '(row) (String 'const_key)))) +(let list1 (ExtractMembers list1 '('c))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverExtend.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverExtend.yql new file mode 100644 index 0000000000..48bd5bd276 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverExtend.yql @@ -0,0 +1,17 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(import id_module '"/lib/yql/id.yql") +(let id (bind id_module 'id)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (Apply id (AsList str1 str2))) +(let list1 (Extend list1 list1)) +(let list1 (ExtractMembers list1 '('a))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverMapNext.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverMapNext.yql new file mode 100644 index 0000000000..72a41ec35c --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverMapNext.yql @@ -0,0 +1,23 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Iterator table1)) +(let table1 (MapNext table1 (lambda '(current next) (AsStruct + '('ks '((Member current 'key) (Member next 'subkey))) + '('sv '((Member current 'subkey) (Member next 'value))) + )))) + +(let table1 (ForwardList table1)) + +(let table1 (OrderedMap table1 (lambda '(row) (AsStruct + '('sv (Member row 'sv)) + )))) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) table1 '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSkip.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSkip.yql new file mode 100644 index 0000000000..5345f0b143 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSkip.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Skip list1 (Uint64 '1))) +(let list1 (ExtractMembers list1 '('a))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort.yql new file mode 100644 index 0000000000..46aec084d2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Sort list1 (Bool 'true) (lambda '(row) (Member row 'a)))) +(let list1 (ExtractMembers list1 '('a))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort1.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort1.yql new file mode 100644 index 0000000000..a7c9bb7e78 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort1.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Sort list1 (Bool 'true) (lambda '(row) (Member row 'b)))) +(let list1 (ExtractMembers list1 '('b))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort2.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort2.yql new file mode 100644 index 0000000000..d3988b8ebb --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverSort2.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Sort list1 (Bool 'true) (lambda '(row) (Member row 'b)))) +(let list1 (ExtractMembers list1 '('b 'c))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverTake.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverTake.yql new file mode 100644 index 0000000000..713a57a0fd --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MoveExtractMembersOverTake.yql @@ -0,0 +1,14 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let str1 (AsStruct '('a (Uint32 '1)) '('b (String 'a)) '('c (Bool 'true)))) +(let str2 (AsStruct '('a (Uint32 '2)) '('b (String 'b)) '('c (Bool 'false)))) +(let list1 (AsList str1 str2)) +(let list1 (Take list1 (Uint64 '1))) +(let list1 (ExtractMembers list1 '('a))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) list1 '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.cfg new file mode 100644 index 0000000000..64df516e6f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.cfg @@ -0,0 +1,3 @@ +in Input1 input1.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.sql new file mode 100644 index 0000000000..3e7b24508a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.sql @@ -0,0 +1,13 @@ +$aggregated = ( + SELECT Group, Name, SUM(Amount) AS TotalAmount + FROM plato.Input1 + GROUP BY Group, Name +); + +SELECT t.Comment, a.TotalAmount +FROM plato.Input1 AS t +INNER JOIN $aggregated AS a +ON t.Group == a.Group AND t.Name == a.Name +ORDER BY t.Comment, a.TotalAmount; + +SELECT TotalAmount FROM $aggregated ORDER BY TotalAmount; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.yql new file mode 100644 index 0000000000..c744fafa76 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/MultiExtractMembersOverReadTable.yql @@ -0,0 +1,83 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(import core_module '"/lib/yql/core.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input1")))) (Void) '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let output (block '( + (let select (block '( + (let core (EquiJoin '(table1 '"t") '((block '( + (let select (block '( + (let core table1) + (let core (Aggregate core '('"Group" '"Name") '('('Sum0 (Apply (bind aggregate_module '"sum_traits_factory") (TypeOf core) (lambda '(row) (Member row '"Amount"))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"Group" (Member row '"Group")) '('"Name" (Member row '"Name")) '('"TotalAmount" (Member row 'Sum0)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) '"a") '('Inner '"t" '"a" '('"t" '"Group" '"t" '"Name") '('"a" '"Group" '"a" '"Name") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"t.Comment" (SqlColumn row '"Comment" '"t")) '('"a.TotalAmount" (SqlColumn row '"TotalAmount" '"a")))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (let select (Sort select '((Bool 'true) (Bool 'true)) (lambda '(row) '((Member row '"t.Comment") (Member row '"a.TotalAmount"))))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"t.Comment" '"a.TotalAmount"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input1")))) (Void) '())) + (let world (Left! x)) + (let table1 (Right! x)) + (let output (block '( + (let select (block '( + (let core (block '( + (let select (block '( + (let core table1) + (let core (Aggregate core '('"Group" '"Name") '('('Sum0 (Apply (bind aggregate_module '"sum_traits_factory") (TypeOf core) (lambda '(row) (Member row '"Amount"))))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"Group" (Member row '"Group")) '('"Name" (Member row '"Name")) '('"TotalAmount" (Member row 'Sum0)))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"TotalAmount" (Member row '"TotalAmount")))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (let select (Sort select (Bool 'true) (lambda '(row) (Member row '"TotalAmount")))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"TotalAmount"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +)
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/NonEquivNestedLambdaOuterSameLevel.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/NonEquivNestedLambdaOuterSameLevel.yql new file mode 100644 index 0000000000..61f9307f1a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/NonEquivNestedLambdaOuterSameLevel.yql @@ -0,0 +1,28 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let outLambda1 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'b))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let outLambda2 (lambda '(sep) (block '( + (let data (AsList (String 'a) (String 'c))) + (let data1 (Map data (lambda '(x) (Concat x sep)))) + (let res data1) + (return res) +)))) + +(let predicate (== (Int32 '1) (Int32 '2))) +(let sep (If predicate (Just (String '_)) (Just (String '+)))) +(let data1 (IfPresent sep outLambda1 (AsList (String 'none)))) +(let data2 (IfPresent sep outLambda2 (AsList (String 'none)))) +(let world (Write! world res_sink (Key) '(data1 data2) '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg new file mode 100644 index 0000000000..bc0dfe8e18 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.cfg @@ -0,0 +1,4 @@ +in Input SomeKeys.txt +out Output output.txt +res result.txt +udf python2_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql new file mode 100644 index 0000000000..a245de4f18 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionByWithInnerSort.yql @@ -0,0 +1,72 @@ +( + # prepare python udf + (let str (DataType 'String)) + (let tupleType (TupleType str str)) + (let udfType (CallableType '() '(tupleType) '(str) '(str))) + (let udfScript (String '@@ +def MyTuple(one, two): + return (one, two) + +@@)) + (let udf (ScriptUdf 'Python 'MyTuple udfType udfScript)) + + (let mr_source (DataSource 'yt 'plato)) + (let x (Read! world mr_source (Key '('table (String 'Input))) (Void) '())) + (let world (Left! x)) + (let table1 (Right! x)) + + (let singleKeySelector (lambda '(x) (Member x 'key))) + (let multiKeySelector (lambda '(x) '((Member x 'key) (Member x 'subkey)))) + (let tupleKeySelector (lambda '(x) (Apply udf (Member x 'key) (Member x 'subkey)))) + (let calcKeySelector (lambda '(x) (Concat (Member x 'key) (Member x 'subkey)))) + + (let singleDirAsc (Bool 'true)) + (let singleDirDesc (Bool 'false)) + (let multiDirAsc '((Bool 'true) (Bool 'true))) + (let multiDirDesc '((Bool 'false) (Bool 'false))) + + (let singleSortKeySelector (lambda '(item) (Member item 'subkey))) + (let multiSortKeySelector (lambda '(item) '((Member item 'subkey) (Member item 'value)))) + (let tupleSortKeySelector (lambda '(item) (Apply udf (Member item 'subkey) (Member item 'value)))) + (let calcSortKeySelector (lambda '(item) (Concat (Member item 'subkey) (Member item 'value)))) + + (let res_sink (DataSink 'result)) + + (let macro (lambda '(world keySelector sortDir sortKeySelector) (block '( + (let res (PartitionByKey table1 keySelector sortDir sortKeySelector (lambda '(stream) (FlatMap stream (lambda '(pair) (Nth pair '1)))))) + (let res (Sort res '((Bool 'true) (Bool 'true)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let world (Write! world res_sink (Key) res '())) + (return world) + )))) + + (let world (Apply macro world singleKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world singleKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world singleKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world singleKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world singleKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world singleKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Apply macro world multiKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world multiKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world multiKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world multiKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world multiKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world multiKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Apply macro world tupleKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world tupleKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world tupleKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world tupleKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world tupleKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world tupleKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Apply macro world calcKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world calcKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world calcKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world calcKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world calcKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world calcKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Commit! world res_sink)) + (return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg new file mode 100644 index 0000000000..bc0dfe8e18 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.cfg @@ -0,0 +1,4 @@ +in Input SomeKeys.txt +out Output output.txt +res result.txt +udf python2_udf diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.yql new file mode 100644 index 0000000000..dda6bd8d62 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PartitionsByWithInnerSort.yql @@ -0,0 +1,72 @@ +( + # prepare python udf + (let str (DataType 'String)) + (let tupleType (TupleType str str)) + (let udfType (CallableType '() '(tupleType) '(str) '(str))) + (let udfScript (String '@@ +def MyTuple(one, two): + return (one, two) + +@@)) + (let udf (ScriptUdf 'Python3 'MyTuple udfType udfScript)) + + (let mr_source (DataSource 'yt 'plato)) + (let x (Read! world mr_source (Key '('table (String 'Input))) (Void) '())) + (let world (Left! x)) + (let table1 (Right! x)) + + (let singleKeySelector (lambda '(x) (Member x 'key))) + (let multiKeySelector (lambda '(x) '((Member x 'key) (Member x 'subkey)))) + (let tupleKeySelector (lambda '(x) (Apply udf (Member x 'key) (Member x 'subkey)))) + (let calcKeySelector (lambda '(x) (Concat (Member x 'key) (Member x 'subkey)))) + + (let singleDirAsc (Bool 'true)) + (let singleDirDesc (Bool 'false)) + (let multiDirAsc '((Bool 'true) (Bool 'true))) + (let multiDirDesc '((Bool 'false) (Bool 'false))) + + (let singleSortKeySelector (lambda '(item) (Member item 'subkey))) + (let multiSortKeySelector (lambda '(item) '((Member item 'subkey) (Member item 'value)))) + (let tupleSortKeySelector (lambda '(item) (Apply udf (Member item 'subkey) (Member item 'value)))) + (let calcSortKeySelector (lambda '(item) (Concat (Member item 'subkey) (Member item 'value)))) + + (let res_sink (DataSink 'result)) + + (let macro (lambda '(world keySelector sortDir sortKeySelector) (block '( + (let res (PartitionsByKeys table1 keySelector sortDir sortKeySelector (lambda '(stream) stream))) + (let res (Sort res '((Bool 'true) (Bool 'true)) (lambda '(item) '((Member item 'key) (Member item 'subkey))))) + (let world (Write! world res_sink (Key) res '())) + (return world) + )))) + + (let world (Apply macro world singleKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world singleKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world singleKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world singleKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world singleKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world singleKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Apply macro world multiKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world multiKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world multiKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world multiKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world multiKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world multiKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Apply macro world tupleKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world tupleKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world tupleKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world tupleKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world tupleKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world tupleKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Apply macro world calcKeySelector singleDirAsc singleSortKeySelector)) + (let world (Apply macro world calcKeySelector singleDirDesc singleSortKeySelector)) + (let world (Apply macro world calcKeySelector multiDirAsc multiSortKeySelector)) + (let world (Apply macro world calcKeySelector multiDirDesc multiSortKeySelector)) + (let world (Apply macro world calcKeySelector multiDirAsc tupleSortKeySelector)) + (let world (Apply macro world calcKeySelector singleDirAsc calcSortKeySelector)) + + (let world (Commit! world res_sink)) + (return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PullLimitAfterFilter.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/PullLimitAfterFilter.yql new file mode 100644 index 0000000000..262c788d71 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PullLimitAfterFilter.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +#(let table1 (AsList (AsStruct '('key (String '100))))) +(let table2 (Filter table1 (lambda '(x) (< (Member x 'key) (String '800))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2 '('('take '2)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PullLimitAfterFilterRef.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/PullLimitAfterFilterRef.yql new file mode 100644 index 0000000000..a96ce7edcf --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PullLimitAfterFilterRef.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +#(let table1 (AsList (AsStruct '('key (String '100))))) +(let table2 (Filter table1 (lambda '(x) (< (Member x 'key) (String '800))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2 '('('take '2) '('autoref)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/PushDownToFlow.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/PushDownToFlow.yql new file mode 100644 index 0000000000..cf614d25ca --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/PushDownToFlow.yql @@ -0,0 +1,17 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'x) (String 'c) (String 'e) (String 'a) (String 'z) (String 'y) (String 'f))) +(let data (Reverse data)) +(let data (Skip data (Uint64 '2))) +(let data (Take data (Uint64 '3))) +(let data (OrderedMap data (lambda '(x) '(x x)))) +(let data (ToFlow data)) +(let data (Collect data)) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/ReverseAfterSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/ReverseAfterSort.yql new file mode 100644 index 0000000000..f4ca885270 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/ReverseAfterSort.yql @@ -0,0 +1,17 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'c) (String 'e) (String 'a))) +(let data (Sort data (Bool 'true) (lambda '(x) x))) +(let data (Reverse data)) +(let world (Write! world res_sink (Key) data '())) +(let data (AsList '((Uint32 '1) (String 'c)) '((Uint32 '1) (String 's)) '((Uint32 '2) (String 'a)))) +(let data (Sort data '((Bool 'true) (Bool 'false)) (lambda '(x) x))) +(let data (Reverse data)) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelectMembersOverOptStruct.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SelectMembersOverOptStruct.yql new file mode 100644 index 0000000000..b9e3b2373c --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelectMembersOverOptStruct.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Map ("/" (Int32 '42) (Int32 '1)) (lambda '(item) (AsStruct + '('magic (Just item)) + '('black_magic ("/" (Int32 '100) item)) + )))) + +(let table1 (SelectMembers table1 '('m))) + +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink (Key) table1 '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.cfg new file mode 100644 index 0000000000..e6d9edcc5d --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.cfg @@ -0,0 +1,2 @@ +in Input InputUniq.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.sql new file mode 100644 index 0000000000..127c25a4d5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.sql @@ -0,0 +1,5 @@ +USE plato; + +$a = (SELECT key,subkey || "a" as subkey from Input where value < "r"); +$b = (SELECT key,subkey || "b" as subkey from Input where value > "c"); +select a.key as k1,a.subkey,b.subkey from $a as a join $b as b using (key); diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.yql new file mode 100644 index 0000000000..3770df6de5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersect.yql @@ -0,0 +1,57 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) '('"value" '"key" '"subkey") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (EquiJoin '((block '( + (let select (block '( + (let core table0) + (let core (Filter core (lambda '(row) (Coalesce ("<" (Member row '"value") (String '"r")) (Bool 'false))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"key" (Member row '"key")) '('"subkey" ("Concat" (Member row '"subkey") (String '"a"))))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) '"a") '((block '( + (let select (block '( + (let core table0) + (let core (Filter core (lambda '(row) (Coalesce (">" (Member row '"value") (String '"c")) (Bool 'false))))) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"key" (Member row '"key")) '('"subkey" ("Concat" (Member row '"subkey") (String '"b"))))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) '"b") '('Inner '"a" '"b" '('"a" '"key") '('"b" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"k1" (SqlColumn row '"key" '"a")) '('"a.subkey" (SqlColumn row '"subkey" '"a")) '('"b.subkey" (SqlColumn row '"subkey" '"b")))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"k1" '"a.subkey" '"b.subkey"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.cfg new file mode 100644 index 0000000000..e6d9edcc5d --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.cfg @@ -0,0 +1,2 @@ +in Input InputUniq.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.sql b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.sql new file mode 100644 index 0000000000..944f3c386f --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.sql @@ -0,0 +1,5 @@ +USE plato; + +$a = (SELECT key,subkey || "a" as subkey from Input where value < "r"); +$b = (SELECT key,subkey || "b" as subkey from Input); +select a.key as k1,a.subkey,b.subkey from Input as a join $b as b using (key); diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.yql new file mode 100644 index 0000000000..45e75bf7d7 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SelfJoinIntersectTab1.yql @@ -0,0 +1,44 @@ +( +(import aggregate_module '"/lib/yql/aggregate.yql") +(import window_module '"/lib/yql/window.yql") +(let world (block '( + (let x (Read! world (DataSource '"yt" '"plato") (MrTableConcat (Key '('table (String '"Input")))) '('"key" '"subkey") '())) + (let world (Left! x)) + (let table0 (Right! x)) + (let output (block '( + (let select (block '( + (let core (EquiJoin '(table0 '"a") '((block '( + (let select (block '( + (let core table0) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"key" (Member row '"key")) '('"subkey" ("Concat" (Member row '"subkey") (String '"b"))))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + )) '"b") '('Inner '"a" '"b" '('"a" '"key") '('"b" '"key") '()) '())) + (let core (FlatMap core (lambda '(row) (block '( + (let res (AsStruct '('"k1" (SqlColumn row '"key" '"a")) '('"a.subkey" (SqlColumn row '"subkey" '"a")) '('"b.subkey" (SqlColumn row '"subkey" '"b")))) + (let res (AsList res)) + (return res) + ))))) + (return core) + ))) + (return select) + ))) + (let world (block '( + (let result_sink (DataSink 'result)) + (let world (Write! world result_sink (Key) output '('('type) '('autoref) '('columns '('"k1" '"a.subkey" '"b.subkey"))))) + (return (Commit! world result_sink)) + ))) + (return world) +))) +(let world (block '( + (let plato_sink (DataSink '"yt" '"plato")) + (let world (Commit! world plato_sink)) + (return world) +))) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipAfterMap.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipAfterMap.yql new file mode 100644 index 0000000000..fbc9dc1ba4 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipAfterMap.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Map table1 (lambda '(x) (AddMember x 'foo (String 'qwe))))) +(let table1 (Skip table1 (Uint64 '2))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table1 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipOverExtend.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipOverExtend.yql new file mode 100644 index 0000000000..3ec439ddde --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipOverExtend.yql @@ -0,0 +1,18 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list1 (AsList (String 'a) (String 'b) (String 'c))) +(let list2 (AsList (String 'd) (String 'e) (String 'f))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + (Skip (Extend list1 list2) (Uint64 '1)) '())) + +(let world (Write! world res_sink + (Key) + (Skip (Extend list1 list2) (Uint64 '5)) '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContent.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContent.cfg new file mode 100644 index 0000000000..bf54cfd5be --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContent.cfg @@ -0,0 +1,2 @@ +in Input input_tbl_content.txt +out Output output.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContent.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContent.yql new file mode 100644 index 0000000000..d51db8e508 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContent.yql @@ -0,0 +1,17 @@ +( + #comment + (let mr_source (DataSource 'yt 'plato)) + (let r (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))) + (let table1 (Right! r)) + (let table1 (Skip table1 (Uint64 '5))) + (let table1 (Take table1 (Uint64 '3))) + (let tresh (String '300)) + (let table1low (FlatMap table1 (lambda '(item) (block '( + (let intValueOpt (Member item 'key)) + (return (ListIf (< intValueOpt tresh) item)) + ))))) + (let mr_sink (DataSink 'yt 'plato)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) table1low '('('mode 'append)))) + (let world (Commit! world mr_sink)) + (return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContentInsideMap.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContentInsideMap.cfg new file mode 100644 index 0000000000..bf54cfd5be --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContentInsideMap.cfg @@ -0,0 +1,2 @@ +in Input input_tbl_content.txt +out Output output.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContentInsideMap.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContentInsideMap.yql new file mode 100644 index 0000000000..5bfc63445e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipTakeOverMrTableContentInsideMap.yql @@ -0,0 +1,27 @@ +( + #comment + (let mr_source (DataSource 'yt 'plato)) + (let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) + (let world (Left! x)) + (let table1 (Right! x)) + + (let r (Read! world mr_source (Key '('table (String 'Input))) (Void) '('('inline)))) + (let mrtc (Right! r)) + (let mrtc (Skip mrtc (Uint64 '5))) + (let mrtc (Take mrtc (Uint64 '3))) + (let mrtc (FlatMap mrtc (lambda '(item) (block '( + (let key (Member item 'key)) + (return (ListIf (> key (String '100)) item)) + ))))) + (let filter (ToDict mrtc (lambda '(x) (Member x 'key)) (lambda '(x) (Member x 'value)) '('Hashed 'One))) + + (let tresh (String '500)) + (let out (FlatMap table1 (lambda '(item) (block '( + (let key (Member item 'key)) + (return (ListIf (And (< key tresh) (Contains filter key)) item)) + ))))) + (let mr_sink (DataSink 'yt 'plato)) + (let world (Write! world mr_sink (Key '('table (String 'Output))) out '('('mode 'append)))) + (let world (Commit! world mr_sink)) + (return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SkipWhileOverLiterals.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipWhileOverLiterals.yql new file mode 100644 index 0000000000..e661d70dde --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SkipWhileOverLiterals.yql @@ -0,0 +1,44 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) + +(let listVal (AsList (Int32 '0))) +(let listVal (Append listVal (Int32 '1))) + +(let emptyList (List (ListType (DataType 'Int32)))) +(let emptyStream (EmptyIterator (StreamType (DataType 'Int32)))) +(let emptyOptional (Nothing (OptionalType (DataType 'Int32)))) + +(let notOptLambda (lambda '(x) (< x (Int32 '1)))) + +# Skip while true: [] +(let res (SkipWhile listVal (lambda '(x) (Bool 'true)))) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Skip while false: [0, 1] +(let res (SkipWhile listVal (lambda '(x) (Bool 'false)))) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Skip from empty list: [] +(let res (SkipWhile emptyList notOptLambda)) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Skip from empty stream: [] +(let res (SkipWhile emptyStream notOptLambda)) +(let res (Collect res)) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Skip from empty optional: [] +(let res (SkipWhile emptyOptional notOptLambda)) +(let world (Write! world res_sink (Key) res '('('type)))) + +# General case: [1] +(let res (SkipWhile listVal notOptLambda)) +(let world (Write! world res_sink (Key) res '('('type)))) + +(let world (Commit! world res_sink)) +(return world) + +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SomeKeys.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/SomeKeys.txt new file mode 100644 index 0000000000..6d0b5b4d40 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SomeKeys.txt @@ -0,0 +1,7 @@ +{"key"="075";"subkey"="1";"value"="abc"}; +{"key"="075";"subkey"="2";"value"="abc2"}; +{"key"="075";"subkey"="3";"value"="abc3"}; +{"key"="800";"subkey"="4";"value"="ddd"}; +{"key"="020";"subkey"="5";"value"="q"}; +{"key"="150";"subkey"="6";"value"="qzz"}; +{"key"="150";"subkey"="7";"value"="zz"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SomeKeys.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/SomeKeys.txt.attr new file mode 100644 index 0000000000..b6100e5fd0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SomeKeys.txt.attr @@ -0,0 +1,30 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SortAfterReverse.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SortAfterReverse.yql new file mode 100644 index 0000000000..21570ead7b --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SortAfterReverse.yql @@ -0,0 +1,13 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'c) (String 'e) (String 'a))) +(let data (Reverse data)) +(let data (Sort data (Bool 'true) (lambda '(x) x))) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SortOfSortedDesc.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SortOfSortedDesc.yql new file mode 100644 index 0000000000..3fbb12f446 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SortOfSortedDesc.yql @@ -0,0 +1,24 @@ +( +(let source (DataSource 'yt 'plato)) +(let sink (DataSink 'yt 'plato)) +(let result (DataSink 'result)) + +(let r (Read! world source (MrTableConcat (Key '('table (String '"Input")))) (Void) '())) +(let input (Right! r)) +(let world (Left! r)) + +(let out (Sort (FlatMap input (lambda '(item) (AsList item))) (Bool 'false) (lambda '(item) (Member item '"key")))) +(let world (Write! world sink (Key '('table (String '"Output"))) out '('('mode 'renew)))) +(let world (Commit! world sink)) + +(let r (Read! world source (MrTableConcat (Key '('table (String '"Output")))) (Void) '())) +(let input (Right! r)) +(let world (Left! r)) + +(let out (Sort (FlatMap input (lambda '(item) (AsList item))) (Bool 'true) (lambda '(item) (Member item '"key")))) +(let world (Write! world result (Key) out '('('type) '('autoref)))) +(let world (Commit! world result)) +(let world (Commit! world sink)) + +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SubsetOfFields.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SubsetOfFields.yql new file mode 100644 index 0000000000..f4d4e775d8 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SubsetOfFields.yql @@ -0,0 +1,32 @@ +( +#comment +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '())) +(let world (Left! x)) +(let table1 (Right! x)) +(let table1map (Map table1 (lambda '(item) (block '( + (let item (RemoveMember item 'value)) + (let s (Struct)) + (let s (AddMember s 'key (Member item 'key))) + (let s (AddMember s 'subkey (String '"s1"))) + (let s (AddMember s 'value (String '"v1"))) + (return s) +))))) +(let mr_sink (DataSink 'yt (quote plato))) +(let world (Write! world mr_sink (Key '('table (String 'Output))) table1map '('('mode 'append)))) +(let world (Commit! world mr_sink)) +(let x (Read! world mr_source (Key '('table (String 'Input))) '('key 'subkey 'value) '('('inline)))) +(let table1 (Right! x)) +(let table1map (Map table1 (lambda '(item) (block '( + (let item (RemoveMember item 'value)) + (let s (Struct)) + (let s (AddMember s 'key (Member item 'key))) + (let s (AddMember s 'subkey (String '"s2"))) + (let s (AddMember s 'value (String '"v2"))) + (return s) +))))) +(let mr_sink (DataSink 'yt (quote plato))) +(let world (Write! world mr_sink (Key '('table (String 'Output))) table1map '('('mode 'append)))) +(let world (Commit! world mr_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/SwitchToVisit.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/SwitchToVisit.yql new file mode 100644 index 0000000000..b703492435 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/SwitchToVisit.yql @@ -0,0 +1,73 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) +(let res_sink (DataSink 'result)) + +(let list (AsList + (AsStruct '('key (String 'aaa)) '('value (String '0))) + (AsStruct '('key (String 'bbb)) '('value (String '1))) + (AsStruct '('key (String 'ccc)) '('value (String '2))) +)) + +(let structType (StructType '('key (DataType 'String)) '('value (DataType 'String)))) +(let tupleType (TupleType structType structType structType)) +(let vt (VariantType tupleType)) +(let vlist (AsList + (Variant (AsStruct '('key (String 'aaa)) '('value (String '0))) '0 vt) + (Variant (AsStruct '('key (String 'bbb)) '('value (String '1))) '1 vt) + (Variant (AsStruct '('key (String 'ccc)) '('value (String '2))) '2 vt) +)) + +(let trivialLambda (lambda '(x) x)) +(let justHandler1 (lambda '(x) (FlatMap x (lambda '(s) (Just (AsStruct '('key (Concat (String '"#0: ") (Member s 'key))) '('value (Member s 'value)))))))) +(let justHandler2 (lambda '(x) (FlatMap x (lambda '(s) (Just (AsStruct '('key (Concat (String '"#1: ") (Member s 'key))) '('value (Member s 'value)))))))) +(let justHandler3 (lambda '(x) (FlatMap x (lambda '(s) (Just (AsStruct '('key (Concat (String '"#3: ") (Member s 'key))) '('value (Member s 'value)))))))) +(let extractMembersHandler (lambda '(x) (FlatMap (ExtractMembers x '('key)) (lambda '(s) (Just (AsStruct '('key (Concat (String '"#3: ") (Member s 'key))))))))) +(let listHandler (lambda '(x) (FlatMap x (lambda '(s) (AsList s s))))) +(let streamHandler (lambda '(x) (FlatMap x (lambda '(s) (Iterator (AsList s s) (DependsOn s) (DependsOn (String '0))))))) +(let justVarHandler1 (lambda '(x) (FlatMap x (lambda '(s) (Just (Variant s '1 vt)))))) +(let justVarHandler2 (lambda '(x) (FlatMap x (lambda '(s) (Just (Variant s '2 vt)))))) + +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'A1))) '1 '('1) trivialLambda)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'A2))) '1 '('0) justHandler1)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'A3))) '1 '('0) listHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'A4))) '1 '('0) streamHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'A5))) '1 '('0) justVarHandler1)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'A6))) '1 '('0) extractMembersHandler)) '('('type)))) + +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B1))) '1 '('0) justHandler1 '('1) justHandler2)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B2))) '1 '('0) justHandler1 '('1) justHandler2 '('2) justHandler3)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B3))) '1 '('0) justHandler1 '('1) trivialLambda)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B4))) '1 '('0) justHandler1 '('1) extractMembersHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B5))) '1 '('0) justHandler1 '('1) justHandler2 '('1) justHandler3 '('2) justHandler3)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B6))) '1 '('0) justHandler1 '('1) justHandler2 '('1) justHandler3)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B7))) '1 '('0) justHandler1 '('0) justHandler2)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B8))) '1 '('0) justHandler1 '('1) justHandler2 '('1) streamHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B9))) '1 '('0) justHandler1 '('1) extractMembersHandler '('1) streamHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B10))) '1 '('0) justHandler1 '('1) listHandler '('1) streamHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B11))) '1 '('0) justHandler1 '('1) listHandler '('1) trivialLambda)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'B12))) '1 '('0) justHandler1 '('1) trivialLambda '('1) trivialLambda)) '('('type)))) + +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'C1))) '1 '('0) justHandler1 '('1) listHandler '('2) trivialLambda)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'C2))) '1 '('0) streamHandler '('1) listHandler '('2) trivialLambda)) '('('type)))) + +# Shouldn't be optimized +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'D1))) '1 '('0) justVarHandler1 '('1) justVarHandler2)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator vlist (DependsOn (String 'D2))) '1 '('0) justHandler1 '('1) justVarHandler2)) '('('type)))) + +# Replicating Switch +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'E1))) '1 '('0) trivialLambda)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'E2))) '1 '('0) justVarHandler1)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'E3))) '1 '('0) trivialLambda '('0) justHandler1 '('0) justHandler2)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'E4))) '1 '('0) justHandler1 '('0) extractMembersHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'E5))) '1 '('0) trivialLambda '('0) justHandler1 '('0) listHandler)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'E6))) '1 '('0) streamHandler '('0) listHandler)) '('('type)))) + +# Shouldn't be optimized +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'F1))) '1 '('0) justVarHandler1 '('0) justVarHandler2)) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Switch (Iterator list (DependsOn (String 'F2))) '1 '('0) justHandler1 '('0) justVarHandler2)) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeAfterMap.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeAfterMap.yql new file mode 100644 index 0000000000..41df996fee --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeAfterMap.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Map table1 (lambda '(x) (AddMember x 'foo (String 'qwe))))) +(let table1 (Take table1 (Uint64 '2))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table1 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterCombine.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterCombine.cfg new file mode 100644 index 0000000000..80d5638bff --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterCombine.cfg @@ -0,0 +1,3 @@ +in Input SomeKeys.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterCombine.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterCombine.yql new file mode 100644 index 0000000000..b263c72063 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterCombine.yql @@ -0,0 +1,30 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let preMap (lambda '(item) (Just item))) +(let keyExtractor (lambda '(item) (Member item 'key))) +(let init (lambda '(key item) (FromString (Member item 'subkey) 'Uint32))) +(let update (lambda '(key item state) (+ state (FromString (Member item 'subkey) 'Uint32)))) +(let finish (lambda '(key state) (block '( + (let s (Struct)) + (let s (AddMember s 'key key)) + (let s (AddMember s 'subkey (String '.))) + (let s (AddMember s 'value (Coalesce (Map state (lambda '(x) (ToString x))) (String '"")))) + (return (Just s)) +)))) + +(let table2 (CombineByKey table1 preMap keyExtractor init update finish)) +(let table2 (Take table2 (Uint64 '2))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterFilter.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterFilter.yql new file mode 100644 index 0000000000..e161a11ee5 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterFilter.yql @@ -0,0 +1,23 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Filter table1 (lambda '(x) (< (Member x 'key) (String '800))))) +(let table2x (Skip (Take table1 (Uint64 '2)) (Uint64 '1))) +(let table2y (Take table1 (Uint64 '3))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2x '())) + +(let world (Write! world res_sink + (Key) + table2y '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterGroup.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterGroup.cfg new file mode 100644 index 0000000000..80d5638bff --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterGroup.cfg @@ -0,0 +1,3 @@ +in Input SomeKeys.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterGroup.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterGroup.yql new file mode 100644 index 0000000000..d0fa6fb1dd --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterGroup.yql @@ -0,0 +1,34 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let keyExtractor (lambda '(item) (Member item 'key))) +(let handler (lambda '(stream) (FlatMap stream (lambda '(pair) (block '( + (let key (Nth pair '0)) + (let list (ForwardList (Nth pair '1))) + (let s (Struct)) + (let s (AddMember s 'key key)) + (let s (AddMember s 'subkey (String '.))) + (let f (Fold list (Uint64 '0) (lambda '(item state) (block '( + (let value (Coalesce (FromString (Member item 'subkey) 'Uint64) (Uint64 '0))) + (return (+ value state)) + ))))) + (let s (AddMember s 'value (ToString f))) + (return (Just s)) +)))))) + +(let table2 (PartitionByKey table1 keyExtractor (Void) (Void) handler)) +(let table2 (Take table2 (Uint64 '4))) +(let table2 (Sort table2 (Bool 'true) (lambda '(item) (Member item 'value)))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterSort.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterSort.yql new file mode 100644 index 0000000000..107a53b598 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeLimitAfterSort.yql @@ -0,0 +1,19 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let keyExtractor (lambda '(item) (Member item 'key))) +(let table2 (Sort table1 (Bool 'true) keyExtractor)) +(let table2 (Take table2 (Uint64 '2))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table2 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeOverExtend.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeOverExtend.yql new file mode 100644 index 0000000000..e7b12fc1c1 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeOverExtend.yql @@ -0,0 +1,18 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let list1 (AsList (String 'a) (String 'b) (String 'c))) +(let list2 (AsList (String 'd) (String 'e) (String 'f))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + (Take (Extend list1 list2) (Uint64 '1)) '())) + +(let world (Write! world res_sink + (Key) + (Take (Extend list1 list2) (Uint64 '5)) '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeOverTop.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeOverTop.yql new file mode 100644 index 0000000000..ad245012d0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeOverTop.yql @@ -0,0 +1,14 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let data (AsList (String 'x) (String 'c) (String 'e) (String 'a) (String 'z) (String 'y) (String 'f))) +(let data (Top data (Uint64 '4) (Bool 'true) (lambda '(x) x))) +(let data (Take data (Uint64 '2))) +(let world (Write! world res_sink (Key) data '())) +(let world (Commit! world res_sink)) +(return world) +) + diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/TakeWhileOverLiterals.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeWhileOverLiterals.yql new file mode 100644 index 0000000000..bc2bdb9037 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/TakeWhileOverLiterals.yql @@ -0,0 +1,44 @@ +( +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) + +(let listVal (AsList (Int32 '0))) +(let listVal (Append listVal (Int32 '1))) + +(let emptyList (List (ListType (DataType 'Int32)))) +(let emptyStream (EmptyIterator (StreamType (DataType 'Int32)))) +(let emptyOptional (Nothing (OptionalType (DataType 'Int32)))) + +(let notOptLambda (lambda '(x) (< x (Int32 '1)))) + +# Take while true: [0, 1] +(let res (TakeWhile listVal (lambda '(x) (Bool 'true)))) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Take while false: [] +(let res (TakeWhile listVal (lambda '(x) (Bool 'false)))) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Take from empty list: [] +(let res (TakeWhile emptyList notOptLambda)) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Take from empty stream: [] +(let res (TakeWhile emptyStream notOptLambda)) +(let res (Collect res)) +(let world (Write! world res_sink (Key) res '('('type)))) + +# Take from empty optional: [] +(let res (TakeWhile emptyOptional notOptLambda)) +(let world (Write! world res_sink (Key) res '('('type)))) + +# General case: [0] +(let res (TakeWhile listVal notOptLambda)) +(let world (Write! world res_sink (Key) res '('('type)))) + +(let world (Commit! world res_sink)) +(return world) + +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Top.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Top.yql new file mode 100644 index 0000000000..e0a2bab036 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Top.yql @@ -0,0 +1,42 @@ +( +#comment +(let mr_source (DataSource 'yt 'plato)) +(let res_sink (DataSink 'result)) + +(let x (Read! world mr_source (Key '('table (String 'Input))) (Void) '())) +(let world (Left! x)) +(let table (Right! x)) + +(let list (AsList + (AsStruct '('a (Uint32 '4)) '('b (String 'c))) + (AsStruct '('a (Uint32 '1)) '('b (String 'd))) + (AsStruct '('a (Uint32 '3)) '('b (String 'b))) +)) + +# Top->Take +(let world (Write! world res_sink (Key) (Top table (Uint64 '3) '() (lambda '(item) '())) '('('type)))) +(let world (Write! world res_sink (Key) (TopSort table (Uint64 '3) '() (lambda '(item) '())) '('('type)))) + +# tuple with single item +(let world (Write! world res_sink (Key) (Top table (Uint64 '3) '((Bool 'True)) (lambda '(item) '((Member item 'key)))) '('('type)))) +(let world (Write! world res_sink (Key) (TopSort table (Uint64 '3) '((Bool 'True)) (lambda '(item) '((Member item 'key)))) '('('type)))) + +# zero count +(let world (Write! world res_sink (Key) (Top table (Uint64 '0) (Bool 'True) (lambda '(item) (Member item 'key))) '('('type)))) +(let world (Write! world res_sink (Key) (TopSort table (Uint64 '0) (Bool 'True) (lambda '(item) (Member item 'key))) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (Top (Iterator table (DependsOn (String 'a))) (Uint64 '0) (Bool 'True) (lambda '(item) (Member item 'key)))) '('('type)))) +(let world (Write! world res_sink (Key) (Collect (TopSort (Iterator table (DependsOn (String 'b))) (Uint64 '0) (Bool 'True) (lambda '(item) (Member item 'key)))) '('('type)))) + +# list <= N +(let world (Write! world res_sink (Key) (Top list (Uint64 '3) (Bool 'True) (lambda '(item) (Member item 'a))) '('('type)))) +(let world (Write! world res_sink (Key) (TopSort list (Uint64 '3) (Bool 'True) (lambda '(item) (Member item 'a))) '('('type)))) + +# ExtractMembers +(let world (Write! world res_sink (Key) (ExtractMembers (Top table (Uint64 '3) (Bool 'True) (lambda '(item) (Member item 'key))) '('key 'value)) '('('type)))) +(let world (Write! world res_sink (Key) (ExtractMembers (TopSort table (Uint64 '3) (Bool 'True) (lambda '(item) (Member item 'key))) '('key 'value)) '('('type)))) +(let world (Write! world res_sink (Key) (ExtractMembers (Top table (Uint64 '3) (Bool 'True) (lambda '(item) (Member item 'key))) '('value)) '('('type)))) +(let world (Write! world res_sink (Key) (ExtractMembers (TopSort table (Uint64 '3) (Bool 'True) (lambda '(item) (Member item 'key))) '('value)) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/UnionOfSortedDesc.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/UnionOfSortedDesc.yql new file mode 100644 index 0000000000..b1de8fceb1 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/UnionOfSortedDesc.yql @@ -0,0 +1,20 @@ +( +(let source (DataSource 'yt 'plato)) +(let sink (DataSink 'yt 'plato)) + +(let r (Read! world source (MrTableConcat (Key '('table (String '"Input")))) (Void) '())) +(let input (Right! r)) +(let world (Left! r)) + +(let mapLambda (lambda '(item) (AsList (AsStruct '('"key" (Member item '"key")))))) +(let keyExtractLambda (lambda '(item) (Member item '"key"))) + +(let out1 (Sort (FlatMap input mapLambda) (Bool 'false) keyExtractLambda)) +(let out2 (Sort (FlatMap input mapLambda) (Bool 'false) keyExtractLambda)) +(let union (UnionAll (Take out1 (Int32 '"3")) (Take out2 (Int32 '"2")))) + +(let world (Write! world sink (Key '('table (String '"Output"))) union '('('mode 'renew)))) +(let world (Commit! world sink)) + +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/UnionOfSortedMix.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/UnionOfSortedMix.yql new file mode 100644 index 0000000000..cbe8c2bb5e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/UnionOfSortedMix.yql @@ -0,0 +1,20 @@ +( +(let source (DataSource 'yt 'plato)) +(let sink (DataSink 'yt 'plato)) + +(let r (Read! world source (MrTableConcat (Key '('table (String '"Input")))) (Void) '())) +(let input (Right! r)) +(let world (Left! r)) + +(let mapLambda (lambda '(item) (AsList (AsStruct '('"key" (Member item '"key")))))) +(let keyExtractLambda (lambda '(item) (Member item '"key"))) + +(let out1 (Sort (FlatMap input mapLambda) (Bool 'false) keyExtractLambda)) +(let out2 (Sort (FlatMap input mapLambda) (Bool 'true) keyExtractLambda)) +(let union (UnionAll (Take out1 (Int32 '"3")) (Take out2 (Int32 '"2")))) + +(let world (Write! world sink (Key '('table (String '"Output"))) union '('('mode 'renew)))) +(let world (Commit! world sink)) + +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/UnusedColumnsAfterTake.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/UnusedColumnsAfterTake.yql new file mode 100644 index 0000000000..65f43e76db --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/UnusedColumnsAfterTake.yql @@ -0,0 +1,18 @@ +( +(let mr_source (DataSource 'yt 'plato)) +(let x (Read! world mr_source + (Key '('table (String 'Input))) + (Void) '())) +(let world (Left! x)) +(let table1 (Right! x)) + +(let table1 (Take table1 (Uint64 '2))) +(let table1 (Map table1 (lambda '(x) (AsStruct '('key (Member x 'key)))))) +(let res_sink (DataSink 'result)) +(let world (Write! world res_sink + (Key) + table1 '())) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/UseEmptyDict.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/UseEmptyDict.yql new file mode 100644 index 0000000000..6f5b78d34e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/UseEmptyDict.yql @@ -0,0 +1,20 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let ed (Dict (DictType (DataType 'Int32) (DataType 'String)))) +(let data (Lookup ed (Int32 '0))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (Contains ed (Int32 '0))) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (DictItems ed)) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (DictKeys ed)) +(let world (Write! world res_sink (Key) data '('('type)))) +(let data (DictPayloads ed)) +(let world (Write! world res_sink (Key) data '('('type)))) +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/VariantItemOverVisit.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/VariantItemOverVisit.yql new file mode 100644 index 0000000000..76b605c271 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/VariantItemOverVisit.yql @@ -0,0 +1,23 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) +(let res_sink (DataSink 'result)) + +(let vt (VariantType (TupleType (DataType 'String) (DataType 'String) (DataType 'String)))) +(let v0 (Variant (String 'aaa) '0 vt)) +(let v1 (Variant (String 'bbb) '1 vt)) +(let v2 (Variant (String 'ccc) '2 vt)) +(let list (AsList v0 v1 v2)) + +(let handler (lambda '(x) (Concat (String '0-) x))) + +(let remap (lambda '(x) (Visit x '0 (lambda '(x) (Variant x '1 vt)) '1 (lambda '(x) (Variant x '2 vt)) '2 (lambda '(x) (Variant x '0 vt))))) +(let def (lambda '(x) (Visit x '0 (lambda '(x) (Variant (Apply handler x) '0 vt)) v1))) + +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (VariantItem (Apply remap x)))) '('('type)))) +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (VariantItem (Apply def x)))) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/Visit1.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/Visit1.yql new file mode 100644 index 0000000000..8c8c4f7726 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/Visit1.yql @@ -0,0 +1,23 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let vt (VariantType (TupleType (DataType 'Int32)))) +(let v1 (Variant (Int32 '34) '0 vt)) + +(let handler (lambda '(x) (+ (Int32 '100) x))) + +(let world (Write! world res_sink (Key) (Visit v1 '0 handler) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v1 '0 handler (Int32 '999)) '('('type)))) + +(let vt (VariantType (StructType '('a (DataType 'Int32))))) +(let v1 (Variant (Int32 '34) 'a vt)) + +(let world (Write! world res_sink (Key) (Visit v1 'a handler) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v1 'a handler (Int32 '999)) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/VisitAllEqualLambdas.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/VisitAllEqualLambdas.yql new file mode 100644 index 0000000000..5f7b186e37 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/VisitAllEqualLambdas.yql @@ -0,0 +1,21 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let vt (VariantType (TupleType (DataType 'String) (DataType 'String) (DataType 'String)))) +(let v0 (Variant (String 'aaa) '0 vt)) +(let v1 (Variant (String 'bbb) '1 vt)) +(let v2 (Variant (String 'ccc) '2 vt)) +(let list (AsList v0 v1 v2)) + +(let handler0 (lambda '(x) (Concat (String '0-) x))) +(let handler1 (lambda '(x) (Concat (String '0-) x))) +(let handler2 (lambda '(x) (Concat (String '0-) x))) + +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (Visit x '0 handler0 '1 handler1 '2 handler2))) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/VisitAsGuess.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/VisitAsGuess.yql new file mode 100644 index 0000000000..271b065f4a --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/VisitAsGuess.yql @@ -0,0 +1,40 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let vt (VariantType (TupleType (DataType 'Int32) (DataType 'String)))) +(let v1 (Variant (Int32 '34) '0 vt)) +(let v2 (Variant (String 'qwe) '1 vt)) + +(let handler (lambda '(x) (Just x))) +(let listHandler (lambda '(x) (AsList x))) + +(let world (Write! world res_sink (Key) (Visit v1 '0 handler (Nothing (OptionalType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v1 '1 handler (Nothing (OptionalType (DataType 'String)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 '0 handler (Nothing (OptionalType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 '1 handler (Nothing (OptionalType (DataType 'String)))) '('('type)))) + +(let world (Write! world res_sink (Key) (Visit v1 '0 listHandler (List (ListType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v1 '1 listHandler (List (ListType (DataType 'String)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 '0 listHandler (List (ListType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 '1 listHandler (List (ListType (DataType 'String)))) '('('type)))) + +(let vt (VariantType (StructType '('a (DataType 'Int32)) '('b (DataType 'String))))) +(let v1 (Variant (Int32 '34) 'a vt)) +(let v2 (Variant (String 'qwe) 'b vt)) + +(let world (Write! world res_sink (Key) (Visit v1 'a handler (Nothing (OptionalType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v1 'b handler (Nothing (OptionalType (DataType 'String)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 'a handler (Nothing (OptionalType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 'b handler (Nothing (OptionalType (DataType 'String)))) '('('type)))) + +(let world (Write! world res_sink (Key) (Visit v1 'a listHandler (List (ListType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v1 'b listHandler (List (ListType (DataType 'String)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 'a listHandler (List (ListType (DataType 'Int32)))) '('('type)))) +(let world (Write! world res_sink (Key) (Visit v2 'b listHandler (List (ListType (DataType 'String)))) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/VisitOverVisit.yql b/yql/essentials/tests/s-expressions/suites/Optimizers/VisitOverVisit.yql new file mode 100644 index 0000000000..8a0197dd5e --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/VisitOverVisit.yql @@ -0,0 +1,29 @@ +( +#comment +(let config (DataSource 'config)) +(let world (Configure! world config 'PureDataSource 'yt)) + +(let res_sink (DataSink 'result)) +(let vt3 (VariantType (TupleType (DataType 'String) (DataType 'String) (DataType 'String)))) +(let vt2 (VariantType (TupleType (DataType 'String) (DataType 'String)))) +(let v0 (Variant (String 'aaa) '0 vt3)) +(let v1 (Variant (String 'bbb) '1 vt3)) +(let v2 (Variant (String 'ccc) '2 vt3)) +(let list (AsList v0 v1 v2)) + +(let handler0 (lambda '(x) (Concat (String '0-) x))) +(let handler1 (lambda '(x) (Concat (String '1-) x))) +(let handler2 (lambda '(x) (Concat (String '2-) x))) + +(let remap (lambda '(x) (Visit x '0 (lambda '(x) (Variant x '1 vt3)) '1 (lambda '(x) (Variant x '2 vt3)) '2 (lambda '(x) (Variant x '0 vt3))))) +(let def (lambda '(x) (Visit x '0 (lambda '(x) (Variant (Apply handler0 x) '0 vt3)) v1))) +(let reduce (lambda '(x) (Visit x '0 (lambda '(x) (Variant x '0 vt2)) '1 (lambda '(x) (Variant x '0 vt2)) '2 (lambda '(x) (Variant x '1 vt2))))) + +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (Visit (Apply remap x) '0 handler0 '1 handler1 '2 handler2))) '('('type)))) +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (Visit (Apply def x) '0 handler0 '1 handler1 (String 'def)))) '('('type)))) +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (Visit (Apply def x) '0 handler0 (String 'def)))) '('('type)))) +(let world (Write! world res_sink (Key) (Map list (lambda '(x) (Visit (Apply reduce x) '0 handler0 '1 handler1))) '('('type)))) + +(let world (Commit! world res_sink)) +(return world) +) diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/default.cfg b/yql/essentials/tests/s-expressions/suites/Optimizers/default.cfg new file mode 100644 index 0000000000..fe2b19a9ff --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/default.cfg @@ -0,0 +1,3 @@ +in Input input.txt +out Output output.txt +res result.txt diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/input.txt new file mode 100644 index 0000000000..65949ea745 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input.txt @@ -0,0 +1,4 @@ +{"key"="075";"subkey"="1";"value"="abc"}; +{"key"="800";"subkey"="2";"value"="ddd"}; +{"key"="020";"subkey"="3";"value"="q"}; +{"key"="150";"subkey"="4";"value"="qzz"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/input.txt.attr new file mode 100644 index 0000000000..b6100e5fd0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input.txt.attr @@ -0,0 +1,30 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input1.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/input1.txt new file mode 100644 index 0000000000..bd567557a0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input1.txt @@ -0,0 +1,19 @@ +{"Amount"=100;"Comment"="Test1";"Group"=1u;"Name"="Name1"}; +{"Amount"=101;"Comment"="Test1";"Group"=1u;"Name"="Name2"}; +{"Amount"=102;"Comment"="Test1";"Group"=1u;"Name"="Name3"}; +{"Amount"=103;"Comment"="Test1";"Group"=2u;"Name"="Name1"}; +{"Amount"=104;"Comment"="Test1";"Group"=3u;"Name"="Name3"}; +{"Amount"=105;"Comment"="Test1";"Group"=4u;"Name"="Name1"}; +{"Amount"=106;"Comment"="Test1";"Group"=4u;"Name"="Name4"}; +{"Amount"=107;"Comment"="Test1";"Group"=5u;"Name"="Name5"}; +{"Amount"=108;"Comment"="Test1";"Group"=6u;"Name"="Name1"}; +{"Amount"=109;"Comment"="Test1";"Group"=6u;"Name"="Name2"}; +{"Amount"=100;"Comment"="Test3";"Group"=10u;"Name"="Name1"}; +{"Amount"=101;"Comment"="Test3";"Group"=10u;"Name"="Name2"}; +{"Amount"=102;"Comment"="Test3";"Group"=10u;"Name"="Name3"}; +{"Amount"=103;"Comment"="Test3";"Group"=10u;"Name"="Name4"}; +{"Amount"=104;"Comment"="Test3";"Group"=10u;"Name"="Name5"}; +{"Amount"=105;"Comment"="Test3";"Group"=10u;"Name"="Name6"}; +{"Amount"=100;"Comment"="Test2";"Group"=100001u;"Name"="Name1"}; +{"Amount"=102;"Comment"="Test2";"Group"=100002u;"Name"="Name2"}; +{"Amount"=103;"Comment"="Test2";"Group"=100003u;"Name"="Name1"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input1.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/input1.txt.attr new file mode 100644 index 0000000000..ffb9a19cab --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input1.txt.attr @@ -0,0 +1,42 @@ +{ + "_yql_row_spec" = { + "UniqueKeys"=%true; + "SortMembers"=["Group"; "Name"]; + "SortedBy"=["Group"; "Name"]; + "SortDirections"=[1; 1]; + "SortedByTypes"=[["DataType";"Uint64"]; ["DataType";"String"]]; + "Type" = [ + "StructType"; + [ + [ + "Group"; + [ + "DataType"; + "Uint64" + ] + ]; + [ + "Name"; + [ + "DataType"; + "String" + ] + ]; + [ + "Amount"; + [ + "DataType"; + "Int64" + ] + ]; + [ + "Comment"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input2.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/input2.txt new file mode 100644 index 0000000000..d1fa0973d9 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input2.txt @@ -0,0 +1,8 @@ +{"key"="075";"subkey"=".";"value"="abc"}; +{"key"="911";"subkey"=".";"value"="kkk"}; +{"key"="023";"subkey"=".";"value"="aaa"}; +{"key"="527";"subkey"=".";"value"="bbb"}; +{"key"="037";"subkey"=".";"value"="ddd"}; +{"key"="761";"subkey"=".";"value"="ccc"}; +{"key"="200";"subkey"=".";"value"="qqq"}; +{"key"="150";"subkey"=".";"value"="zzz"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input2.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/input2.txt.attr new file mode 100644 index 0000000000..b6100e5fd0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input2.txt.attr @@ -0,0 +1,30 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input3.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/input3.txt new file mode 100644 index 0000000000..045bc6bd14 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input3.txt @@ -0,0 +1,200 @@ +{"key"="180";"subkey"="7";"value"="Value #44"}; +{"key"="178";"subkey"="17";"value"="Value #7"}; +{"key"="6";"subkey"="4";"value"="Value #1"}; +{"key"="53";"subkey"="19";"value"="Value #41"}; +{"key"="112";"subkey"="15";"value"="Value #14"}; +{"key"="121";"subkey"="11";"value"="Value #58"}; +{"key"="69";"subkey"="5";"value"="Value #49"}; +{"key"="142";"subkey"="3";"value"="Value #5"}; +{"key"="73";"subkey"="8";"value"="Value #33"}; +{"key"="52";"subkey"="14";"value"="Value #45"}; +{"key"="18";"subkey"="6";"value"="Value #40"}; +{"key"="141";"subkey"="16";"value"="Value #35"}; +{"key"="63";"subkey"="18";"value"="Value #12"}; +{"key"="5";"subkey"="13";"value"="Value #28"}; +{"key"="128";"subkey"="2";"value"="Value #56"}; +{"key"="48";"subkey"="12";"value"="Value #13"}; +{"key"="93";"subkey"="9";"value"="Value #20"}; +{"key"="49";"subkey"="0";"value"="Value #30"}; +{"key"="95";"subkey"="1";"value"="Value #34"}; +{"key"="159";"subkey"="10";"value"="Value #52"}; +{"key"="55";"subkey"="7";"value"="Value #15"}; +{"key"="7";"subkey"="17";"value"="Value #24"}; +{"key"="35";"subkey"="4";"value"="Value #51"}; +{"key"="82";"subkey"="19";"value"="Value #0"}; +{"key"="170";"subkey"="15";"value"="Value #26"}; +{"key"="150";"subkey"="11";"value"="Value #27"}; +{"key"="26";"subkey"="5";"value"="Value #54"}; +{"key"="58";"subkey"="3";"value"="Value #37"}; +{"key"="16";"subkey"="8";"value"="Value #46"}; +{"key"="166";"subkey"="14";"value"="Value #4"}; +{"key"="86";"subkey"="6";"value"="Value #16"}; +{"key"="101";"subkey"="16";"value"="Value #32"}; +{"key"="160";"subkey"="18";"value"="Value #39"}; +{"key"="199";"subkey"="13";"value"="Value #25"}; +{"key"="138";"subkey"="2";"value"="Value #6"}; +{"key"="96";"subkey"="12";"value"="Value #57"}; +{"key"="33";"subkey"="9";"value"="Value #21"}; +{"key"="9";"subkey"="0";"value"="Value #42"}; +{"key"="21";"subkey"="1";"value"="Value #55"}; +{"key"="176";"subkey"="10";"value"="Value #23"}; +{"key"="0";"subkey"="7";"value"="Value #18"}; +{"key"="66";"subkey"="17";"value"="Value #3"}; +{"key"="198";"subkey"="4";"value"="Value #22"}; +{"key"="186";"subkey"="19";"value"="Value #17"}; +{"key"="83";"subkey"="15";"value"="Value #2"}; +{"key"="179";"subkey"="11";"value"="Value #19"}; +{"key"="64";"subkey"="5";"value"="Value #38"}; +{"key"="56";"subkey"="3";"value"="Value #50"}; +{"key"="155";"subkey"="8";"value"="Value #43"}; +{"key"="143";"subkey"="14";"value"="Value #9"}; +{"key"="188";"subkey"="6";"value"="Value #8"}; +{"key"="172";"subkey"="16";"value"="Value #53"}; +{"key"="103";"subkey"="18";"value"="Value #11"}; +{"key"="44";"subkey"="13";"value"="Value #36"}; +{"key"="173";"subkey"="2";"value"="Value #10"}; +{"key"="133";"subkey"="12";"value"="Value #48"}; +{"key"="168";"subkey"="9";"value"="Value #29"}; +{"key"="157";"subkey"="0";"value"="Value #31"}; +{"key"="152";"subkey"="1";"value"="Value #47"}; +{"key"="74";"subkey"="10";"value"="Value #59"}; +{"key"="154";"subkey"="7";"value"="Value #44"}; +{"key"="40";"subkey"="17";"value"="Value #7"}; +{"key"="89";"subkey"="4";"value"="Value #1"}; +{"key"="41";"subkey"="19";"value"="Value #41"}; +{"key"="24";"subkey"="15";"value"="Value #14"}; +{"key"="182";"subkey"="11";"value"="Value #58"}; +{"key"="80";"subkey"="5";"value"="Value #49"}; +{"key"="196";"subkey"="3";"value"="Value #5"}; +{"key"="43";"subkey"="8";"value"="Value #33"}; +{"key"="156";"subkey"="14";"value"="Value #45"}; +{"key"="34";"subkey"="6";"value"="Value #40"}; +{"key"="88";"subkey"="16";"value"="Value #35"}; +{"key"="22";"subkey"="18";"value"="Value #12"}; +{"key"="27";"subkey"="13";"value"="Value #28"}; +{"key"="84";"subkey"="2";"value"="Value #56"}; +{"key"="12";"subkey"="12";"value"="Value #13"}; +{"key"="98";"subkey"="9";"value"="Value #20"}; +{"key"="140";"subkey"="0";"value"="Value #30"}; +{"key"="31";"subkey"="1";"value"="Value #34"}; +{"key"="105";"subkey"="10";"value"="Value #52"}; +{"key"="149";"subkey"="7";"value"="Value #15"}; +{"key"="153";"subkey"="17";"value"="Value #24"}; +{"key"="177";"subkey"="4";"value"="Value #51"}; +{"key"="14";"subkey"="19";"value"="Value #0"}; +{"key"="190";"subkey"="15";"value"="Value #26"}; +{"key"="118";"subkey"="11";"value"="Value #27"}; +{"key"="174";"subkey"="5";"value"="Value #54"}; +{"key"="104";"subkey"="3";"value"="Value #37"}; +{"key"="47";"subkey"="8";"value"="Value #46"}; +{"key"="46";"subkey"="14";"value"="Value #4"}; +{"key"="124";"subkey"="6";"value"="Value #16"}; +{"key"="70";"subkey"="16";"value"="Value #32"}; +{"key"="110";"subkey"="18";"value"="Value #39"}; +{"key"="91";"subkey"="13";"value"="Value #25"}; +{"key"="192";"subkey"="2";"value"="Value #6"}; +{"key"="183";"subkey"="12";"value"="Value #57"}; +{"key"="100";"subkey"="9";"value"="Value #21"}; +{"key"="38";"subkey"="0";"value"="Value #42"}; +{"key"="71";"subkey"="1";"value"="Value #55"}; +{"key"="29";"subkey"="10";"value"="Value #23"}; +{"key"="51";"subkey"="7";"value"="Value #18"}; +{"key"="32";"subkey"="17";"value"="Value #3"}; +{"key"="130";"subkey"="4";"value"="Value #22"}; +{"key"="77";"subkey"="19";"value"="Value #17"}; +{"key"="4";"subkey"="15";"value"="Value #2"}; +{"key"="97";"subkey"="11";"value"="Value #19"}; +{"key"="67";"subkey"="5";"value"="Value #38"}; +{"key"="158";"subkey"="3";"value"="Value #50"}; +{"key"="25";"subkey"="8";"value"="Value #43"}; +{"key"="119";"subkey"="14";"value"="Value #9"}; +{"key"="2";"subkey"="6";"value"="Value #8"}; +{"key"="167";"subkey"="16";"value"="Value #53"}; +{"key"="193";"subkey"="18";"value"="Value #11"}; +{"key"="11";"subkey"="13";"value"="Value #36"}; +{"key"="129";"subkey"="2";"value"="Value #10"}; +{"key"="187";"subkey"="12";"value"="Value #48"}; +{"key"="20";"subkey"="9";"value"="Value #29"}; +{"key"="134";"subkey"="0";"value"="Value #31"}; +{"key"="115";"subkey"="1";"value"="Value #47"}; +{"key"="94";"subkey"="10";"value"="Value #59"}; +{"key"="30";"subkey"="7";"value"="Value #44"}; +{"key"="175";"subkey"="17";"value"="Value #7"}; +{"key"="62";"subkey"="4";"value"="Value #1"}; +{"key"="147";"subkey"="19";"value"="Value #41"}; +{"key"="87";"subkey"="15";"value"="Value #14"}; +{"key"="99";"subkey"="11";"value"="Value #58"}; +{"key"="114";"subkey"="5";"value"="Value #49"}; +{"key"="117";"subkey"="3";"value"="Value #5"}; +{"key"="10";"subkey"="8";"value"="Value #33"}; +{"key"="162";"subkey"="14";"value"="Value #45"}; +{"key"="171";"subkey"="6";"value"="Value #40"}; +{"key"="108";"subkey"="16";"value"="Value #35"}; +{"key"="60";"subkey"="18";"value"="Value #12"}; +{"key"="144";"subkey"="13";"value"="Value #28"}; +{"key"="113";"subkey"="2";"value"="Value #56"}; +{"key"="102";"subkey"="12";"value"="Value #13"}; +{"key"="194";"subkey"="9";"value"="Value #20"}; +{"key"="76";"subkey"="0";"value"="Value #30"}; +{"key"="189";"subkey"="1";"value"="Value #34"}; +{"key"="164";"subkey"="10";"value"="Value #52"}; +{"key"="23";"subkey"="7";"value"="Value #15"}; +{"key"="65";"subkey"="17";"value"="Value #24"}; +{"key"="54";"subkey"="4";"value"="Value #51"}; +{"key"="148";"subkey"="19";"value"="Value #0"}; +{"key"="123";"subkey"="15";"value"="Value #26"}; +{"key"="185";"subkey"="11";"value"="Value #27"}; +{"key"="28";"subkey"="5";"value"="Value #54"}; +{"key"="13";"subkey"="3";"value"="Value #37"}; +{"key"="136";"subkey"="8";"value"="Value #46"}; +{"key"="57";"subkey"="14";"value"="Value #4"}; +{"key"="184";"subkey"="6";"value"="Value #16"}; +{"key"="36";"subkey"="16";"value"="Value #32"}; +{"key"="132";"subkey"="18";"value"="Value #39"}; +{"key"="120";"subkey"="13";"value"="Value #25"}; +{"key"="50";"subkey"="2";"value"="Value #6"}; +{"key"="195";"subkey"="12";"value"="Value #57"}; +{"key"="135";"subkey"="9";"value"="Value #21"}; +{"key"="92";"subkey"="0";"value"="Value #42"}; +{"key"="151";"subkey"="1";"value"="Value #55"}; +{"key"="125";"subkey"="10";"value"="Value #23"}; +{"key"="146";"subkey"="7";"value"="Value #18"}; +{"key"="45";"subkey"="17";"value"="Value #3"}; +{"key"="90";"subkey"="4";"value"="Value #22"}; +{"key"="126";"subkey"="19";"value"="Value #17"}; +{"key"="145";"subkey"="15";"value"="Value #2"}; +{"key"="19";"subkey"="11";"value"="Value #19"}; +{"key"="127";"subkey"="5";"value"="Value #38"}; +{"key"="79";"subkey"="3";"value"="Value #50"}; +{"key"="131";"subkey"="8";"value"="Value #43"}; +{"key"="111";"subkey"="14";"value"="Value #9"}; +{"key"="75";"subkey"="6";"value"="Value #8"}; +{"key"="191";"subkey"="16";"value"="Value #53"}; +{"key"="3";"subkey"="18";"value"="Value #11"}; +{"key"="165";"subkey"="13";"value"="Value #36"}; +{"key"="85";"subkey"="2";"value"="Value #10"}; +{"key"="1";"subkey"="12";"value"="Value #48"}; +{"key"="161";"subkey"="9";"value"="Value #29"}; +{"key"="37";"subkey"="0";"value"="Value #31"}; +{"key"="107";"subkey"="1";"value"="Value #47"}; +{"key"="122";"subkey"="10";"value"="Value #59"}; +{"key"="139";"subkey"="7";"value"="Value #44"}; +{"key"="15";"subkey"="17";"value"="Value #7"}; +{"key"="106";"subkey"="4";"value"="Value #1"}; +{"key"="59";"subkey"="19";"value"="Value #41"}; +{"key"="61";"subkey"="15";"value"="Value #14"}; +{"key"="17";"subkey"="11";"value"="Value #58"}; +{"key"="68";"subkey"="5";"value"="Value #49"}; +{"key"="163";"subkey"="3";"value"="Value #5"}; +{"key"="197";"subkey"="8";"value"="Value #33"}; +{"key"="81";"subkey"="14";"value"="Value #45"}; +{"key"="169";"subkey"="6";"value"="Value #40"}; +{"key"="8";"subkey"="16";"value"="Value #35"}; +{"key"="109";"subkey"="18";"value"="Value #12"}; +{"key"="78";"subkey"="13";"value"="Value #28"}; +{"key"="181";"subkey"="2";"value"="Value #56"}; +{"key"="116";"subkey"="12";"value"="Value #13"}; +{"key"="137";"subkey"="9";"value"="Value #20"}; +{"key"="39";"subkey"="0";"value"="Value #30"}; +{"key"="72";"subkey"="1";"value"="Value #34"}; +{"key"="42";"subkey"="10";"value"="Value #52"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input3.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/input3.txt.attr new file mode 100644 index 0000000000..b6100e5fd0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input3.txt.attr @@ -0,0 +1,30 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input4.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/input4.txt new file mode 100644 index 0000000000..0827c839a2 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input4.txt @@ -0,0 +1,13 @@ +{"key"="023";"subkey"="3";"value"="aaa"}; +{"key"="037";"subkey"="5";"value"="ddd"}; +{"key"="075";"subkey"="1";"value"="abc"}; +{"key"="150";"subkey"="1";"value"="aaa"}; +{"key"="150";"subkey"="3";"value"="iii"}; +{"key"="150";"subkey"="8";"value"="zzz"}; +{"key"="200";"subkey"="7";"value"="qqq"}; +{"key"="527";"subkey"="4";"value"="bbb"}; +{"key"="761";"subkey"="6";"value"="ccc"}; +{"key"="911";"subkey"="1";"value"="kkk"}; +{"key"="911";"subkey"="2";"value"="hjk"}; +{"key"="911";"subkey"="3";"value"="def"}; +{"key"="911";"subkey"="4";"value"="xyz"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input4.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/input4.txt.attr new file mode 100644 index 0000000000..6ee50d08cc --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input4.txt.attr @@ -0,0 +1,11 @@ +{"_yql_row_spec"={ + "Type"=["StructType";[ + ["key";["DataType";"String"]]; + ["subkey";["DataType";"String"]]; + ["value";["DataType";"String"]] + ]]; + "SortDirections"=[1;1;]; + "SortedBy"=["key";"subkey";]; + "SortedByTypes"=[["DataType";"String";];["DataType";"String";];]; + "SortMembers"=["key";"subkey";]; +}} diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/inputDistinctKey.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/inputDistinctKey.txt new file mode 100644 index 0000000000..6b1dac36b1 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/inputDistinctKey.txt @@ -0,0 +1,8 @@ +{"key"="1";"subkey"=".";"value"="abc"}; +{"key"="1";"subkey"=".";"value"="kkk"}; +{"key"="2";"subkey"=".";"value"="aaa"}; +{"key"="2";"subkey"=".";"value"="bbb"}; +{"key"="2";"subkey"=".";"value"="ddd"}; +{"key"="bad";"subkey"=".";"value"="ccc"}; +{"key"="1";"subkey"=".";"value"="qqq"}; +{"key"="1";"subkey"=".";"value"="zzz"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/inputDistinctKey.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/inputDistinctKey.txt.attr new file mode 100644 index 0000000000..b6100e5fd0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/inputDistinctKey.txt.attr @@ -0,0 +1,30 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion1.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion1.txt new file mode 100644 index 0000000000..93251b80a0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion1.txt @@ -0,0 +1,6 @@ +{"age"="30";"name1"="bob"}; +{"age"="16";"name1"="alice"}; +{"age"="55";"name1"="jone"}; +{"age"="45";"name1"="silver"}; +{"age"="45";"name1"="smith"}; +{"age"="90";"name1"="stive"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion1.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion1.txt.attr new file mode 100644 index 0000000000..bdbbea169d --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion1.txt.attr @@ -0,0 +1,3 @@ +{ + "_yql_row_spec"={"Type"=["StructType";[["age";["DataType";"String"]];["name1";["DataType";"String"]]]]} +} diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion2.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion2.txt new file mode 100644 index 0000000000..1d9b6e8b42 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion2.txt @@ -0,0 +1,6 @@ +{"age"="30";"name2"="bob"}; +{"age"="16";"name2"="alice"}; +{"age"="55";"name2"="jone"}; +{"age"="45";"name2"="silver"}; +{"age"="45";"name2"="smith"}; +{"age"="90";"name2"="stive"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion2.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion2.txt.attr new file mode 100644 index 0000000000..6d032642cf --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/inputUnion2.txt.attr @@ -0,0 +1,3 @@ +{ + "_yql_row_spec"={"Type"=["StructType";[["age";["DataType";"String"]];["name2";["DataType";"String"]]]]} +} diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input_tbl_content.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/input_tbl_content.txt new file mode 100644 index 0000000000..d1fa0973d9 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input_tbl_content.txt @@ -0,0 +1,8 @@ +{"key"="075";"subkey"=".";"value"="abc"}; +{"key"="911";"subkey"=".";"value"="kkk"}; +{"key"="023";"subkey"=".";"value"="aaa"}; +{"key"="527";"subkey"=".";"value"="bbb"}; +{"key"="037";"subkey"=".";"value"="ddd"}; +{"key"="761";"subkey"=".";"value"="ccc"}; +{"key"="200";"subkey"=".";"value"="qqq"}; +{"key"="150";"subkey"=".";"value"="zzz"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/input_tbl_content.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/input_tbl_content.txt.attr new file mode 100644 index 0000000000..b6100e5fd0 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/input_tbl_content.txt.attr @@ -0,0 +1,30 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +}
\ No newline at end of file diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/typed_table.txt b/yql/essentials/tests/s-expressions/suites/Optimizers/typed_table.txt new file mode 100644 index 0000000000..cc1df00106 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/typed_table.txt @@ -0,0 +1,4 @@ +{"key"="075";"subkey"="";"value"="a7c"}; +{"key"="800";"subkey"="2";"value"="ddd"}; +{"key"="020";"subkey"="";"value"="q"}; +{"key"="150";"subkey"="4";"value"="q8z"}; diff --git a/yql/essentials/tests/s-expressions/suites/Optimizers/typed_table.txt.attr b/yql/essentials/tests/s-expressions/suites/Optimizers/typed_table.txt.attr new file mode 100644 index 0000000000..9061763bd6 --- /dev/null +++ b/yql/essentials/tests/s-expressions/suites/Optimizers/typed_table.txt.attr @@ -0,0 +1,32 @@ +{ + "_yql_view_opt_struct"="SELECT AsList(1,2,3) as b FROM self;"; + "_yql_syntax_version_opt_struct"=1; + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "key"; + [ + "DataType"; + "String" + ] + ]; + [ + "subkey"; + [ + "DataType"; + "String" + ] + ]; + [ + "value"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +} |