Make SQL external declare priority less than text declare

initial

5 files changed, 46 insertions, 17 deletions

diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h
index 753397717d..f1d909de62 100644
--- a/ydb/library/yql/sql/settings/translation_settings.h
+++ b/ydb/library/yql/sql/settings/translation_settings.h
@@ -70,6 +70,7 @@ namespace NSQLTranslation {
 
         // each (name, type) entry in this map is equivalent to
         // DECLARE $name AS type;
+        // NOTE: DECLARE statement in SQL text overrides entry in DeclaredNamedExprs
         TMap<TString, TString> DeclaredNamedExprs;
 
         ESqlMode Mode;
diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp
index 7d73876a5b..a82b3de6d9 100644
--- a/ydb/library/yql/sql/v1/context.cpp
+++ b/ydb/library/yql/sql/v1/context.cpp
@@ -299,12 +299,18 @@ TNodePtr TContext::UniversalAlias(const TString& baseName, TNodePtr&& node) {
 }
 
 bool TContext::IsAlreadyDeclared(const TString& varName) const {
-    return Variables.find(varName) != Variables.end();
+    return Variables.find(varName) != Variables.end() && !WeakVariables.contains(varName);
 }
 
-void TContext::DeclareVariable(const TString& varName, const TNodePtr& typeNode) {
-    auto inserted = Variables.emplace(varName, typeNode);
-    YQL_ENSURE(inserted.second);
+void TContext::DeclareVariable(const TString& varName, const TNodePtr& typeNode, bool isWeak) {
+    if (isWeak) {
+        auto inserted = Variables.emplace(varName, typeNode);
+        YQL_ENSURE(inserted.second);
+        WeakVariables.insert(varName);
+    } else {
+        WeakVariables.erase(WeakVariables.find(varName));
+        Variables[varName] = typeNode;
+    }
 }
 
 bool TContext::AddExport(TPosition pos, const TString& name) {
diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h
index c1ba253767..60e4d74cee 100644
--- a/ydb/library/yql/sql/v1/context.h
+++ b/ydb/library/yql/sql/v1/context.h
@@ -164,7 +164,7 @@ namespace NSQLTranslationV1 {
         }
 
         bool IsAlreadyDeclared(const TString& varName) const;
-        void DeclareVariable(const TString& varName, const TNodePtr& typeNode);
+        void DeclareVariable(const TString& varName, const TNodePtr& typeNode, bool isWeak = false);
 
         bool AddExport(TPosition symbolPos, const TString& symbolName);
         TString AddImport(const TVector<TString>& modulePath);
@@ -215,6 +215,7 @@ namespace NSQLTranslationV1 {
 
     public:
         THashMap<TString, TNodePtr> Variables;
+        THashSet<TString> WeakVariables;
         NSQLTranslation::TTranslationSettings Settings;
         std::unique_ptr<TMemoryPool> Pool;
         NYql::TIssues& Issues;
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index 1ebe4e1c37..c5261a1073 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -11101,7 +11101,10 @@ TNodePtr TSqlQuery::Build(const TSQLv1ParserAST& ast) {
             TNodePtr typeNode = BuildBuiltinFunc(Ctx, Ctx.Pos(), "ParseType", { BuildLiteralRawString(Ctx.Pos(), type) });
             PushNamedAtom(Ctx.Pos(), varName);
             // no duplicates are possible at this stage
-            Ctx.DeclareVariable(varName, typeNode);
+            bool isWeak = true;
+            Ctx.DeclareVariable(varName, typeNode, isWeak);
+            // avoid 'Symbol is not used' warning for externally declared expression
+            YQL_ENSURE(GetNamedNode(varName));
         }
     }
 
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 968d01b358..d6775fb53b 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -3796,6 +3796,14 @@ select FormatType($f());
         UNIT_ASSERT(!res.Root);
         UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:17: Error: Named subquery can not be used as a top level statement in libraries\n");
     }
+
+    Y_UNIT_TEST(SelectingFromMonitoringIsNotAllowed) {
+        NSQLTranslation::TTranslationSettings settings;
+        auto res = SqlToYqlWithSettings("select * from mon.zzz;", settings);
+        UNIT_ASSERT(!res.Root);
+        UNIT_ASSERT_NO_DIFF(Err2Str(res),
+            "<main>:1:15: Error: Selecting data from monitoring source is not supported\n");
+    }
 }
 
 void CheckUnused(const TString& req, const TString& symbol, unsigned row, unsigned col) {
@@ -4782,12 +4790,31 @@ Y_UNIT_TEST_SUITE(ExternalDeclares) {
         UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]);
     }
 
-    Y_UNIT_TEST(NoDeclareOverrides) {
+    Y_UNIT_TEST(DeclareOverrides) {
         NSQLTranslation::TTranslationSettings settings;
         settings.DeclaredNamedExprs["foo"] = "String";
         auto res = SqlToYqlWithSettings("declare $foo as Int32; select $foo;", settings);
         UNIT_ASSERT(res.IsOk());
-        UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:9: Warning: Duplicate declaration of '$foo' will be ignored, code: 4536\n");
+        UNIT_ASSERT(res.Issues.Size() == 0);
+
+        TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
+            if (word == "declare") {
+                UNIT_ASSERT_VALUES_UNEQUAL(TString::npos, line.find(R"__((declare $foo (DataType 'Int32)))__"));
+            }
+        };
+
+        TWordCountHive elementStat = {{TString("declare"), 0}};
+        VerifyProgram(res, elementStat, verifyLine);
+
+        UNIT_ASSERT_VALUES_EQUAL(1, elementStat["declare"]);
+    }
+
+    Y_UNIT_TEST(UnusedDeclareDoesNotProduceWarning) {
+        NSQLTranslation::TTranslationSettings settings;
+        settings.DeclaredNamedExprs["foo"] = "String";
+        auto res = SqlToYqlWithSettings("select 1;", settings);
+        UNIT_ASSERT(res.IsOk());
+        UNIT_ASSERT(res.Issues.Size() == 0);
 
         TVerifyLineFunc verifyLine = [](const TString& word, const TString& line) {
             if (word == "declare") {
@@ -4810,14 +4837,6 @@ Y_UNIT_TEST_SUITE(ExternalDeclares) {
             "<main>:0:5: Error: Unknown type: 'BadType'\n"
             "<main>: Error: Failed to parse type for externally declared name 'foo'\n");
     }
-
-    Y_UNIT_TEST(SelectingFromMonitoringIsNotAllowed) {
-        NSQLTranslation::TTranslationSettings settings;
-        auto res = SqlToYqlWithSettings("select * from mon.zzz;", settings);
-        UNIT_ASSERT(!res.Root);
-        UNIT_ASSERT_NO_DIFF(Err2Str(res),
-            "<main>:1:15: Error: Selecting data from monitoring source is not supported\n");
-    }
 }
 
 Y_UNIT_TEST_SUITE(ExternalDataSource) {
@@ -5200,7 +5219,6 @@ Y_UNIT_TEST_SUITE(TopicsDDL) {
         TStringBuilder finalQuery;
 
         finalQuery << "use plato;" << Endl << query;
-        Cerr << "Run query: " << query << Endl;
         auto res = SqlToYql(finalQuery, 10, "kikimr");
         if (expectOk) {
             UNIT_ASSERT_C(res.IsOk(), res.Issues.ToString());