fix for ansi lexer

commit_hash:b8fff1753a6450e4a83544790767f14230b380b0

4 files changed, 22 insertions, 8 deletions

diff --git a/yql/essentials/sql/v1/format/sql_format.cpp b/yql/essentials/sql/v1/format/sql_format.cpp
index 7ee10cd5e1a..7c6d70ca098 100644
--- a/yql/essentials/sql/v1/format/sql_format.cpp
+++ b/yql/essentials/sql/v1/format/sql_format.cpp
@@ -524,10 +524,11 @@ private:
 class TPrettyVisitor {
 friend struct TStaticData;
 public:
-    TPrettyVisitor(const TParsedTokenList& parsedTokens, const TParsedTokenList& comments)
+    TPrettyVisitor(const TParsedTokenList& parsedTokens, const TParsedTokenList& comments, bool ansiLexer)
         : StaticData(TStaticData::GetInstance())
         , ParsedTokens(parsedTokens)
         , Comments(comments)
+        , AnsiLexer(ansiLexer)
     {
     }
 
@@ -1737,7 +1738,7 @@ private:
             }
         }
 
-        if (ParsedTokens[TokenIndex].Name == "STRING_VALUE") {
+        if (!AnsiLexer && ParsedTokens[TokenIndex].Name == "STRING_VALUE") {
             TStringBuf checkStr = str;
             if (checkStr.SkipPrefix("\"") && checkStr.ChopSuffix("\"") && !checkStr.Contains("'")) {
                 str = TStringBuilder() << '\'' << checkStr << '\'';
@@ -2929,6 +2930,7 @@ private:
     const TStaticData& StaticData;
     const TParsedTokenList& ParsedTokens;
     const TParsedTokenList& Comments;
+    const bool AnsiLexer;
     TStringBuilder SB;
     ui32 OutColumn = 0;
     ui32 OutLine = 1;
@@ -3236,7 +3238,7 @@ public:
                 continue;
             }
 
-            TPrettyVisitor visitor(parsedTokens, comments);
+            TPrettyVisitor visitor(parsedTokens, comments, parsedSettings.AnsiLexer);
             bool addLineBefore = false;
             bool addLineAfter = false;
             TMaybe<ui32> stmtCoreAltCase;
diff --git a/yql/essentials/sql/v1/format/sql_format_ut.cpp b/yql/essentials/sql/v1/format/sql_format_ut.cpp
index 3cfd7031159..e721f1fae6e 100644
--- a/yql/essentials/sql/v1/format/sql_format_ut.cpp
+++ b/yql/essentials/sql/v1/format/sql_format_ut.cpp
@@ -11,9 +11,10 @@ namespace {
 using TCases = TVector<std::pair<TString, TString>>;
 
 struct TSetup {
-    TSetup() {
+    TSetup(bool ansiLexer = false) {
         NSQLTranslation::TTranslationSettings settings;
         settings.Arena = &Arena;
+        settings.AnsiLexer = ansiLexer;
         Formatter = NSQLFormat::MakeSqlFormatter(settings);
     }
 
diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h
index e56415feda1..2cf4894d085 100644
--- a/yql/essentials/sql/v1/format/sql_format_ut.h
+++ b/yql/essentials/sql/v1/format/sql_format_ut.h
@@ -1737,7 +1737,7 @@ Y_UNIT_TEST(ResourcePoolClassifierOperations) {
 Y_UNIT_TEST(Backup) {
     TCases cases = {
         {"\tBaCKup\n\n TestCollection      incremENTAl",
-         "BACKUP TestCollection INCREMENTAL;\n"},
+            "BACKUP TestCollection INCREMENTAL;\n"},
     };
 
     TSetup setup;
@@ -1747,9 +1747,19 @@ Y_UNIT_TEST(Backup) {
 Y_UNIT_TEST(Restore) {
     TCases cases = {
         {"resToRe\n\n\n TestCollection       aT\n  \t \n     '2024-06-16_20-14-02'",
-         "RESTORE TestCollection AT '2024-06-16_20-14-02';\n"},
+            "RESTORE TestCollection AT '2024-06-16_20-14-02';\n"},
     };
 
     TSetup setup;
     setup.Run(cases);
 }
+
+Y_UNIT_TEST(AnsiLexer) {
+    TCases cases = {
+        {"select 'a', \"a\" from (select 1 as \"a\")",
+            "SELECT\n\t'a',\n\t\"a\"\nFROM (\n\tSELECT\n\t\t1 AS \"a\"\n);\n"},
+    };
+
+    TSetup setup(/* ansiLexer = */ true);
+    setup.Run(cases);
+}
diff --git a/yql/essentials/sql/v1/format/sql_format_ut_antlr4.cpp b/yql/essentials/sql/v1/format/sql_format_ut_antlr4.cpp
index 63c08bb8b7e..12e80587eac 100644
--- a/yql/essentials/sql/v1/format/sql_format_ut_antlr4.cpp
+++ b/yql/essentials/sql/v1/format/sql_format_ut_antlr4.cpp
@@ -11,13 +11,14 @@ namespace {
 using TCases = TVector<std::pair<TString, TString>>;
 
 struct TSetup {
-    TSetup() {
+    TSetup(bool ansiLexer = false) {
         NSQLTranslation::TTranslationSettings settings;
         settings.Arena = &Arena;
         settings.Antlr4Parser = true;
+        settings.AnsiLexer = ansiLexer;
         Formatter = NSQLFormat::MakeSqlFormatter(settings);
     }
-    
+
     void Run(const TCases& cases, NSQLFormat::EFormatMode mode = NSQLFormat::EFormatMode::Pretty) {
         for (const auto& c : cases) {
             NYql::TIssues issues;