add one more test to defaults and fix issues with generating defaults for non-key columns KIKIMR-19547

8 files changed, 148 insertions, 35 deletions

diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
index 3cb2588ccf..f37566c798 100644
--- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
+++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json
@@ -204,7 +204,7 @@
                 {"Index": 0, "Name": "Table", "Type": "TKqpTable"},
                 {"Index": 1, "Name": "Input", "Type": "TExprBase"},
                 {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"},
-                {"Index": 3, "Name": "AutoIncrementColumns", "Type": "TCoAtomList"},
+                {"Index": 3, "Name": "DefaultConstraintColumns", "Type": "TCoAtomList"},
                 {"Index": 4, "Name": "InputItemType", "Type": "TExprBase"}
             ]
         },
@@ -461,7 +461,7 @@
             "Children": [
                 {"Index": 1, "Name": "Table", "Type": "TKqpTable"},
                 {"Index": 2, "Name": "Columns", "Type": "TCoAtomList"},
-                {"Index": 3, "Name": "AutoIncrementColumns", "Type": "TCoAtomList"},
+                {"Index": 3, "Name": "DefaultConstraintColumns", "Type": "TCoAtomList"},
                 {"Index": 4, "Name": "InputItemType", "Type": "TExprBase"}
             ]
         },
diff --git a/ydb/core/kqp/host/kqp_type_ann.cpp b/ydb/core/kqp/host/kqp_type_ann.cpp
index b4bdd2612a..3a5946e3eb 100644
--- a/ydb/core/kqp/host/kqp_type_ann.cpp
+++ b/ydb/core/kqp/host/kqp_type_ann.cpp
@@ -1269,23 +1269,15 @@ TStatus AnnotateSequencer(const TExprNode::TPtr& node, TExprContext& ctx, const
 
     auto table = resolveResult.second;
     YQL_ENSURE(rowType);
-    absl::flat_hash_set<TString, THash<TString>> missingKeyColumns;
-    for(const auto& key: table->Metadata->KeyColumnNames) {
-        if (rowType->FindItem(key)) {
-            continue;
-        }
-
-        const auto& infoIt = table->Metadata->Columns.find(key);
-        YQL_ENSURE(infoIt != table->Metadata->Columns.end());
-        const auto& info = infoIt->second;
-        if (info.IsDefaultKindDefined()) {
-            auto [_, inserted] = missingKeyColumns.emplace(key);
-            YQL_ENSURE(inserted, "unexpected duplicates in key columns.");
-        }
+    absl::flat_hash_set<TString, THash<TString>> columnsToGenerate;
+    TCoAtomList generatedOnWriteColumns (node->Child(TKqlSequencer::idx_DefaultConstraintColumns)); 
+    for(const auto& col : generatedOnWriteColumns) {
+        auto [_, inserted] = columnsToGenerate.emplace(TString(col.Value()));
+        YQL_ENSURE(inserted, "unexpected duplicates in the names of columns.");
     }
 
     TVector<const TItemExprType *> seqRowTypeItems = rowType->GetItems();
-    for (auto &column : missingKeyColumns) {
+    for (auto &column : columnsToGenerate) {
         auto columnType = table->GetColumnType(column);
         YQL_ENSURE(columnType);
         seqRowTypeItems.push_back(
@@ -1508,7 +1500,7 @@ TStatus AnnotateSequencerConnection(const TExprNode::TPtr& node, TExprContext& c
         return TStatus::Error;
     }
 
-    if (!EnsureTupleOfAtoms(*node->Child(TKqpCnSequencer::idx_AutoIncrementColumns), ctx)) {
+    if (!EnsureTupleOfAtoms(*node->Child(TKqpCnSequencer::idx_DefaultConstraintColumns), ctx)) {
         return TStatus::Error;
     }
 
diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp
index 413007e649..653b58aeb9 100644
--- a/ydb/core/kqp/opt/kqp_opt_kql.cpp
+++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp
@@ -224,14 +224,14 @@ TExprNode::TPtr IsUpdateSetting(TExprContext& ctx, const TPositionHandle& pos) {
 }
 
 TExprBase BuildKqlSequencer(TExprBase& input, const TKikimrTableDescription& table,
-    const TCoAtomList& outputCols, const TCoAtomList& autoIncrement,
+    const TCoAtomList& outputCols, const TCoAtomList& defaultConstraintColumns,
     TPositionHandle pos, TExprContext& ctx)
 {
     return Build<TKqlSequencer>(ctx, pos)
         .Input(input.Ptr())
         .Table(BuildTableMeta(table, pos, ctx))
         .Columns(outputCols.Ptr())
-        .AutoIncrementColumns(autoIncrement.Ptr())
+        .DefaultConstraintColumns(defaultConstraintColumns.Ptr())
         .InputItemType(ExpandType(pos, *input.Ref().GetTypeAnn(), ctx))
         .Done();
 }
@@ -807,12 +807,12 @@ TVector<TExprBase> HandleWriteTable(const TKiWriteTable& write, TExprContext& ct
     YQL_ENSURE(inputColumnsSetting);
     auto inputColumns = TCoNameValueTuple(inputColumnsSetting).Value().Cast<TCoAtomList>();
 
-    auto autoincrementColumnsSetting = GetSetting(write.Settings().Ref(), "autoincrement_columns");
-    YQL_ENSURE(autoincrementColumnsSetting);
-    auto autoincrementColumns = TCoNameValueTuple(autoincrementColumnsSetting).Value().Cast<TCoAtomList>();
+    auto defaultConstraintColumnsNode = GetSetting(write.Settings().Ref(), "default_constraint_columns");
+    YQL_ENSURE(defaultConstraintColumnsNode);
+    auto defaultConstraintColumns = TCoNameValueTuple(defaultConstraintColumnsNode).Value().Cast<TCoAtomList>();
 
     auto op = GetTableOp(write);
-    if (autoincrementColumns.Ref().ChildrenSize() > 0) {
+    if (defaultConstraintColumns.Ref().ChildrenSize() > 0) {
         if (op == TYdbOperation::UpdateOn || op == TYdbOperation::DeleteOn) {
             const TString err = "Key columns are not specified.";
             ctx.AddError(YqlIssue(ctx.GetPosition(write.Pos()), TIssuesIds::KIKIMR_BAD_REQUEST, err));
@@ -821,9 +821,9 @@ TVector<TExprBase> HandleWriteTable(const TKiWriteTable& write, TExprContext& ct
     }
 
     if (HasIndexesToWrite(tableData)) {
-        return { WriteTableWithIndexUpdate(write, inputColumns, autoincrementColumns, tableData, ctx) } ;
+        return { WriteTableWithIndexUpdate(write, inputColumns, defaultConstraintColumns, tableData, ctx) } ;
     } else {
-        return { WriteTableSimple(write, inputColumns, autoincrementColumns, tableData, ctx) } ;
+        return { WriteTableSimple(write, inputColumns, defaultConstraintColumns, tableData, ctx) } ;
     }
 }
 
diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp
index 03e47725e6..854ab938f3 100644
--- a/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp
+++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_build_stage.cpp
@@ -514,7 +514,7 @@ NYql::NNodes::TExprBase KqpBuildSequencerStages(NYql::NNodes::TExprBase node, NY
             .Build()
             .Table(sequencer.Table())
             .Columns(sequencer.Columns())
-            .AutoIncrementColumns(sequencer.AutoIncrementColumns())
+            .DefaultConstraintColumns(sequencer.DefaultConstraintColumns())
             .InputItemType(sequencer.InputItemType())
             .Done();
     } else if (sequencer.Input().Maybe<TDqCnUnionAll>()) {
@@ -524,7 +524,7 @@ NYql::NNodes::TExprBase KqpBuildSequencerStages(NYql::NNodes::TExprBase node, NY
             .Output(output)
             .Table(sequencer.Table())
             .Columns(sequencer.Columns())
-            .AutoIncrementColumns(sequencer.AutoIncrementColumns())
+            .DefaultConstraintColumns(sequencer.DefaultConstraintColumns())
             .InputItemType(sequencer.InputItemType())
             .Done();
 
diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
index 803593a9a4..d71a8a9602 100644
--- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
@@ -393,12 +393,13 @@ private:
             return TStatus::Error;
         }
 
-        THashSet<TString> autoincrementColumns;
+        THashSet<TString> defaultConstraintColumnsSet;
         for (auto& keyColumnName : table->Metadata->KeyColumnNames) {
             const auto& columnInfo = table->Metadata->Columns.at(keyColumnName);
             if (rowType->FindItem(keyColumnName)) {
                 continue;
             }
+
             if (!columnInfo.IsDefaultKindDefined())  {
                 ctx.AddError(YqlIssue(pos, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder()
                     << "Missing key column in input: " << keyColumnName
@@ -406,7 +407,21 @@ private:
                 return TStatus::Error;
             }
 
-            autoincrementColumns.emplace(keyColumnName);
+            defaultConstraintColumnsSet.emplace(keyColumnName);
+        }
+
+        for(const auto& [name, info] : table->Metadata->Columns) {
+            if (rowType->FindItem(name)) {
+                continue;
+            }
+
+            if (op == TYdbOperation::UpdateOn) {
+                continue;
+            }
+
+            if (info.IsDefaultKindDefined()) {
+                defaultConstraintColumnsSet.emplace(name);
+            }
         }
 
         if (op == TYdbOperation::InsertAbort || op == TYdbOperation::InsertRevert ||
@@ -460,9 +475,9 @@ private:
                 columns.push_back(ctx.NewAtom(node.Pos(), item->GetName()));
             }
 
-            TExprNode::TListType autoincrementColumnsList;
-            for(auto& autoincrement: autoincrementColumns) {
-                autoincrementColumnsList.push_back(ctx.NewAtom(node.Pos(), autoincrement));
+            TExprNode::TListType defaultConstraintColumns;
+            for(auto& generatedColumn: defaultConstraintColumnsSet) {
+                defaultConstraintColumns.push_back(ctx.NewAtom(node.Pos(), generatedColumn));
             }
 
             node.Ptr()->ChildRef(TKiWriteTable::idx_Settings) = Build<TCoNameValueTupleList>(ctx, node.Pos())
@@ -474,9 +489,9 @@ private:
                         .Build()
                     .Build()
                 .Add()
-                    .Name().Build("autoincrement_columns")
+                    .Name().Build("default_constraint_columns")
                     .Value<TCoAtomList>()
-                        .Add(autoincrementColumnsList)
+                        .Add(defaultConstraintColumns)
                         .Build()
                     .Build()
                 .Done()
diff --git a/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp b/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
index ecb0f892e1..62d751870f 100644
--- a/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
+++ b/ydb/core/kqp/query_compiler/kqp_query_compiler.cpp
@@ -1092,7 +1092,7 @@ private:
             const auto inputItemType = inputNodeType->Cast<TListExprType>()->GetItemType();
             sequencerProto.SetInputType(NMiniKQL::SerializeNode(CompileType(pgmBuilder, *inputItemType), TypeEnv));
 
-            auto autoIncrementColumns = sequencer.AutoIncrementColumns();
+            auto autoIncrementColumns = sequencer.DefaultConstraintColumns();
             for(const auto& column : autoIncrementColumns) {
                 sequencerProto.AddAutoIncrementColumns(column.StringValue());
             }
diff --git a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp
index 880fa41e0c..5a8e89b754 100644
--- a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp
+++ b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp
@@ -466,5 +466,110 @@ Y_UNIT_TEST_SUITE(KqpConstraints) {
         }        
     }
 
+    Y_UNIT_TEST(AlterTableAddNotNullWithDefault) {
+
+        NKikimrConfig::TAppConfig appConfig;
+        appConfig.MutableTableServiceConfig()->SetEnableSequences(false);
+        appConfig.MutableTableServiceConfig()->SetEnableColumnsWithDefault(true);
+
+        TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig()).SetAppConfig(appConfig));
+        auto db = kikimr.GetTableClient();
+        auto session = db.CreateSession().GetValueSync().GetSession();
+
+        {
+            auto query = R"(
+                --!syntax_v1
+                CREATE TABLE `/Root/AlterTableAddNotNullColumn` (
+                    Key Uint32,
+                    Value String,
+                    PRIMARY KEY (Key)
+                );
+            )";
+
+            auto result = session.ExecuteSchemeQuery(query).GetValueSync();
+            UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
+                                       result.GetIssues().ToString());
+        }
+
+        {
+            TString query = R"(
+                UPSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value) VALUES (1, "Old");
+            )";
+
+            NYdb::NTable::TExecDataQuerySettings execSettings;
+            execSettings.KeepInQueryCache(true);
+            execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+            auto result =
+                session
+                    .ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(),
+                                      execSettings)
+                    .ExtractValueSync();
+
+            UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
+                                       result.GetIssues().ToString());
+        }
+
+        {
+            auto query = R"(
+                --!syntax_v1
+                ALTER TABLE `/Root/AlterTableAddNotNullColumn` ADD COLUMN Exists Int32 NOT NULL DEFAULT 1;
+            )";
+
+            auto result = session.ExecuteSchemeQuery(query).GetValueSync();
+            UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
+                                       result.GetIssues().ToString());
+        }
+
+        Sleep(TDuration::Seconds(3));
+
+        {
+            TString query = R"(
+                INSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value) VALUES (2, "New");
+            )";
+
+            NYdb::NTable::TExecDataQuerySettings execSettings;
+            execSettings.KeepInQueryCache(true);
+            execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+            auto result =
+                session
+                    .ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(),
+                                      execSettings)
+                    .ExtractValueSync();
+
+            UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
+                                       result.GetIssues().ToString());
+        }
+
+        {
+            TString query = R"(
+                SELECT * FROM `/Root/AlterTableAddNotNullColumn` ORDER BY Key;
+            )";
+
+            NYdb::NTable::TExecDataQuerySettings execSettings;
+            execSettings.KeepInQueryCache(true);
+            execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic);
+
+            auto result =
+                session
+                    .ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(),
+                                      execSettings)
+                    .ExtractValueSync();
+
+            UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS,
+                                       result.GetIssues().ToString());
+
+            Cerr << NYdb::FormatResultSetYson(result.GetResultSet(0)) << Endl;
+            CompareYson(R"(
+                    [
+                        [[1u];["Old"];[1]];[[2u];["New"];[1]]
+                    ]
+                )",
+                NYdb::FormatResultSetYson(result.GetResultSet(0)));
+        }
+
+    }
+
 }
 } // namespace NKikimr::NKqp
\ No newline at end of file
diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
index bb5be86567..bc0596e6bf 100644
--- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
@@ -97,6 +97,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> AlterMainTablePropose(
 
             col->SetType(NScheme::TypeName(typeInfo, typeMod));
             col->SetName(colInfo.ColumnName);
+            col->MutableDefaultFromLiteral()->CopyFrom(colInfo.DefaultFromLiteral);
         }
 
     } else {