diff options
author | Aleksey Myasnikov <79263256394@ya.ru> | 2022-05-11 14:32:27 +0300 |
---|---|---|
committer | Aleksey Myasnikov <79263256394@ya.ru> | 2022-05-11 14:32:27 +0300 |
commit | 8606684d047c20a14a423f0fe0a4f0e7de670c60 (patch) | |
tree | 2a3afd65946e0cdee149b4757f2526d5148e7265 | |
parent | 75902786de9844f219bb757ddcd4c51e02955cec (diff) | |
download | ydb-8606684d047c20a14a423f0fe0a4f0e7de670c60.tar.gz |
add upsert, bulk-upsert recipes for go
ref:95593950c11279da6dff369c73256b6225739370
10 files changed, 327 insertions, 2 deletions
diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md new file mode 100644 index 0000000000..2c8d8246f3 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md @@ -0,0 +1,70 @@ +```go +package main + +import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open( + ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer func() { + _ = db.Close(ctx) + }() + type logMessage struct { + App string + Host string + Timestamp time.Time + HTTPCode uint32 + Message string + } + // prepare native go data + const batchSize = 10000 + logs := make([]logMessage, 0, batchSize) + for i := 0; i < batchSize; i++ { + message := logMessage{ + App: fmt.Sprintf("App_%d", i/256), + Host: fmt.Sprintf("192.168.0.%d", i%256), + Timestamp: time.Now().Add(time.Millisecond * time.Duration(i%1000)), + HTTPCode: 200, + } + if i%2 == 0 { + message.Message = "GET / HTTP/1.1" + } else { + message.Message = "GET /images/logo.png HTTP/1.1" + } + logs = append(logs, message) + } + // execute bulk upsert with native ydb data + err = db.Table().Do( // Do retry operation on errors with best effort + ctx, // context manage exiting from Do + func(ctx context.Context, s table.Session) (err error) { // retry operation + rows := make([]types.Value, 0, len(logs)) + for _, msg := range logs { + rows = append(rows, types.StructValue( + types.StructFieldValue("App", types.UTF8Value(msg.App)), + types.StructFieldValue("Host", types.UTF8Value(msg.Host)), + types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)), + types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)), + types.StructFieldValue("Message", types.UTF8Value(msg.Message)), + )) + } + return s.BulkUpsert(ctx, "/local/bulk_upsert_example", types.ListValue(rows...)) + }, + ) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } +} +``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md new file mode 100644 index 0000000000..58e47d3068 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md @@ -0,0 +1,16 @@ +# Bulk-upserting data + +{% include [work in progress message](../_includes/addition.md) %} + +{{ ydb-short-name }} supports bulk-upserting data without atomicity guarantees. Writing is being performed in multiple independent parallel transactions, within a single partition each. For that reason, `BulkUpsert` is more efficient than `YQL`. When completed successfully, `BulkUpsert` guarantees that all data is written and committed. + +Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for bulk-upserting data: + +{% list tabs %} + +- Go + + + {% include [go.md](_includes/go.md) %} + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml index 4443196364..02069df58d 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml @@ -10,4 +10,8 @@ items: - name: Troubleshooting include: { mode: link, path: debug/toc_p.yaml } - name: Setting the session pool size - href: session_pool_limit/index.md
\ No newline at end of file + href: session_pool_limit/index.md +- name: Upserting data + href: upsert/index.md +- name: Bulk-upserting data + href: bulk_upsert/index.md diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md new file mode 100644 index 0000000000..82b74ed301 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md @@ -0,0 +1,59 @@ +```go +package main + +import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open( + ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer func() { + _ = db.Close(ctx) + }() + // execute bulk upsert with native ydb data + err = db.Table().DoTx( // Do retry operation on errors with best effort + ctx, // context manages exiting from Do + func(ctx context.Context, tx table.TransactionActor) (err error) { // retry operation + res, err = tx.Execute( + ctx,` + PRAGMA TablePathPrefix("/path/to/table"); + DECLARE $seriesID AS Uint64; + DECLARE $seasonID AS Uint64; + DECLARE $episodeID AS Uint64; + DECLARE $views AS Uint64; + UPSERT INTO episodes ( series_id, season_id, episode_id, views ) + VALUES ( $seriesID, $seasonID, $episodeID, $views ); + `, + table.NewQueryParameters( + table.ValueParam("$seriesID", types.Uint64Value(1)), + table.ValueParam("$seasonID", types.Uint64Value(1)), + table.ValueParam("$episodeID", types.Uint64Value(1)), + table.ValueParam("$views", types.Uint64Value(1)), // increment views + ), + ) + if err != nil { + return err + } + if err = res.Err(); err != nil { + return err + } + return res.Close() + }, + ) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } +} +``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md new file mode 100644 index 0000000000..562af07e2c --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md @@ -0,0 +1,14 @@ +# Вставка данных + +{% include [work in progress message](../_includes/addition.md) %} + +Below are code examples showing the {{ ydb-short-name }} SDK methods for upserting data: + +{% list tabs %} + +- Go + + + {% include [go.md](_includes/go.md) %} + +{% endlist %} diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md new file mode 100644 index 0000000000..2c8d8246f3 --- /dev/null +++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md @@ -0,0 +1,70 @@ +```go +package main + +import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open( + ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer func() { + _ = db.Close(ctx) + }() + type logMessage struct { + App string + Host string + Timestamp time.Time + HTTPCode uint32 + Message string + } + // prepare native go data + const batchSize = 10000 + logs := make([]logMessage, 0, batchSize) + for i := 0; i < batchSize; i++ { + message := logMessage{ + App: fmt.Sprintf("App_%d", i/256), + Host: fmt.Sprintf("192.168.0.%d", i%256), + Timestamp: time.Now().Add(time.Millisecond * time.Duration(i%1000)), + HTTPCode: 200, + } + if i%2 == 0 { + message.Message = "GET / HTTP/1.1" + } else { + message.Message = "GET /images/logo.png HTTP/1.1" + } + logs = append(logs, message) + } + // execute bulk upsert with native ydb data + err = db.Table().Do( // Do retry operation on errors with best effort + ctx, // context manage exiting from Do + func(ctx context.Context, s table.Session) (err error) { // retry operation + rows := make([]types.Value, 0, len(logs)) + for _, msg := range logs { + rows = append(rows, types.StructValue( + types.StructFieldValue("App", types.UTF8Value(msg.App)), + types.StructFieldValue("Host", types.UTF8Value(msg.Host)), + types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)), + types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)), + types.StructFieldValue("Message", types.UTF8Value(msg.Message)), + )) + } + return s.BulkUpsert(ctx, "/local/bulk_upsert_example", types.ListValue(rows...)) + }, + ) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } +} +``` diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md new file mode 100644 index 0000000000..2264fff951 --- /dev/null +++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md @@ -0,0 +1,16 @@ +# Пакетная вставка данных + +{% include [work in progress message](../_includes/addition.md) %} + +{{ ydb-short-name }} поддерживает пакетную вставку большого количества строк без гарантий атомарности. Запись данных разбивается на несколько независимых транзакций, каждая их которых затрагивает единственную партицию, с параллельным исполнением. За счет этого такой подход более эффективен чем `YQL`. В случае успеха метод `BulkUpsert` гарантирует вставку всех данных, переданных в рамках данного запроса. + +Ниже приведены примеры кода использования встроенных в {{ ydb-short-name }} SDK средств выполнения пакетной вставки: + +{% list tabs %} + +- Go + + + {% include [go.md](_includes/go.md) %} + +{% endlist %} diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml b/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml index e9a0b6ffb4..ac6f587dcf 100644 --- a/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml +++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml @@ -11,4 +11,7 @@ items: include: { mode: link, path: debug/toc_p.yaml } - name: Установить размер пула сессий href: session_pool_limit/index.md - +- name: Вставка данных + href: upsert/index.md +- name: Пакетная вставка данных + href: bulk_upsert/index.md diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md new file mode 100644 index 0000000000..82b74ed301 --- /dev/null +++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md @@ -0,0 +1,59 @@ +```go +package main + +import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open( + ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer func() { + _ = db.Close(ctx) + }() + // execute bulk upsert with native ydb data + err = db.Table().DoTx( // Do retry operation on errors with best effort + ctx, // context manages exiting from Do + func(ctx context.Context, tx table.TransactionActor) (err error) { // retry operation + res, err = tx.Execute( + ctx,` + PRAGMA TablePathPrefix("/path/to/table"); + DECLARE $seriesID AS Uint64; + DECLARE $seasonID AS Uint64; + DECLARE $episodeID AS Uint64; + DECLARE $views AS Uint64; + UPSERT INTO episodes ( series_id, season_id, episode_id, views ) + VALUES ( $seriesID, $seasonID, $episodeID, $views ); + `, + table.NewQueryParameters( + table.ValueParam("$seriesID", types.Uint64Value(1)), + table.ValueParam("$seasonID", types.Uint64Value(1)), + table.ValueParam("$episodeID", types.Uint64Value(1)), + table.ValueParam("$views", types.Uint64Value(1)), // increment views + ), + ) + if err != nil { + return err + } + if err = res.Err(); err != nil { + return err + } + return res.Close() + }, + ) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } +} +``` diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md new file mode 100644 index 0000000000..225e03dccd --- /dev/null +++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md @@ -0,0 +1,14 @@ +# Вставка данных + +{% include [work in progress message](../_includes/addition.md) %} + +Ниже приведены примеры кода использования встроенных в {{ ydb-short-name }} SDK средств выполнения вставки: + +{% list tabs %} + +- Go + + + {% include [go.md](_includes/go.md) %} + +{% endlist %} |