aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAleksey Myasnikov <79263256394@ya.ru>2022-05-11 14:32:27 +0300
committerAleksey Myasnikov <79263256394@ya.ru>2022-05-11 14:32:27 +0300
commit8606684d047c20a14a423f0fe0a4f0e7de670c60 (patch)
tree2a3afd65946e0cdee149b4757f2526d5148e7265
parent75902786de9844f219bb757ddcd4c51e02955cec (diff)
downloadydb-8606684d047c20a14a423f0fe0a4f0e7de670c60.tar.gz
add upsert, bulk-upsert recipes for go
ref:95593950c11279da6dff369c73256b6225739370
-rw-r--r--ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md70
-rw-r--r--ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md16
-rw-r--r--ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml6
-rw-r--r--ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md59
-rw-r--r--ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md14
-rw-r--r--ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md70
-rw-r--r--ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md16
-rw-r--r--ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml5
-rw-r--r--ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md59
-rw-r--r--ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md14
10 files changed, 327 insertions, 2 deletions
diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md
new file mode 100644
index 0000000000..2c8d8246f3
--- /dev/null
+++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md
@@ -0,0 +1,70 @@
+```go
+package main
+
+import (
+ "context"
+ "os"
+
+ "github.com/ydb-platform/ydb-go-sdk/v3"
+)
+
+func main() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ db, err := ydb.Open(
+ ctx,
+ os.Getenv("YDB_CONNECTION_STRING"),
+ ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")),
+ )
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ _ = db.Close(ctx)
+ }()
+ type logMessage struct {
+ App string
+ Host string
+ Timestamp time.Time
+ HTTPCode uint32
+ Message string
+ }
+ // prepare native go data
+ const batchSize = 10000
+ logs := make([]logMessage, 0, batchSize)
+ for i := 0; i < batchSize; i++ {
+ message := logMessage{
+ App: fmt.Sprintf("App_%d", i/256),
+ Host: fmt.Sprintf("192.168.0.%d", i%256),
+ Timestamp: time.Now().Add(time.Millisecond * time.Duration(i%1000)),
+ HTTPCode: 200,
+ }
+ if i%2 == 0 {
+ message.Message = "GET / HTTP/1.1"
+ } else {
+ message.Message = "GET /images/logo.png HTTP/1.1"
+ }
+ logs = append(logs, message)
+ }
+ // execute bulk upsert with native ydb data
+ err = db.Table().Do( // Do retry operation on errors with best effort
+ ctx, // context manage exiting from Do
+ func(ctx context.Context, s table.Session) (err error) { // retry operation
+ rows := make([]types.Value, 0, len(logs))
+ for _, msg := range logs {
+ rows = append(rows, types.StructValue(
+ types.StructFieldValue("App", types.UTF8Value(msg.App)),
+ types.StructFieldValue("Host", types.UTF8Value(msg.Host)),
+ types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)),
+ types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)),
+ types.StructFieldValue("Message", types.UTF8Value(msg.Message)),
+ ))
+ }
+ return s.BulkUpsert(ctx, "/local/bulk_upsert_example", types.ListValue(rows...))
+ },
+ )
+ if err != nil {
+ fmt.Printf("unexpected error: %v", err)
+ }
+}
+```
diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md
new file mode 100644
index 0000000000..58e47d3068
--- /dev/null
+++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md
@@ -0,0 +1,16 @@
+# Bulk-upserting data
+
+{% include [work in progress message](../_includes/addition.md) %}
+
+{{ ydb-short-name }} supports bulk-upserting data without atomicity guarantees. Writing is being performed in multiple independent parallel transactions, within a single partition each. For that reason, `BulkUpsert` is more efficient than `YQL`. When completed successfully, `BulkUpsert` guarantees that all data is written and committed.
+
+Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for bulk-upserting data:
+
+{% list tabs %}
+
+- Go
+
+
+ {% include [go.md](_includes/go.md) %}
+
+{% endlist %}
diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml
index 4443196364..02069df58d 100644
--- a/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml
+++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml
@@ -10,4 +10,8 @@ items:
- name: Troubleshooting
include: { mode: link, path: debug/toc_p.yaml }
- name: Setting the session pool size
- href: session_pool_limit/index.md \ No newline at end of file
+ href: session_pool_limit/index.md
+- name: Upserting data
+ href: upsert/index.md
+- name: Bulk-upserting data
+ href: bulk_upsert/index.md
diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md
new file mode 100644
index 0000000000..82b74ed301
--- /dev/null
+++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md
@@ -0,0 +1,59 @@
+```go
+package main
+
+import (
+ "context"
+ "os"
+
+ "github.com/ydb-platform/ydb-go-sdk/v3"
+)
+
+func main() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ db, err := ydb.Open(
+ ctx,
+ os.Getenv("YDB_CONNECTION_STRING"),
+ ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")),
+ )
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ _ = db.Close(ctx)
+ }()
+ // execute bulk upsert with native ydb data
+ err = db.Table().DoTx( // Do retry operation on errors with best effort
+ ctx, // context manages exiting from Do
+ func(ctx context.Context, tx table.TransactionActor) (err error) { // retry operation
+ res, err = tx.Execute(
+ ctx,`
+ PRAGMA TablePathPrefix("/path/to/table");
+ DECLARE $seriesID AS Uint64;
+ DECLARE $seasonID AS Uint64;
+ DECLARE $episodeID AS Uint64;
+ DECLARE $views AS Uint64;
+ UPSERT INTO episodes ( series_id, season_id, episode_id, views )
+ VALUES ( $seriesID, $seasonID, $episodeID, $views );
+ `,
+ table.NewQueryParameters(
+ table.ValueParam("$seriesID", types.Uint64Value(1)),
+ table.ValueParam("$seasonID", types.Uint64Value(1)),
+ table.ValueParam("$episodeID", types.Uint64Value(1)),
+ table.ValueParam("$views", types.Uint64Value(1)), // increment views
+ ),
+ )
+ if err != nil {
+ return err
+ }
+ if err = res.Err(); err != nil {
+ return err
+ }
+ return res.Close()
+ },
+ )
+ if err != nil {
+ fmt.Printf("unexpected error: %v", err)
+ }
+}
+```
diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md
new file mode 100644
index 0000000000..562af07e2c
--- /dev/null
+++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md
@@ -0,0 +1,14 @@
+# Вставка данных
+
+{% include [work in progress message](../_includes/addition.md) %}
+
+Below are code examples showing the {{ ydb-short-name }} SDK methods for upserting data:
+
+{% list tabs %}
+
+- Go
+
+
+ {% include [go.md](_includes/go.md) %}
+
+{% endlist %}
diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md
new file mode 100644
index 0000000000..2c8d8246f3
--- /dev/null
+++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md
@@ -0,0 +1,70 @@
+```go
+package main
+
+import (
+ "context"
+ "os"
+
+ "github.com/ydb-platform/ydb-go-sdk/v3"
+)
+
+func main() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ db, err := ydb.Open(
+ ctx,
+ os.Getenv("YDB_CONNECTION_STRING"),
+ ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")),
+ )
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ _ = db.Close(ctx)
+ }()
+ type logMessage struct {
+ App string
+ Host string
+ Timestamp time.Time
+ HTTPCode uint32
+ Message string
+ }
+ // prepare native go data
+ const batchSize = 10000
+ logs := make([]logMessage, 0, batchSize)
+ for i := 0; i < batchSize; i++ {
+ message := logMessage{
+ App: fmt.Sprintf("App_%d", i/256),
+ Host: fmt.Sprintf("192.168.0.%d", i%256),
+ Timestamp: time.Now().Add(time.Millisecond * time.Duration(i%1000)),
+ HTTPCode: 200,
+ }
+ if i%2 == 0 {
+ message.Message = "GET / HTTP/1.1"
+ } else {
+ message.Message = "GET /images/logo.png HTTP/1.1"
+ }
+ logs = append(logs, message)
+ }
+ // execute bulk upsert with native ydb data
+ err = db.Table().Do( // Do retry operation on errors with best effort
+ ctx, // context manage exiting from Do
+ func(ctx context.Context, s table.Session) (err error) { // retry operation
+ rows := make([]types.Value, 0, len(logs))
+ for _, msg := range logs {
+ rows = append(rows, types.StructValue(
+ types.StructFieldValue("App", types.UTF8Value(msg.App)),
+ types.StructFieldValue("Host", types.UTF8Value(msg.Host)),
+ types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)),
+ types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)),
+ types.StructFieldValue("Message", types.UTF8Value(msg.Message)),
+ ))
+ }
+ return s.BulkUpsert(ctx, "/local/bulk_upsert_example", types.ListValue(rows...))
+ },
+ )
+ if err != nil {
+ fmt.Printf("unexpected error: %v", err)
+ }
+}
+```
diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md
new file mode 100644
index 0000000000..2264fff951
--- /dev/null
+++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/bulk_upsert/index.md
@@ -0,0 +1,16 @@
+# Пакетная вставка данных
+
+{% include [work in progress message](../_includes/addition.md) %}
+
+{{ ydb-short-name }} поддерживает пакетную вставку большого количества строк без гарантий атомарности. Запись данных разбивается на несколько независимых транзакций, каждая их которых затрагивает единственную партицию, с параллельным исполнением. За счет этого такой подход более эффективен чем `YQL`. В случае успеха метод `BulkUpsert` гарантирует вставку всех данных, переданных в рамках данного запроса.
+
+Ниже приведены примеры кода использования встроенных в {{ ydb-short-name }} SDK средств выполнения пакетной вставки:
+
+{% list tabs %}
+
+- Go
+
+
+ {% include [go.md](_includes/go.md) %}
+
+{% endlist %}
diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml b/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml
index e9a0b6ffb4..ac6f587dcf 100644
--- a/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml
+++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/toc_i.yaml
@@ -11,4 +11,7 @@ items:
include: { mode: link, path: debug/toc_p.yaml }
- name: Установить размер пула сессий
href: session_pool_limit/index.md
-
+- name: Вставка данных
+ href: upsert/index.md
+- name: Пакетная вставка данных
+ href: bulk_upsert/index.md
diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md
new file mode 100644
index 0000000000..82b74ed301
--- /dev/null
+++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/_includes/go.md
@@ -0,0 +1,59 @@
+```go
+package main
+
+import (
+ "context"
+ "os"
+
+ "github.com/ydb-platform/ydb-go-sdk/v3"
+)
+
+func main() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ db, err := ydb.Open(
+ ctx,
+ os.Getenv("YDB_CONNECTION_STRING"),
+ ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")),
+ )
+ if err != nil {
+ panic(err)
+ }
+ defer func() {
+ _ = db.Close(ctx)
+ }()
+ // execute bulk upsert with native ydb data
+ err = db.Table().DoTx( // Do retry operation on errors with best effort
+ ctx, // context manages exiting from Do
+ func(ctx context.Context, tx table.TransactionActor) (err error) { // retry operation
+ res, err = tx.Execute(
+ ctx,`
+ PRAGMA TablePathPrefix("/path/to/table");
+ DECLARE $seriesID AS Uint64;
+ DECLARE $seasonID AS Uint64;
+ DECLARE $episodeID AS Uint64;
+ DECLARE $views AS Uint64;
+ UPSERT INTO episodes ( series_id, season_id, episode_id, views )
+ VALUES ( $seriesID, $seasonID, $episodeID, $views );
+ `,
+ table.NewQueryParameters(
+ table.ValueParam("$seriesID", types.Uint64Value(1)),
+ table.ValueParam("$seasonID", types.Uint64Value(1)),
+ table.ValueParam("$episodeID", types.Uint64Value(1)),
+ table.ValueParam("$views", types.Uint64Value(1)), // increment views
+ ),
+ )
+ if err != nil {
+ return err
+ }
+ if err = res.Err(); err != nil {
+ return err
+ }
+ return res.Close()
+ },
+ )
+ if err != nil {
+ fmt.Printf("unexpected error: %v", err)
+ }
+}
+```
diff --git a/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md
new file mode 100644
index 0000000000..225e03dccd
--- /dev/null
+++ b/ydb/docs/ru/core/reference/ydb-sdk/recipes/upsert/index.md
@@ -0,0 +1,14 @@
+# Вставка данных
+
+{% include [work in progress message](../_includes/addition.md) %}
+
+Ниже приведены примеры кода использования встроенных в {{ ydb-short-name }} SDK средств выполнения вставки:
+
+{% list tabs %}
+
+- Go
+
+
+ {% include [go.md](_includes/go.md) %}
+
+{% endlist %}