diff options
author | alextarazanov <alextarazanov@yandex-team.com> | 2022-11-02 16:58:55 +0300 |
---|---|---|
committer | alextarazanov <alextarazanov@yandex-team.com> | 2022-11-02 16:58:55 +0300 |
commit | 32569a5359dc4482027118462be35f14f9ba5e73 (patch) | |
tree | 87c631575333456c1138612ff13462e3527e3230 | |
parent | f246826629d054f1262d48720cdc14e7276e18ba (diff) | |
download | ydb-32569a5359dc4482027118462be35f14f9ba5e73.tar.gz |
[review] [YDB] Check "add recipes for driver init and go's database/sql" translate
Лог локальной сборки без ошибок
90 files changed, 2325 insertions, 1318 deletions
diff --git a/ydb/docs/en/core/reference/ydb-sdk/_includes/auth.md b/ydb/docs/en/core/reference/ydb-sdk/_includes/auth.md index 1c2d0c9430f..42450eed8b0 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/_includes/auth.md +++ b/ydb/docs/en/core/reference/ydb-sdk/_includes/auth.md @@ -13,7 +13,7 @@ If the token generation object is not defined, the driver won't add any authenti ## Methods for creating token generation objects {#auth-provider} -You can click any of the methods below to go to the source code of an example in the repository. You can also learn about the [authentication code recipes](../recipes/auth/index.md). +You can click any of the methods below to go to the source code of an example in the repository. You can also learn about the [authentication code recipes](../recipes/auth.md). {% list tabs %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/_includes/install.md b/ydb/docs/en/core/reference/ydb-sdk/_includes/install.md index fee42d89236..694fe51b4fe 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/_includes/install.md +++ b/ydb/docs/en/core/reference/ydb-sdk/_includes/install.md @@ -20,7 +20,7 @@ The build process using the source code is described in the source code reposito {% include [install/cmd_go.md](install/cmd_go.md) %} - To ensure the installation is successful, first install [Go](https://go.dev/doc/install) 1.13 or higher in your environment. + To ensure that the installation is successful, make sure that your environment is running [Go](https://go.dev/doc/install) 1.16 or higher. - C# (.NET) diff --git a/ydb/docs/en/core/reference/ydb-sdk/example/archive/example-go-v2.md b/ydb/docs/en/core/reference/ydb-sdk/example/archive/example-go-v2.md index 9bb8752e2b7..950c2031ca3 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/example/archive/example-go-v2.md +++ b/ydb/docs/en/core/reference/ydb-sdk/example/archive/example-go-v2.md @@ -136,28 +136,28 @@ if err != nil { Query results: ```go - var ( - id *uint64 - title *string - date *[]byte - ) + var ( + id *uint64 + title *string + date *[]byte + ) log.Println("> select_simple_transaction:") - for res.NextResultSet(ctx, "series_id", "title", "release_date") { - for res.NextRow() { - err = res.Scan(&id, &title, &date) - if err != nil { - return err - } + for res.NextResultSet(ctx, "series_id", "title", "release_date") { + for res.NextRow() { + err = res.Scan(&id, &title, &date) + if err != nil { + return err + } log.Printf( "# SeriesID: %d , Title: %s, Date: %s\n", *id, *title, *date, ) - } - } - if err = res.Err(); err != nil { - return err - } - return nil + } + } + if err = res.Err(); err != nil { + return err + } + return nil } ``` @@ -209,4 +209,3 @@ Query results: ``` {% include [error_handling.md](../_includes/steps/50_error_handling.md) %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/example/go/index.md b/ydb/docs/en/core/reference/ydb-sdk/example/go/index.md index 8ed7fe660c7..7d828a8135e 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/example/go/index.md +++ b/ydb/docs/en/core/reference/ydb-sdk/example/go/index.md @@ -24,17 +24,19 @@ To work with `YDB` in `Go`, import the `ydb-go-sdk` driver package: ```go import ( - // general imports + // general imports from standard library "context" + "log" "path" // importing the packages ydb-go-sdk "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/table" // needed to work with the table service - "github.com/ydb-platform/ydb-go-sdk/v3/table/options" // needed to work with the table service - "github.com/ydb-platform/ydb-go-sdk/v3/table/result" // needed to work with the table service - "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" // needed to work with the table service - "github.com/ydb-platform/ydb-go-sdk-auth-environ" // needed to authenticate using the environment variables + "github.com/ydb-platform/ydb-go-sdk/v3/table" // needed to work with table service + "github.com/ydb-platform/ydb-go-sdk/v3/table/options" // needed to work with table service + "github.com/ydb-platform/ydb-go-sdk/v3/table/result" // needed to work with table service + "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" // needed to work with table service + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" // needed to work with YDB types and values + "github.com/ydb-platform/ydb-go-sdk-auth-environ" // needed to authenticate using environment variables "github.com/ydb-platform/ydb-go-yc" // to work with YDB in Yandex Cloud ) ``` @@ -48,8 +50,7 @@ dsn := "grpcs://ydb.serverless.yandexcloud.net:2135/?database=/ru-central1/b1g8s // IAM token token := "t1.9euelZrOy8aVmZKJm5HGjceMkMeVj-..." // create a connection object called db, it is an entry point for YDB services -db, err := ydb.Open(ctx, - dsn, +db, err := ydb.Open(ctx, dsn, // yc.WithInternalCA(), // use Yandex Cloud certificates ydb.WithAccessTokenCredentials(token), // authenticate using the token // ydb.WithAnonimousCredentials(), // authenticate anonymously (for example, using docker ydb) @@ -61,9 +62,7 @@ if err != nil { // handle a connection error } // driver must be closed when done -defer func() { - _ = db.Close(ctx) -} +defer db.Close(ctx) ``` The `db` object is an input point for working with `YDB` services. @@ -77,8 +76,7 @@ The session has an exhaustive `API` that lets you perform `DDL`, `DML`, `DQL`, a To create tables, use the `table.Session.CreateTable()` method: ```go -err = db.Table().Do( - ctx, +err = db.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { return s.CreateTable(ctx, path.Join(db.Name(), "series"), options.WithColumn("series_id", types.TypeUint64), // not null column @@ -95,11 +93,10 @@ if err != nil { } ``` -You can use the `table.Session.DescribeTable()` method to output information about the table structure and make sure that it was properly created: +You can use the `table.Session.DescribeTable()` method to print information about the table structure and make sure that it was properly created: ```go -err = db.Table().Do( - ctx, +err = db.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { desc, err := s.DescribeTable(ctx, path.Join(db.Name(), "series")) if err != nil { @@ -113,7 +110,7 @@ err = db.Table().Do( } ) if err != nil { - // handling a situation when a query has failed + // handling query execution failure } ``` @@ -131,8 +128,7 @@ var ( table.CommitTx(), ) ) -err := db.Table().Do( - ctx, +err := db.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { var ( res result.Result @@ -153,17 +149,15 @@ err := db.Table().Do( series WHERE series_id = $seriesID; - `, - table.NewQueryParameters( + ` +, table.NewQueryParameters( table.ValueParam("$seriesID", types.Uint64Value(1)), // insert into the query criteria ), ) if err != nil { return err } - defer func() { - _ = res.Close() // result must be closed - }() + defer res.Close() // result must be closed log.Printf("> select_simple_transaction:\n") for res.NextResultSet(ctx) { for res.NextRow() { @@ -205,8 +199,7 @@ var ( ` res result.StreamResult ) -err = c.Do( - ctx, +err = c.Do(ctx, func(ctx context.Context, s table.Session) (err error) { res, err = s.StreamExecuteScanQuery(ctx, query, table.NewQueryParameters( @@ -221,9 +214,7 @@ err = c.Do( if err != nil { return err } - defer func() { - _ = res.Close() // result must be closed - }() + defer res.Close() // be sure to close the result var ( seriesID uint64 seasonID uint64 diff --git a/ydb/docs/en/core/reference/ydb-sdk/example/java/index.md b/ydb/docs/en/core/reference/ydb-sdk/example/java/index.md index 61626ee6be4..24b2a0c87bb 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/example/java/index.md +++ b/ydb/docs/en/core/reference/ydb-sdk/example/java/index.md @@ -27,8 +27,8 @@ Next, from the same working directory, run the command to start the test app. Th Main driver initialization parameters * A connection string containing details about an [endpoint](../../../../concepts/connect.md#endpoint) and [database](../../../../concepts/connect.md#database). This is the only parameter that is required. -* [Authentication](../../auth.md##auth-provider) provider. Unless explicitly specified, an [anonymous connection](../../../../concepts/auth.md) is used. -* [Session pool](../../recipes/session_pool_limit/index.md) settings +* [Authentication](../../recipes/auth.md##auth-provider) provider. Unless explicitly specified, an [anonymous connection](../../../../concepts/auth.md) is used. +* [Session pool](../../recipes/session-pool-limit.md) settings App code snippet for driver initialization: diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/go_appendix.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/_includes/debug-logs-go-appendix.md index e69de29bb2d..e69de29bb2d 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/go_appendix.md +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/_includes/debug-logs-go-appendix.md diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/_includes/debug-logs-go-sql-appendix.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/_includes/debug-logs-go-sql-appendix.md new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/_includes/debug-logs-go-sql-appendix.md diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-access-token.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-access-token.md new file mode 100644 index 00000000000..7088a14035f --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-access-token.md @@ -0,0 +1,124 @@ +# Authentication using a token + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for authentication using a token in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + {% cut "If you use a connector to create a connection to {{ ydb-short-name }}" %} + + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + + {% endcut %} + + {% cut "If you use a connection string" %} + + ```go + package main + + import ( + "context" + "database/sql" + "os" + + _ "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + db, err := sql.Open("ydb", "grpcs://localohost:2135/local?token="+os.Getenv("YDB_TOKEN")) + if err != nil { + panic(err) + } + defer db.Close() + ... + } + ``` + + {% endcut %} + + +- Java + + ```java + public void work(String connectionString, String accessToken) { + AuthProvider authProvider = CloudAuthProvider.newAuthProvider( + IamTokenCredentialProvider.builder() + .token(accessToken) + .build() + ); + + GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withAuthProvider(authProvider) + .build(); + + TableClient tableClient = TableClient + .newClient(GrpcTableRpc.ownTransport(transport)) + .build()); + + doWork(tableClient); + + tableClient.close(); + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-anonymous.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-anonymous.md new file mode 100644 index 00000000000..1900802fd55 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-anonymous.md @@ -0,0 +1,93 @@ +# Anonymous authentication + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for anonymous authentication in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + By default, anonymous authentication is used. + You can explicitly enable anonymous authentication as follows: + ```go + package main + + import ( + "context" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAnonymousCredentials(), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + By default, anonymous authentication is used. + You can explicitly enable anonymous authentication as follows: + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAnonymousCredentials(), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +- Java + + ```java + public void work(String connectionString) { + AuthProvider authProvider = NopAuthProvider.INSTANCE; + + GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withAuthProvider(authProvider) + .build(); + + TableClient tableClient = TableClient + .newClient(GrpcTableRpc.ownTransport(transport)) + .build()); + + doWork(tableClient); + + tableClient.close(); + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-env.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-env.md new file mode 100644 index 00000000000..b9355312d37 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-env.md @@ -0,0 +1,101 @@ +# Authentication using environment variables + +{% include [work in progress message](_includes/addition.md) %} + +When using this method, the authentication mode and its parameters are defined by the environment that an application is run in, [as described here](../auth.md#env). + +By setting one of the following environment variables, you can control the authentication method: + +* `YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=<path/to/sa_key_file>`: Use a service account file in Yandex Cloud. +* `YDB_ANONYMOUS_CREDENTIALS="1"`: Use anonymous authentication. Relevant for testing against a Docker container with {{ ydb-short-name }}. +* `YDB_METADATA_CREDENTIALS="1"`: Use the metadata service inside Yandex Cloud (a Yandex function or a VM). +* `YDB_ACCESS_TOKEN_CREDENTIALS=<access_token>`: Use token-based authentication. + +Below are examples of the code for authentication using environment variables in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + environ.WithEnvironCredentials(ctx), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + ```go + package main + + import ( + "context" + "database/sql" + "os" + + environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + environ.WithEnvironCredentials(ctx), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +- Java + + ```java + public void work(String connectionString) { + AuthProvider authProvider = CloudAuthHelper.getAuthProviderFromEnviron(); + + GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withAuthProvider(authProvider) + .build(); + + TableClient tableClient = TableClient + .newClient(GrpcTableRpc.ownTransport(transport)) + .build()); + + doWork(tableClient); + + tableClient.close(); + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-metadata.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-metadata.md new file mode 100644 index 00000000000..5cc4e48de84 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-metadata.md @@ -0,0 +1,93 @@ +# Authentication using the metadata service + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for authentication using environment variables in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + yc "github.com/ydb-platform/ydb-go-yc" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + yc.WithMetadataCredentials(ctx), + yc.WithInternalCA(), // append Yandex Cloud certificates + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + yc "github.com/ydb-platform/ydb-go-yc" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + yc.WithMetadataCredentials(ctx), + yc.WithInternalCA(), // append Yandex Cloud certificates + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +- Java + + ```java + public void work(String connectionString) { + AuthProvider authProvider = CloudAuthHelper.getMetadataAuthProvider(); + + GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withAuthProvider(authProvider) + .build()); + + TableClient tableClient = TableClient.newClient(transport).build(); + + doWork(tableClient); + + tableClient.close(); + transport.close(); + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-service-account.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-service-account.md new file mode 100644 index 00000000000..0d2525c80c2 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-service-account.md @@ -0,0 +1,97 @@ +# Authentication using a service account file + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for authentication using a service account file in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + yc "github.com/ydb-platform/ydb-go-yc" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + yc.WithServiceAccountKeyFileCredentials( + os.Getenv("YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"), + ), + yc.WithInternalCA(), // append Yandex Cloud certificates + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + yc "github.com/ydb-platform/ydb-go-yc" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + yc.WithServiceAccountKeyFileCredentials( + os.Getenv("YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"), + ), + yc.WithInternalCA(), // append Yandex Cloud certificates + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +- Java + + ```java + public void work(String connectionString, String saKeyPath) { + AuthProvider authProvider = CloudAuthHelper.getServiceAccountFileAuthProvider(saKeyPath); + + GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withAuthProvider(authProvider) + .build()); + + TableClient tableClient = TableClient.newClient(transport).build(); + + doWork(tableClient); + + tableClient.close(); + transport.close(); + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-static.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-static.md new file mode 100644 index 00000000000..845d5c95737 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth-static.md @@ -0,0 +1,130 @@ +# Username and password based authentication + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for authentication based on a username and token in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- C++ + + ```c++ + auto driverConfig = NYdb::TDriverConfig() + .SetEndpoint(endpoint) + .SetDatabase(database) + .SetCredentialsProviderFactory(NYdb::CreateLoginCredentialsProviderFactory({ + .User = "user", + .Password = "password", + })); + + NYdb::TDriver driver(driverConfig); + ``` + +- Go (native) + + You can pass the username and password in the connection string. For example: + ```shell + "grpcs://login:password@localohost:2135/local" + ``` + + You can also explicitly pass them using the `ydb.WithStaticCredentials` parameter: + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithStaticCredentials("user", "password"), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + You can pass the username and password in the connection string. For example: + ```go + package main + + import ( + "context" + + _ "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + db, err := sql.Open("ydb", "grpcs://login:password@localohost:2135/local") + if err != nil { + panic(err) + } + defer db.Close() + ... + } + ``` + + You can also explicitly pass the username and password at driver initialization via a connector using the `ydb.WithStaticCredentials` parameter: + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithStaticCredentials("user", "password"), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +- Java + + ```java + public void work(String connectionString, String username, String password) { + AuthProvider authProvider = new StaticCredentials(username, password); + + GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withAuthProvider(authProvider) + .build()); + + TableClient tableClient = TableClient.newClient(transport).build(); + + doWork(tableClient); + + tableClient.close(); + transport.close(); + } + ``` + + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth.md new file mode 100644 index 00000000000..a47870511dd --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth.md @@ -0,0 +1,15 @@ +# Authentication + +{% include [work in progress message](_includes/addition.md) %} + +{{ ydb-short-name }} supports multiple authentication methods when connecting to the server side. Each of them is usually specific to a particular pair of environments, that is, depends on where you run your client application (in the trusted {{ ydb-short-name }} zone or outside it) and the {{ ydb-short-name }} server part (in a Docker container, {{ yandex-cloud }}, data cloud, or an independent cluster). + +This section contains code recipes with authentication settings in different {{ ydb-short-name }} SDKs. For a general description of the SDK authentication principles, see the [Authentication in an SDK](auth.md). + +Table of contents: +- [Using a token](auth-access-token.md) +- [Anonymous](auth-anonymous.md) +- [Service account file](auth-service-account.md) +- [Metadata service](auth-metadata.md) +- [Using environment variables](auth-env.md) +- [Username and password based](auth-static.md) diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token.md deleted file mode 100644 index 4620a56fe19..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token.md +++ /dev/null @@ -1,18 +0,0 @@ -# Authentication using a token - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for authentication using a token in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](access_token/go.md) %} - -- Java - - {% include [java.md](access_token/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token/go.md deleted file mode 100644 index dacd18352a6..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token/go.md +++ /dev/null @@ -1,25 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token/java.md deleted file mode 100644 index fd573eabae1..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/access_token/java.md +++ /dev/null @@ -1,16 +0,0 @@ -```java -public void work(String connectionString, String accessToken) { - AuthProvider authProvider = new TokenAuthProvider(accessToken); - - GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(authProvider) - .build()); - - TableClient tableClient = TableClient.newClient(transport).build(); - - doWork(tableClient); - - tableClient.close(); - transport.close(); -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous.md deleted file mode 100644 index c4657264c18..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous.md +++ /dev/null @@ -1,18 +0,0 @@ -# Anonymous authentication - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for anonymous authentication in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](anonymous/go.md) %} - -- Java - - {% include [java.md](anonymous/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous/go.md deleted file mode 100644 index dcb95d4a3cc..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous/go.md +++ /dev/null @@ -1,24 +0,0 @@ -```go -package main - -import ( - "context" - - "github.com/ydb-platform/ydb-go-sdk/v3" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithAnonymousCredentials(), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous/java.md deleted file mode 100644 index 001243f50bc..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/anonymous/java.md +++ /dev/null @@ -1,16 +0,0 @@ -```java -public void work(String connectionString) { - AuthProvider authProvider = NopAuthProvider.INSTANCE; - - GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(authProvider) - .build()); - - TableClient tableClient = TableClient.newClient(transport).build(); - - doWork(tableClient); - - tableClient.close(); - transport.close(); -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env.md deleted file mode 100644 index c06b896b06d..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env.md +++ /dev/null @@ -1,27 +0,0 @@ -# Authentication using environment variables - -{% include [work in progress message](../../_includes/addition.md) %} - -When using this method, the authentication mode and its parameters are defined by the environment that an application is run in, [as described here](../../../auth.md#env). - -By setting one of the following environment variables, you can control the authentication method: - -* `YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=<path/to/sa_key_file>`: Use a service account file in Yandex Cloud. -* `YDB_ANONYMOUS_CREDENTIALS="1"`: Use anonymous authentication. Relevant for testing against a Docker container with {{ ydb-short-name }}. -* `YDB_METADATA_CREDENTIALS="1"`: Use the metadata service inside Yandex Cloud (a Yandex function or a VM). -* `YDB_ACCESS_TOKEN_CREDENTIALS=<access_token>`: Use token-based authentication. - -Below are examples of the code for authentication using environment variables in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](env/go.md) %} - -- Java - - {% include [java.md](env/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env/go.md deleted file mode 100644 index 3d5ab05993e..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env/go.md +++ /dev/null @@ -1,26 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" - "github.com/ydb-platform/ydb-go-sdk/v3" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - environ.WithEnvironCredentials(ctx), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env/java.md deleted file mode 100644 index 75a11980281..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/env/java.md +++ /dev/null @@ -1,16 +0,0 @@ -```java -public void work(String connectionString) { - AuthProvider authProvider = CloudAuthHelper.getAuthProviderFromEnviron(); - - GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(authProvider) - .build()); - - TableClient tableClient = TableClient.newClient(transport).build(); - - doWork(tableClient); - - tableClient.close(); - transport.close(); -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/index.md deleted file mode 100644 index 9de0e174fb5..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/index.md +++ /dev/null @@ -1,8 +0,0 @@ -# Authentication - -{% include [work in progress message](../../_includes/addition.md) %} - -{{ ydb-short-name }} supports multiple authentication methods when connecting to the server side. Each of them is usually specific to a particular environment pair: where the client application is located (in the trusted {{ ydb-short-name }} zone or outside it) and the {{ ydb-short-name }} server side (a Docker container, {{ yandex-cloud }}, data cloud, or deployment on a separate cluster). - -This section contains code recipes with authentication settings in different {{ ydb-short-name }} SDKs. For a general description of the SDK authentication principles, see the [Authentication in an SDK](../../../auth.md) article. - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata.md deleted file mode 100644 index 0cdc2a2bc54..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata.md +++ /dev/null @@ -1,18 +0,0 @@ -# Authentication using the metadata service - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for authentication using environment variables in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](metadata/go.md) %} - -- Java - - {% include [java.md](metadata/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata/go.md deleted file mode 100644 index ac77d35c623..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata/go.md +++ /dev/null @@ -1,27 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - yc "github.com/ydb-platform/ydb-go-yc" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - yc.WithMetadataCredentials(ctx), - yc.WithInternalCA(), // append Yandex Cloud certificates - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata/java.md deleted file mode 100644 index 8d90b551d82..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/metadata/java.md +++ /dev/null @@ -1,16 +0,0 @@ -```java -public void work(String connectionString) { - AuthProvider authProvider = CloudAuthHelper.getMetadataAuthProvider(); - - GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(authProvider) - .build()); - - TableClient tableClient = TableClient.newClient(transport).build(); - - doWork(tableClient); - - tableClient.close(); - transport.close(); -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account.md deleted file mode 100644 index 94d56c8980e..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account.md +++ /dev/null @@ -1,18 +0,0 @@ -# Authentication using a service account file - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for authentication using a service account file in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](service_account/go.md) %} - -- Java - - {% include [java.md](service_account/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account/go.md deleted file mode 100644 index 1a5d26021a0..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account/go.md +++ /dev/null @@ -1,29 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - yc "github.com/ydb-platform/ydb-go-yc" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - yc.WithServiceAccountKeyFileCredentials( - os.Getenv("YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"), - ), - yc.WithInternalCA(), // append Yandex Cloud certificates - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account/java.md deleted file mode 100644 index 8467eb57167..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/_includes/service_account/java.md +++ /dev/null @@ -1,16 +0,0 @@ -```java -public void work(String connectionString, String saKeyPath) { - AuthProvider authProvider = CloudAuthHelper.getServiceAccountFileAuthProvider(saKeyPath); - - GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(authProvider) - .build()); - - TableClient tableClient = TableClient.newClient(transport).build(); - - doWork(tableClient); - - tableClient.close(); - transport.close(); -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/access_token.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/access_token.md deleted file mode 100644 index 606d617e7d0..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/access_token.md +++ /dev/null @@ -1,3 +0,0 @@ - -{% include [access_token.md](_includes/access_token.md) %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/anonymous.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/anonymous.md deleted file mode 100644 index 49b00e9286b..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/anonymous.md +++ /dev/null @@ -1,2 +0,0 @@ - -{% include [anonymous.md](_includes/anonymous.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/env.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/env.md deleted file mode 100644 index 004e4f5d392..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/env.md +++ /dev/null @@ -1 +0,0 @@ -{% include [index.md](_includes/env.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/index.md deleted file mode 100644 index bd4e95cb411..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/index.md +++ /dev/null @@ -1,3 +0,0 @@ - -{% include [index.md](_includes/index.md) %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/metadata.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/metadata.md deleted file mode 100644 index d15b521addf..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/metadata.md +++ /dev/null @@ -1 +0,0 @@ -{% include [index.md](_includes/metadata.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/service_account.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/service_account.md deleted file mode 100644 index 70996edf715..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/service_account.md +++ /dev/null @@ -1 +0,0 @@ -{% include [index.md](_includes/service_account.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/static.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/static.md deleted file mode 100644 index 2b3ea4dfa72..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/static.md +++ /dev/null @@ -1,51 +0,0 @@ -# Username and password based authentication - -{% include [work in progress message](../_includes/addition.md) %} - -Below are examples of the code for authentication based on a username and token in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- C++ - - ```c++ - auto driverConfig = NYdb::TDriverConfig() - .SetEndpoint(endpoint) - .SetDatabase(database) - .SetCredentialsProviderFactory(NYdb::CreateLoginCredentialsProviderFactory({ - .User = "user", - .Password = "password", - })); - - NYdb::TDriver driver(driverConfig); - ``` - -- Go - - ```go - package main - - import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithStaticCredentials("user", "password"), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - } - ``` - -{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/toc_i.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/toc_i.yaml deleted file mode 100644 index e6b09bf483f..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/toc_i.yaml +++ /dev/null @@ -1,15 +0,0 @@ -items: -- name: Overview - href: index.md -- name: Using a token - href: access_token.md -- name: Anonymous - href: anonymous.md -- name: Service account file - href: service_account.md -- name: Metadata service - href: metadata.md -- name: Using environment variables - href: env.md -- name: Username and password based - href: static.md
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/toc_p.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/toc_p.yaml deleted file mode 100644 index 5bfec4365de..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/auth/toc_p.yaml +++ /dev/null @@ -1,2 +0,0 @@ -items: -- include: { mode: link, path: toc_i.yaml }
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-prefer-local.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-prefer-local.md new file mode 100644 index 00000000000..eb6aea59d5d --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-prefer-local.md @@ -0,0 +1,85 @@ +# Prefer the nearest data center + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for setting the "prefer the nearest data center" balancing algorithm option in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithBalancer( + balancers.PreferLocalDC( + balancers.RandomChoice(), + ), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + Client load balancing in the {{ ydb-short-name }} `database/sql` driver is performed only when establishing a new connection (in `database/sql` terms), which is a {{ ydb-short-name }} session on a specific node. Once the session is created, all queries in this session are passed to the node where the session was created. Queries in the same {{ ydb-short-name }} session are not balanced between different {{ ydb-short-name }} nodes. + + Example of the code for setting the "prefer the nearest data center" balancing algorithm option: + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithBalancer( + balancers.PreferLocalDC( + balancers.RandomChoice(), + ), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-prefer-location.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-prefer-location.md new file mode 100644 index 00000000000..6f0d1c37967 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-prefer-location.md @@ -0,0 +1,89 @@ +# Prefer the specific availability zone + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for setting the "prefer the availability zone" balancing algorithm option in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithBalancer( + balancers.PreferLocations( + balancers.RandomChoice(), + "a", + "b", + ), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + Client load balancing in the {{ ydb-short-name }} `database/sql` driver is performed only when establishing a new connection (in `database/sql` terms), which is a {{ ydb-short-name }} session on a specific node. Once the session is created, all queries in this session are passed to the node where the session was created. Queries in the same {{ ydb-short-name }} session are not balanced between different {{ ydb-short-name }} nodes. + + Example of the code for setting the "prefer the availability zone": + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithBalancer( + balancers.PreferLocations( + balancers.RandomChoice(), + "a", + "b", + ), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-random-choice.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-random-choice.md new file mode 100644 index 00000000000..2321e734f1d --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing-random-choice.md @@ -0,0 +1,83 @@ +# Random choice + +{% include [work in progress message](_includes/addition.md) %} + +The {{ ydb-short-name }} SDK uses the `random_choice` algorithm by default. + +Below are examples of the code for forced setting of the "random choice" balancing algorithm in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithBalancer( + balancers.RandomChoice(), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + Client load balancing in the {{ ydb-short-name }} `database/sql` driver is performed only when establishing a new connection (in `database/sql` terms), which is a {{ ydb-short-name }} session on a specific node. Once the session is created, all queries in this session are passed to the node where the session was created. Queries in the same {{ ydb-short-name }} session are not balanced between different {{ ydb-short-name }} nodes. + + Example of the code for setting the "random choice" balancing algorithm: + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithBalancer( + balancers.RandomChoice(), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + +{% endlist %}
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing.md new file mode 100644 index 00000000000..ab88e6dedd2 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing.md @@ -0,0 +1,16 @@ + +# Balancing + +{% include [work in progress message](_includes/addition.md) %} + +{{ ydb-short-name }} uses client load balancing because it is more efficient when a lot of traffic from multiple client applications comes to a database. +In most cases, it just works in the {{ ydb-short-name }} SDK. However, sometimes specific settings for client load balancing are required, for example, to reduce server hops and request time or to distribute the load across availability zones. + +Note that custom balancing is limited when it comes to {{ ydb-short-name }} sessions. Custom balancing in the {{ ydb-short-name }} SDKs is performed only when creating a new {{ ydb-short-name }} session on a specific node. Once the session is created, all queries in this session are passed to the node where the session was created. Queries in the same {{ ydb-short-name }} session are not balanced between different {{ ydb-short-name }} nodes. + +This section contains code recipes with client load balancing settings in different {{ ydb-short-name }} SDKs. + +Table of contents: +- [Random choice](balancing-random-choice.md) +- [Prefer the nearest data center](balancing-prefer-local.md) +- [Prefer the specific availability zone](balancing-prefer-location.md) diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/index.md deleted file mode 100644 index be77aa2245a..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/index.md +++ /dev/null @@ -1,9 +0,0 @@ -# Balancing - -{% include [work in progress message](../../_includes/addition.md) %} - -{{ ydb-short-name }} uses client load balancing because it is more efficient when a lot of traffic from multiple client applications comes to a database. -In most cases, it just works in the {{ ydb-short-name }} SDK. However, sometimes specific settings for client load balancing are required, for example, to reduce server hops and request time or to distribute the load across availability zones. - -This section contains code recipes with client load balancing settings in different {{ ydb-short-name }} SDKs. - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_local.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_local.md deleted file mode 100644 index 9a507843230..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_local.md +++ /dev/null @@ -1,14 +0,0 @@ -# Prefer the nearest data center - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for setting the "prefer the nearest data center" balancing algorithm option in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](prefer_local/go.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_local/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_local/go.md deleted file mode 100644 index fc43c67df84..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_local/go.md +++ /dev/null @@ -1,30 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/balancers" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithBalancer( - balancers.PreferLocalDC( - balancers.RandomChoice(), - ), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_location.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_location.md deleted file mode 100644 index b93efe00d8b..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_location.md +++ /dev/null @@ -1,14 +0,0 @@ -# Prefer the availability zone - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for setting the "prefer the availability zone" balancing algorithm option in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](prefer_location/go.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_location/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_location/go.md deleted file mode 100644 index 7d360849b94..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/prefer_location/go.md +++ /dev/null @@ -1,32 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/balancers" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithBalancer( - balancers.PreferLocations( - balancers.RandomChoice(), - "a", - "b", - ), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/random_choice.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/random_choice.md deleted file mode 100644 index a42b83814fa..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/random_choice.md +++ /dev/null @@ -1,16 +0,0 @@ -# Random choice - -{% include [work in progress message](../../_includes/addition.md) %} - -The {{ ydb-short-name }} SDK uses the `random_choice` algorithm by default. - -Below are examples of the code for forced setting of the "random choice" balancing algorithm in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](random_choice/go.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/random_choice/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/random_choice/go.md deleted file mode 100644 index a52aaa7827b..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/_includes/random_choice/go.md +++ /dev/null @@ -1,28 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/balancers" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithBalancer( - balancers.RandomChoice(), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/index.md deleted file mode 100644 index e5687972c21..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/index.md +++ /dev/null @@ -1,3 +0,0 @@ - -{% include [balancing.md](_includes/index.md) %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/prefer_local.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/prefer_local.md deleted file mode 100644 index ba06e91a9af..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/prefer_local.md +++ /dev/null @@ -1 +0,0 @@ -{% include [index.md](_includes/prefer_local.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/prefer_location.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/prefer_location.md deleted file mode 100644 index 825e8ccb448..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/prefer_location.md +++ /dev/null @@ -1 +0,0 @@ -{% include [index.md](_includes/prefer_location.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/random_choice.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/random_choice.md deleted file mode 100644 index ecd24b7afc9..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/random_choice.md +++ /dev/null @@ -1 +0,0 @@ -{% include [index.md](_includes/random_choice.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/toc_i.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/toc_i.yaml deleted file mode 100644 index 780d63a598c..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/toc_i.yaml +++ /dev/null @@ -1,9 +0,0 @@ -items: -- name: Overview - href: index.md -- name: Random choice - href: random_choice.md -- name: Prefer the nearest data center - href: prefer_local.md -- name: Prefer the availability zone - href: prefer_location.md
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/toc_p.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/toc_p.yaml deleted file mode 100644 index 5bfec4365de..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/balancing/toc_p.yaml +++ /dev/null @@ -1,2 +0,0 @@ -items: -- include: { mode: link, path: toc_i.yaml }
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk-upsert.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk-upsert.md new file mode 100644 index 00000000000..a005235b04e --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk-upsert.md @@ -0,0 +1,89 @@ +# Bulk upsert of data + +{% include [work in progress message](_includes/addition.md) %} + +{{ ydb-short-name }} supports bulk upsert of many records without atomicity guarantees. The upsert process is split into multiple independent parallel transactions, each covering a single partition. For that reason, this approach is more effective than using `YQL`. If successful, the `BulkUpsert` method guarantees inserting all the data transmitted by the query. + +Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for bulk upsert: + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + type logMessage struct { + App string + Host string + Timestamp time.Time + HTTPCode uint32 + Message string + } + // prepare native go data + const batchSize = 10000 + logs := make([]logMessage, 0, batchSize) + for i := 0; i < batchSize; i++ { + message := logMessage{ + App: fmt.Sprintf("App_%d", i/256), + Host: fmt.Sprintf("192.168.0.%d", i%256), + Timestamp: time.Now().Add(time.Millisecond * time.Duration(i%1000)), + HTTPCode: 200, + } + if i%2 == 0 { + message.Message = "GET / HTTP/1.1" + } else { + message.Message = "GET /images/logo.png HTTP/1.1" + } + logs = append(logs, message) + } + // execute bulk upsert with native ydb data + err = db.Table().Do( // Do retry operation on errors with best effort + ctx, // context manage exiting from Do + func(ctx context.Context, s table.Session) (err error) { // retry operation + rows := make([]types.Value, 0, len(logs)) + for _, msg := range logs { + rows = append(rows, types.StructValue( + types.StructFieldValue("App", types.UTF8Value(msg.App)), + types.StructFieldValue("Host", types.UTF8Value(msg.Host)), + types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)), + types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)), + types.StructFieldValue("Message", types.UTF8Value(msg.Message)), + )) + } + return s.BulkUpsert(ctx, "/local/bulk_upsert_example", types.ListValue(rows...)) + }, + ) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } + } + ``` + +- Go (database/sql) + + The implementation of {{ ydb-short-name }} `database/sql` doesn't support bulk nontransactional upsert of data. + For bulk upsert, use [transactional upsert](./upsert.md). + + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md deleted file mode 100644 index 4f7e25ffa62..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/_includes/go.md +++ /dev/null @@ -1,71 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/table" - "github.com/ydb-platform/ydb-go-sdk/v3/table/types" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - type logMessage struct { - App string - Host string - Timestamp time.Time - HTTPCode uint32 - Message string - } - // prepare native go data - const batchSize = 10000 - logs := make([]logMessage, 0, batchSize) - for i := 0; i < batchSize; i++ { - message := logMessage{ - App: fmt.Sprintf("App_%d", i/256), - Host: fmt.Sprintf("192.168.0.%d", i%256), - Timestamp: time.Now().Add(time.Millisecond * time.Duration(i%1000)), - HTTPCode: 200, - } - if i%2 == 0 { - message.Message = "GET / HTTP/1.1" - } else { - message.Message = "GET /images/logo.png HTTP/1.1" - } - logs = append(logs, message) - } - // execute bulk upsert with native ydb data - err = db.Table().Do( // Do retry operation on errors with best effort - ctx, // context manages exiting from Do - func(ctx context.Context, s table.Session) (err error) { // retry operation - rows := make([]types.Value, 0, len(logs)) - for _, msg := range logs { - rows = append(rows, types.StructValue( - types.StructFieldValue("App", types.UTF8Value(msg.App)), - types.StructFieldValue("Host", types.UTF8Value(msg.Host)), - types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)), - types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)), - types.StructFieldValue("Message", types.UTF8Value(msg.Message)), - )) - } - return s.BulkUpsert(ctx, "/local/bulk_upsert_example", types.ListValue(rows...)) - }, - ) - if err != nil { - fmt.Printf("unexpected error: %v", err) - } -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md deleted file mode 100644 index 5d7268a66fc..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/bulk_upsert/index.md +++ /dev/null @@ -1,15 +0,0 @@ -# Batch data insertion - -{% include [work in progress message](../_includes/addition.md) %} - -{{ ydb-short-name }} supports bulk-upserting data without atomicity guarantees. Writing is being performed in multiple independent parallel transactions, within a single partition each. For that reason, `BulkUpsert` is more efficient than `YQL`. When completed successfully, `BulkUpsert` guarantees that all data is written and committed. - -Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for bulk-upserting data: - -{% list tabs %} - -- Go - - {% include [go.md](_includes/go.md) %} - -{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-jaeger.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-jaeger.md new file mode 100644 index 00000000000..2129febbe47 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-jaeger.md @@ -0,0 +1,139 @@ +# Enabling tracing in Jaeger + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code enabling Jaeger tracing in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "time" + + "github.com/opentracing/opentracing-go" + jaegerConfig "github.com/uber/jaeger-client-go/config" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + + tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" + ) + + const ( + tracerURL = "localhost:5775" + serviceName = "ydb-go-sdk" + ) + + func main() { + tracer, closer, err := jaegerConfig.Configuration{ + ServiceName: serviceName, + Sampler: &jaegerConfig.SamplerConfig{ + Type: "const", + Param: 1, + }, + Reporter: &jaegerConfig.ReporterConfig{ + LogSpans: true, + BufferFlushInterval: 1 * time.Second, + LocalAgentHostPort: tracerURL, + }, + }.NewTracer() + if err != nil { + panic(err) + } + + defer closer.Close() + + // set global tracer of this application + opentracing.SetGlobalTracer(tracer) + + span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") + defer span.Finish() + + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + tracing.WithTraces(tracing.WithDetails(trace.DetailsAll)), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + ```go + package main + + import ( + "context" + "database/sql" + "time" + + "github.com/opentracing/opentracing-go" + jaegerConfig "github.com/uber/jaeger-client-go/config" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + + tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" + ) + + const ( + tracerURL = "localhost:5775" + serviceName = "ydb-go-sdk" + ) + + func main() { + tracer, closer, err := jaegerConfig.Configuration{ + ServiceName: serviceName, + Sampler: &jaegerConfig.SamplerConfig{ + Type: "const", + Param: 1, + }, + Reporter: &jaegerConfig.ReporterConfig{ + LogSpans: true, + BufferFlushInterval: 1 * time.Second, + LocalAgentHostPort: tracerURL, + }, + }.NewTracer() + if err != nil { + panic(err) + } + + defer closer.Close() + + // set global tracer of this application + opentracing.SetGlobalTracer(tracer) + + span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") + defer span.Finish() + + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + tracing.WithTraces(tracing.WithDetails(trace.DetailsAll)), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpnDB(connector) + defer db.Close() + ... + } + ``` + + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-logs.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-logs.md new file mode 100644 index 00000000000..6abb0e3444f --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-logs.md @@ -0,0 +1,341 @@ +# Enabling logging + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of code that enables logging in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + There are several ways to enable logs in an application that uses `ydb-go-sdk`: + + {% cut "Using the `YDB_LOG_SEVERITY_LEVEL` environment variable" %} + + This environment variable enables the built-in `ydb-go-sdk` logger (synchronous, non-block) and prints to the standard output stream. + You can set the environment variable as follows: + ```shell + export YDB_LOG_SEVERITY_LEVEL=info + ``` + (possible values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and `quiet`, defaults to `quiet`). + + {% endcut %} + + {% cut "Enable a third-party logger `go.uber.org/zap`" %} + ```go + package main + + import ( + "context" + "os" + + "go.uber.org/zap" + + ydbZap "github.com/ydb-platform/ydb-go-sdk-zap" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var log *zap.Logger // zap-logger with init out of this scope + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydbZap.WithTraces( + log, + ydbZap.WithDetails(trace.DetailsAll), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + {% endcut %} + + {% cut "Enable a third-party logger `github.com/rs/zerolog`" %} + ```go + package main + + import ( + "context" + "os" + + "github.com/rs/zerolog" + + ydbZerolog "github.com/ydb-platform/ydb-go-sdk-zerolog" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var log zerolog.Logger // zap-logger with init out of this scope + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydbZerolog.WithTraces( + log, + ydbZerolog.WithDetails(trace.DetailsAll), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + {% endcut %} + + {% include [overlay](_includes/debug-logs-go-appendix.md) %} + + {% cut "Enable a custom logger implementation `github.com/ydb-platform/ydb-go-sdk/v3/log.Logger`" %} + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/log" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var logger log.Logger // logger implementation with init out of this scope + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithLogger( + trace.DetailsAll, + ydb.WithExternalLogger(logger), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + {% endcut %} + + {% cut "Implement your own logging package" %} + + You can implement your own logging package based on the driver events in the `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package. The `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package describes all logged driver events. + + {% endcut %} + +- Go (database/sql) + + There are several ways to enable logs in an application that uses `ydb-go-sdk`: + + {% cut "Using the `YDB_LOG_SEVERITY_LEVEL` environment variable" %} + + This environment variable enables the built-in `ydb-go-sdk` logger (synchronous, non-block) and prints to the standard output stream. + You can set the environment variable as follows: + ```shell + export YDB_LOG_SEVERITY_LEVEL=info + ``` + (possible values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and `quiet`, defaults to `quiet`). + + {% endcut %} + + {% cut "Enable a third-party logger `go.uber.org/zap`" %} + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "go.uber.org/zap" + + ydbZap "github.com/ydb-platform/ydb-go-sdk-zap" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var log *zap.Logger // zap-logger with init out of this scope + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydbZap.WithTraces( + log, + ydbZap.WithDetails(trace.DetailsAll), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpnDB(connector) + defer db.Close() + ... + } + ``` + {% endcut %} + + {% cut "Enable a third-party logger `github.com/rs/zerolog`" %} + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/rs/zerolog" + + ydbZerolog "github.com/ydb-platform/ydb-go-sdk-zerolog" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var log zerolog.Logger // zap-logger with init out of this scope + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydbZerolog.WithTraces( + log, + ydbZerolog.WithDetails(trace.DetailsAll), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpnDB(connector) + defer db.Close() + ... + } + ``` + {% endcut %} + + {% include [overlay](_includes/debug-logs-go-sql-appendix.md) %} + + {% cut "Enable a custom logger implementation `github.com/ydb-platform/ydb-go-sdk/v3/log.Logger`" %} + ```go + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/log" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var logger log.Logger // logger implementation with init out of this scope + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithLogger( + trace.DetailsAll, + ydb.WithExternalLogger(logger), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpnDB(connector) + defer db.Close() + ... + } + ``` + {% endcut %} + + {% cut "Implement your own logging package" %} + + You can implement your own logging package based on the driver events in the `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package. The `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package describes all logged driver events. + + {% endcut %} + +- Java + + For logging purposes, the {{ ydb-short-name }} Java SDK uses the slf4j library, which supports multiple logging levels (`error`, `warn`, `info`, `debug`, `trace`) for one or many loggers. The current implementation supports the following loggers: + + * The `com.yandex.ydb.core.grpc` logger provides information about the internal implementation of the gRPC protocol + * The `debug` level logs all operations run over gRPC, so we recommend using it only for debugging purposes. + * The `info` level is recommended by default + + * On the `debug` level, the `com.yandex.ydb.table.impl` logger enables you to track the internal status of the YDB driver, including session pool health. + + * On the `debug` level, the `com.yandex.ydb.table.SessionRetryContext` logger will inform you of the number of retries, results of executed queries, execution time of specific retries, and the total operation execution time + + * On the `debug` level, the `com.yandex.ydb.table.Session` logger gives you details about the query text, response status, and execution time of specific operations within the session + + + Enabling and configuration the Java SDK loggers depends on the `slf4j-api` implementation used. + Here's an example of a `log4j2` configuration for the `log4j-slf4j-impl` library + + ```xml + <Configuration status="WARN"> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> + </Console> + </Appenders> + + <Loggers> + <Logger name="io.netty" level="warn" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + <Logger name="io.grpc.netty" level="warn" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + <Logger name="com.yandex.ydb.core.grpc" level="info" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + <Logger name="com.yandex.ydb.table.impl" level="info" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + <Logger name="com.yandex.ydb.table.SessionRetryContext" level="debug" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + <Logger name="com.yandex.ydb.table.Session" level="debug" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + + <Root level="debug" > + <AppenderRef ref="Console"/> + </Root> + </Loggers> + </Configuration> + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-prometheus.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-prometheus.md new file mode 100644 index 00000000000..c49d5a2defb --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug-prometheus.md @@ -0,0 +1,85 @@ +# Enabling metrics in Prometheus + +{% include [work in progress message](_includes/addition.md) %} + +Below are examples of the code for enabling metrics in Prometheus in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + metrics "github.com/ydb-platform/ydb-go-sdk-prometheus" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx := context.Background() + registry := prometheus.NewRegistry() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + metrics.WithTraces( + registry, + metrics.WithDetails(trace.DetailsAll), + metrics.WithSeparator("_"), + ), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + ```go + package main + + import ( + "context" + "database/sql" + + "github.com/prometheus/client_golang/prometheus" + metrics "github.com/ydb-platform/ydb-go-sdk-prometheus" + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/trace" + ) + + func main() { + ctx := context.Background() + registry := prometheus.NewRegistry() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + metrics.WithTraces( + registry, + metrics.WithDetails(trace.DetailsAll), + metrics.WithSeparator("_"), + ), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + + db := sql.OpnDB(connector) + defer db.Close() + ... + } + ``` + + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug.md index b59dd7a7e25..0989d386e30 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/index.md +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug.md @@ -1,8 +1,13 @@ # Troubleshooting -{% include [work in progress message](../../_includes/addition.md) %} +{% include [work in progress message](_includes/addition.md) %} When troubleshooting issues with {{ ydb-short-name }}, diagnostics tools such as logging, metrics, OpenTracing/Jaeger tracing are helpful. We strongly recommend that you enable them in advance before any problems occur. This will help see changes in the overall picture before, during, and after an issue when troubleshooting it. This greatly speeds up our investigation into incidents and lets us provide assistance much faster. This section contains code recipes for enabling diagnostics tools in different {{ ydb-short-name }} SDKs. +Table of contents: +- [Enable logging](debug-logs.md) +- [Enable metrics in Prometheus](debug-prometheus.md) +- [Enable tracing in Jaeger](debug-jaeger.md) + diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/jaeger.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/jaeger.md deleted file mode 100644 index 45e762184ef..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/jaeger.md +++ /dev/null @@ -1,14 +0,0 @@ -# Enabling tracing in Jaeger - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code enabling Jaeger tracing in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](jaeger/go.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/jaeger/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/jaeger/go.md deleted file mode 100644 index a99be9ede99..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/jaeger/go.md +++ /dev/null @@ -1,60 +0,0 @@ -```go -package main - -import ( - "context" - "time" - - "github.com/opentracing/opentracing-go" - jaegerConfig "github.com/uber/jaeger-client-go/config" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" -) - -const ( - tracerURL = "localhost:5775" - serviceName = "ydb-go-sdk" -) - -func main() { - tracer, closer, err := jaegerConfig.Configuration{ - ServiceName: serviceName, - Sampler: &jaegerConfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jaegerConfig.ReporterConfig{ - LogSpans: true, - BufferFlushInterval: 1 * time.Second, - LocalAgentHostPort: tracerURL, - }, - }.NewTracer() - if err != nil { - panic(err) - } - - defer closer.Close() - - // set global tracer of this application - opentracing.SetGlobalTracer(tracer) - - span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") - defer span.Finish() - - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - tracing.WithTraces( - tracing.WithDetails(trace.DetailsAll), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs.md deleted file mode 100644 index ce23b98d821..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs.md +++ /dev/null @@ -1,18 +0,0 @@ -# Enabling logging - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of code that enables logging in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](logs/go.md) %} - -- Java - - {% include [java.md](logs/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/go.md deleted file mode 100644 index a34b657302f..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/go.md +++ /dev/null @@ -1,117 +0,0 @@ -There are several ways to enable logs in an application that uses `ydb-go-sdk`: - -* Set the environment variable `YDB_LOG_SEVERITY_LEVEL=info` (possible values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and `quiet`, defaults to `quiet`). - This environment variable enables the built-in `ydb-go-sdk` logger (synchronous, non-block) with output to the standard output stream. -* {% cut "Connect a third-party logger `go.uber.org/zap`" %} - - ```go - package main - - import ( - "context" - - "go.uber.org/zap" - - ydbZap "github.com/ydb-platform/ydb-go-sdk-zap" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var log *zap.Logger // zap-logger with init out of this scope - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbZap.WithTraces( - log, - ydbZap.WithDetails(trace.DetailsAll), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - } - ``` - - {% endcut %} -* {% cut "Connect a third-party logger `go.uber.org/zap`" %} - - ```go - package main - - import ( - "context" - "os" - - "github.com/rs/zerolog" - - ydbZerolog "github.com/ydb-platform/ydb-go-sdk-zerolog" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var log zerolog.Logger // zap-logger with init out of this scope - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbZerolog.WithTraces( - log, - ydbZerolog.WithDetails(trace.DetailsAll), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - } - ``` - - {% endcut %} -* {% cut "Connect your own logger implementation `github.com/ydb-platform/ydb-go-sdk/v3/log.Logger`" %} - - ```go - package main - - import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/log" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var logger log.Logger // logger implementation with init out of this scope - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithLogger( - trace.DetailsAll, - ydb.WithExternalLogger(logger), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - } - ``` - - {% endcut %} - -{% include [overlay](go_appendix.md) %} - -* Implement your own logging package based on the `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package. - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/java.md deleted file mode 100644 index aa7273385ce..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/logs/java.md +++ /dev/null @@ -1,50 +0,0 @@ -In the {{ ydb-short-name }} Java SDK, logging is done using the slf4j library. It lets you use different logging levels (`error`, `warn`, `info`, `debug`, and `trace`) for one or more loggers. The following loggers are available in the current implementation: - -* The `tech.ydb.core.grpc` logger provides information about the internal implementation of gRPC. - * At the `debug` level, all operations are logged using gRPC. This level is recommended for debugging only. - * Use the `info` level by default. - -* The `tech.ydb.table.impl` logger at the `debug` level lets you monitor the internal state of the ydb driver, including session pool performance. - -* The `tech.ydb.table.SessionRetryContext` logger at the `debug` level provides information about the number of retries, the results of executed queries, the execution time of individual retries, and the total execution time of the entire operation. - -* The `tech.ydb.table.Session` logger at the `debug` level provides information about the query text, response status, and execution time for various session operations. - -Enabling and configuring Java SDK loggers depends on the `slf4j-api` implementation used. -Below is an example of the `log4j2` configuration for the `log4j-slf4j-impl` library. - -```xml -<Configuration status="WARN"> - <Appenders> - <Console name="Console" target="SYSTEM_OUT"> - <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> - </Console> - </Appenders> - - <Loggers> - <Logger name="io.netty" level="warn" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="io.grpc.netty" level="warn" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.core.grpc" level="info" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.table.impl" level="info" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.table.SessionRetryContext" level="debug" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.table.Session" level="debug" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - - <Root level="debug" > - <AppenderRef ref="Console"/> - </Root> - </Loggers> -</Configuration> -``` - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/prometheus.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/prometheus.md deleted file mode 100644 index 83f44762e09..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/prometheus.md +++ /dev/null @@ -1,14 +0,0 @@ -# Enabling metrics in Prometheus - -{% include [work in progress message](../../_includes/addition.md) %} - -Below are examples of the code for enabling metrics in Prometheus in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](prometheus/go.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/prometheus/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/prometheus/go.md deleted file mode 100644 index 466bb1b8385..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/_includes/prometheus/go.md +++ /dev/null @@ -1,31 +0,0 @@ -```go -package main - -import ( - "context" - - "github.com/prometheus/client_golang/prometheus" - metrics "github.com/ydb-platform/ydb-go-sdk-prometheus" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" -) - -func main() { - ctx := context.Background() - registry := prometheus.NewRegistry() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - metrics.WithTraces( - registry, - metrics.WithDetails(trace.DetailsAll), - metrics.WithSeparator("_"), - ), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/index.md deleted file mode 100644 index e5687972c21..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/index.md +++ /dev/null @@ -1,3 +0,0 @@ - -{% include [balancing.md](_includes/index.md) %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/jaeger.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/jaeger.md deleted file mode 100644 index 4941b019de1..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/jaeger.md +++ /dev/null @@ -1,2 +0,0 @@ - -{% include [index.md](_includes/jaeger.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/logs.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/logs.md deleted file mode 100644 index c37ce7b35ea..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/logs.md +++ /dev/null @@ -1,2 +0,0 @@ - -{% include [index.md](_includes/logs.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/prometheus.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/prometheus.md deleted file mode 100644 index 33c41f2c801..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/prometheus.md +++ /dev/null @@ -1,2 +0,0 @@ - -{% include [index.md](_includes/prometheus.md) %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/toc_i.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/toc_i.yaml deleted file mode 100644 index fa5d09fcc6e..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/toc_i.yaml +++ /dev/null @@ -1,9 +0,0 @@ -items: -- name: Overview - href: index.md -- name: Enable logging - href: logs.md -- name: Enable metrics in Prometheus - href: prometheus.md -- name: Enable tracing in Jaeger - href: jaeger.md
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/toc_p.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/toc_p.yaml deleted file mode 100644 index 5bfec4365de..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/debug/toc_p.yaml +++ /dev/null @@ -1,2 +0,0 @@ -items: -- include: { mode: link, path: toc_i.yaml }
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/index.md index bd4e95cb411..530d0e58f60 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/index.md +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/index.md @@ -1,3 +1,29 @@ +# Code recipes -{% include [index.md](_includes/index.md) %} +{% include [work in progress message](_includes/addition.md) %} + +This section contains code recipes in different programming languages for a variety of tasks that are common when working with the {{ ydb-short-name }} SDK. + +Table of contents: +- [Overview](index.md) +- [Initializing the driver](init.md) +- [Authentication](auth.md) + - [Using a token](auth-access-token.md) + - [Anonymous](auth-anonymous.md) + - [Service account file](auth-service-account.md) + - [Metadata service](auth-metadata.md) + - [Using environment variables](auth-env.md) + - [Username and password based](auth-static.md) +- [Balancing](balancing.md) + - [Random choice](balancing-random-choice.md) + - [Prefer the nearest data center](balancing-prefer-local.md) + - [Prefer the availability zone](balancing-prefer-location.md) +- [Running repeat queries](retry.md) +- [Setting the session pool size](session-pool-limit.md) +- [Inserting data](upsert.md) +- [Bulk upsert of data](bulk-upsert.md) +- [Troubleshooting](debug.md) + - [Enable logging](debug-logs.md) + - [Enable metrics in Prometheus](debug-prometheus.md) + - [Enable tracing in Jaeger](debug-jaeger.md) diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/init.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/init.md new file mode 100644 index 00000000000..dbc3ac735b8 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/init.md @@ -0,0 +1,96 @@ +# Initialize the driver + +{% include [work in progress message](_includes/addition.md) %} + +To connect to {{ ydb-short-name }}, you need to specify the required and additional parameters that define the driver's behavior (learn more in [Connecting to the {{ ydb-short-name }} server](../../../concepts/connect.md)). + +Below are examples of the code for connecting to {{ ydb-short-name }} (driver creation) in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```golang + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + db, err := ydb.Open(ctx, "grpcs://localhost:2135/local") + if err != nil { + log.Fatal(err) + } + ... + } + ``` + +- Go (database/sql) + + {% cut "Using a connector (recommended)" %} + ```golang + package main + + import ( + "context" + "database/sql" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + nativeDriver, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ) + if err != nil { + panic(err) + } + defer nativeDriver.Close(ctx) + connector, err := ydb.Connector(nativeDriver) + if err != nil { + panic(err) + } + db := sql.OpenDB(connector) + defer db.Close() + ... + } + ``` + {% endcut %} + + {% cut "Using a connection string" %} + + The `database/sql` driver is registered when importing the package of a specific driver separated by an underscore: + ```golang + package main + + import ( + "context" + "database/sql" + "os" + + _ "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + db, err := sql.Open("ydb", "grpcs://localhost:2135/local") + if err != nil { + log.Fatal(err) + } + ... + } + ``` + {% endcut %} + +- Java + + + {% include [work in progress message](_includes/addition.md) %} + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/retry.md new file mode 100644 index 00000000000..e1be79c35fa --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/retry.md @@ -0,0 +1,324 @@ +# Retrying + +{% include [work in progress message](_includes/addition.md) %} + +{{ ydb-short-name }} is a distributed database management system with automatic load scaling. +Routine maintenance can be carried out on the server side, with server racks or entire data centers temporarily shut down. +This may result in errors arising from {{ ydb-short-name }} operation. +There are different response scenarios depending on the error type. +{{ ydb-short-name }} + To ensure high database availability, SDKs provide built-in tools for retries, +accounting for error types and responses to them. + +Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for retries: + +{% list tabs %} + +- Go (native) + + In the {{ ydb-short-name }} Go SDK, correct error handling is implemented by several programming interfaces: + + {% cut "General-purpose repeat function" %} + + The basic logic of error handling is implemented by the helper `retry.Retry` function + The details of repeat query execution are mostly hidden. + The user can affect the logic of the `retry.Retry` function in two ways: + * Via the context (where you can set the deadline and cancel) + * Via the operation's idempotency flag `retry.WithIdempotent()`. By default, the operation is considered non-idempotent. + + The user passes a custom function to `retry.Retry` that returns an error by its signature. + If the custom function returns `nil`, then repeat queries stop. + If the custom function returns an error, the {{ ydb-short-name }} Go SDK tries to identify this error and executes retries depending on it. + + Example of the code that uses the `retry.Retry` function: + ```golang + package main + + import ( + "context" + "time" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/retry" + ) + + func main() { + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + var cancel context.CancelFunc + // fix deadline for retries + ctx, cancel := context.WithTimeout(ctx, time.Second) + err = retry.Retry( + ctx, + func(ctx context.Context) error { + whoAmI, err := db.Discovery().WhoAmI(ctx) + if err != nil { + return err + } + fmt.Println(whoAmI) + }, + retry.WithIdempotent(true), + ) + if err != nil { + panic(err) + } + } + ``` + + {% endcut %} + + {% cut "Repeat attempts in case of failed {{ ydb-short-name }} session objects" %} + + For repeat error handling at the level of a {{ ydb-short-name }} table service session, you can use the `db.Table().Do(ctx, op)` function, which provides a prepared session for query execution. + `db.Table().Do(ctx, op)` uses the `retry` package and tracks the lifetime of the {{ ydb-short-name }} sessions. + Based on its signature, the user's operation `op` should return an error or `nil` so that the driver can "decide" what to do based on the error type: repeat the operation or not, with delay or without, and in this session or a new one. + The user can affect the logic of repeat queries using the context and the idempotence flag, while the {{ ydb-short-name }} Go SDK interprets errors returned by `op`. + + Example of the code that uses the `db.Table().Do(ctx, op)` function: + ```golang + err := db.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { + desc, err = s.DescribeTableOptions(ctx) + return + }, table.WithIdempotent()) + if err != nil { + return err + } + ``` + + {% endcut %} + + {% cut "Repeat attempts in case of failed {{ ydb-short-name }} interactive transaction objects" %} + + For repeat error handling at the level of a {{ ydb-short-name }} table service interactive transaction, you can use the `db.Table().DoTx(ctx, txOp)` function, which provides a {{ ydb-short-name }} prepared session transaction for query execution. + `db.Table().DoTx(ctx, txOp)` uses the `retry` package and tracks the lifetime of the {{ ydb-short-name }} sessions. + Based on its signature, the user's operation `txOp` should return an error or `nil` so that the driver can "decide" what to do based on the error type: repeat the operation or not, with delay or without, and in this transaction or a new one. + The user can affect the logic of repeat queries using the context and the idempotence flag, while the {{ ydb-short-name }} Go SDK interprets errors returned by `op`. + + Example of the code that uses the `db.Table().DoTx(ctx, op)` function: + ```golang + err := db.Table().DoTx(ctx, func(ctx context.Context, tx table.TransactionActor) error { + _, err := tx.Execute(ctx, + "DECLARE $id AS Int64; INSERT INTO test (id, val) VALUES($id, 'asd')", + table.NewQueryParameters(table.ValueParam("$id", types.Int64Value(100500))), + ) + return err + }, table.WithIdempotent()) + if err != nil { + return err + } + ``` + + {% endcut %} + + {% cut "Queries to other {{ ydb-short-name }} services" %} + + (`db.Scripting()`, `db.Scheme()`, `db.Coordination()`, `db.Ratelimiter()`, `db.Discovery()`) also use the `retry.Retry` function inside to execute repeat queries and don't require external auxiliary functions for repeats. + + {% endcut %} + +- Go (database/sql) + + The standard `database/sql` package uses the internal logic of repeats based on the errors a specific driver implementation returns. + For example, the `database/sql` [code](https://github.com/golang/go/tree/master/src/database/sql) frequently shows the three-attempt repeats policy: + - Two attempts at a present connection or new one (if the `database/sql` connection pool is empty). + - One attempt at a new connection. + + This repeat policy is mostly enough to survive temporary unavailability of {{ ydb-short-name }} nodes or issues with a {{ ydb-short-name }} session. + + The {{ ydb-short-name }} Go SDK provides special functions to ensure execution of a user's operation: + + {% cut "Repeat attempts in case of failed `*sql.Conn` connection objects:" %} + + For repeat error handling at `*sql.Conn` connection objects, you can use the auxiliary `retry.Do(ctx, db, op)` function, which provides a prepared `*sql.Conn` session for query execution. + You need to pass the context, database object, and the user's operation for execution to the `retry.Do` function. + The user's code can affect the logic of repeat queries using the context and the idempotence flag, while the {{ ydb-short-name }} Go SDK, in turn, interprets errors returned by `op`. + + The user's `op` operation must return an error or `nil`: + - If the custom function returns `nil`, then repeat queries stop. + - If the custom function returns an error, the {{ ydb-short-name }} Go SDK tries to identify this error and performs retries depending on it. + + Example of the code that uses the `retry.Do` function: + ```golang + import ( + "context" + "database/sql" + "fmt" + "log" + + "github.com/ydb-platform/ydb-go-sdk/v3/retry" + ) + + func main() { + ... + err = retry.Do(ctx, db, func(ctx context.Context, cc *sql.Conn) (err error) { + row = cc.QueryRowContext(ctx, ` + PRAGMA TablePathPrefix("/local"); + DECLARE $seriesID AS Uint64; + DECLARE $seasonID AS Uint64; + DECLARE $episodeID AS Uint64; + SELECT views FROM episodes WHERE series_id = $seriesID AND season_id = $seasonID AND episode_id = $episodeID; + `, + sql.Named("seriesID", uint64(1)), + sql.Named("seasonID", uint64(1)), + sql.Named("episodeID", uint64(1)), + ) + var views sql.NullFloat64 + if err = row.Scan(&views); err != nil { + return fmt.Errorf("cannot scan views: %w", err) + } + if views.Valid { + return fmt.Errorf("unexpected valid views: %v", views.Float64) + } + log.Printf("views = %v", views) + return row.Err() + }, retry.WithDoRetryOptions(retry.WithIdempotent(true))) + if err != nil { + log.Printf("retry.Do failed: %v\n", err) + } + } + ``` + + {% endcut %} + + {% cut "Repeat attempts in case of failed `*sql.Tx` interactive transaction objects:" %} + + For repeat error handling at `*sql.Tx` interactive transaction objects, you can use the auxiliary `retry.DoTx(ctx, db, op)` function, which provides a prepared `*sql.Tx` transaction for query execution. + You need to pass the context, database object, and the user's operation for execution to the `retry.DoTx` function. + The function is passed a prepared `*sql.Tx` transaction, where queries to {{ ydb-short-name }} should be executed. + The user's code can affect the logic of repeat queries using the context and the operation idempotence flag, while the {{ ydb-short-name }} Go SDK, in turn, interprets errors returned by `op`. + + The user's `op` operation must return an error or `nil`: + - If the custom function returns `nil`, then repeat queries stop. + - If the custom function returns an error, the {{ ydb-short-name }} Go SDK tries to identify this error and performs retries depending on it. + + By default, `retry.DoTx` uses the read-write isolation mode of the `sql.LevelDefault` transaction and you can change it using the `retry.WithTxOptions` parameter. + + Example of the code that uses the `retry.Do` function: + ```golang + import ( + "context" + "database/sql" + "fmt" + "log" + + "github.com/ydb-platform/ydb-go-sdk/v3/retry" + ) + + func main() { + ... + err = retry.DoTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + row := tx.QueryRowContext(ctx,` + PRAGMA TablePathPrefix("/local"); + DECLARE $seriesID AS Uint64; + DECLARE $seasonID AS Uint64; + DECLARE $episodeID AS Uint64; + SELECT views FROM episodes WHERE series_id = $seriesID AND season_id = $seasonID AND episode_id = $episodeID; + `, + sql.Named("seriesID", uint64(1)), + sql.Named("seasonID", uint64(1)), + sql.Named("episodeID", uint64(1)), + ) + var views sql.NullFloat64 + if err = row.Scan(&views); err != nil { + return fmt.Errorf("cannot select current views: %w", err) + } + if !views.Valid { + return fmt.Errorf("unexpected invalid views: %v", views) + } + t.Logf("views = %v", views) + if views.Float64 != 1 { + return fmt.Errorf("unexpected views value: %v", views) + } + return nil + }, retry.WithDoTxRetryOptions(retry.WithIdempotent(true)), retry.WithTxOptions(&sql.TxOptions{ + Isolation: sql.LevelSnapshot, + ReadOnly: true, + })) + if err != nil { + log.Printf("do tx failed: %v\n", err) + } + } + ``` + + {% endcut %} + +- Java + + In the {{ ydb-short-name }} Java SDK, repeat queries are implemented by the `com.yandex.ydb.table.SessionRetryContext` helper class. This class is constructed with the `SessionRetryContext.create` method to which you pass the `SessionSupplier` interface implementation (usually an instance of the `TableClient` class). + Additionally, the user can specify some other options + * `maxRetries(int maxRetries)`: The maximum number of operation retries, not counting the first execution. Default value: `10` + * `retryNotFound(boolean retryNotFound)`: The option to retry operations that returned the `NOT_FOUND` status. Enabled by default. + * `idempotent(boolean idempotent)`: Indicates idempotency of operations. Idempotent operations will be retried for a broader range of errors. Disabled by default. + + The `SessionRetryContext` class provides two methods to run operations with retries. + * `CompletableFuture<Status> supplyStatus`: Executing the operation that returns the status. As an argument, it accepts the lambda `Function<Session, CompletableFuture<Status>> fn` + * `CompletableFuture<Result<T>> supplyResult`: Executing the operation that returns data. As an argument, it accepts the lambda `Function<Session, CompletableFuture<Result<T>>> fn` + + When using the `SessionRetryContext` class, make sure that the operation will be retried in the following cases + * The lambda function returned a [retryable](../error_handling.md) error code + * The lambda function invoked an `UnexpectedResultException` with a [retryable](../error_handling.md) error code + + + + {% cut "Sample code using SessionRetryContext.supplyStatus:" %} + + ```java + private void createTable(TableClient tableClient, String database, String tableName) { + SessionRetryContext retryCtx = SessionRetryContext.create(tableClient).build(); + TableDescription pets = TableDescription.newBuilder() + .addNullableColumn("species", PrimitiveType.utf8()) + .addNullableColumn("name", PrimitiveType.utf8()) + .addNullableColumn("color", PrimitiveType.utf8()) + .addNullableColumn("price", PrimitiveType.float32()) + .setPrimaryKeys("species", "name") + .build(); + + String tablePath = database + "/" + tableName; + retryCtx.supplyStatus(session -> session.createTable(tablePath, pets)) + .join().expect("ok"); + } + ``` + + {% endcut %} + + {% cut "Sample code using SessionRetryContext.supplyResult:" %} + + ```java + private void selectData(TableClient tableClient, String tableName) { + SessionRetryContext retryCtx = SessionRetryContext.create(tableClient).build(); + String selectQuery + = "DECLARE $species AS Utf8;" + + "DECLARE $name AS Utf8;" + + "SELECT * FROM " + tableName + " " + + "WHERE species = $species AND name = $name;"; + + Params params = Params.of( + "$species", PrimitiveValue.utf8("cat"), + "$name", PrimitiveValue.utf8("Tom") + ); + + DataQueryResult data = retryCtx + .supplyResult(session -> session.executeDataQuery(selectQuery, TxControl.onlineRo(), params)) + .join().expect("ok"); + + ResultSetReader rsReader = data.getResultSet(0); + logger.info("Result of select query:"); + while (rsReader.next()) { + logger.info(" species: {}, name: {}, color: {}, price: {}", + rsReader.getColumn("species").getUtf8(), + rsReader.getColumn("name").getUtf8(), + rsReader.getColumn("color").getUtf8(), + rsReader.getColumn("price").getFloat32() + ); + } + } + ``` + + {% endcut %} + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/_includes/go.md deleted file mode 100644 index 6218b03dbae..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/_includes/go.md +++ /dev/null @@ -1,62 +0,0 @@ -In the {{ ydb-short-name }} Go SDK, correct error handling is implemented by several programming interfaces: - -* The basic logic of error handling is implemented by the helper `retry.Retry` function. - The details of making request retries are hidden as much as possible. - The user can affect the logic of executing the `retry.Retry` function in two ways: - * Via the context (where you can set the deadline and cancel). - * Via the operation's idempotency flag `retry.WithIdempotent()`. By default, the operation is considered non-idempotent. - - The user passes a custom function to `retry.Retry` that returns an error by its signature. -If the custom function returns `nil`, then repeat queries stop. -If the custom function returns an error, the {{ ydb-short-name }} Go SDK tries to identify this error and executes retries depending on it. - - {% cut "Example of code, using `retry.Retry` function:" %} - - ```go - package main - - import ( - "context" - "time" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/retry" - ) - - func main() { - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - var cancel context.CancelFunc - // fix deadline for retries - ctx, cancel := context.WithTimeout(ctx, time.Second) - err = retry.Retry(ctx, - func(ctx context.Context) error { - whoAmI, err := db.Discovery().WhoAmI(ctx) - if err != nil { - return err - } - fmt.Println(whoAmI) - }, - retry.WithIdempotent(true), - ) - if err != nil { - panic(err) - } - } - ``` - - {% endcut %} - -* The `db.Table()` table query service immediately provides the `table.Client` programming interface that uses the `retry` package and tracks the lifetime of the {{ ydb-short-name }} sessions. - Two public functions are available to the user: `db.Table().Do(ctx, op)` (where `op` provides a session) and `db.Table().DoTx(ctx, op)` (where `op` provides a transaction). - As in the previous case, the user can affect the logic of repeat queries using the context and the idempotence flag, while the {{ ydb-short-name }} Go SDK interprets errors returned by `op`. - -* Queries to other {{ ydb-short-name }} services (`db.Scripting()`, `db.Scheme()`, `db.Coordination()`, `db.Ratelimiter()`, and `db.Discovery()`) also use the `retry.Retry` function internally to make repeat queries. - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/_includes/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/_includes/java.md deleted file mode 100644 index 6c0a024fab0..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/_includes/java.md +++ /dev/null @@ -1,74 +0,0 @@ -In the {{ ydb-short-name }} Java SDK, the request retry mechanism is implemented as the `com.yandex.ydb.table.SessionRetryContext` helper class. This class is built using the `SessionRetryContext.create` method, where you should pass the implementation of the `SessionSupplier` interface (usually, this is an instance of the `TableClient` class). -Additionally, the user can set some other options. - -* `maxRetries(int maxRetries)`: The maximum number of operation retries, excluding the first execution. Defaults to `10`. -* `retryNotFound(boolean retryNotFound)`: The option to retry operations that return the `NOT_FOUND` status. Enabled by default. -* `idempotent(boolean idempotent)`: Indicates if an operation is idempotent. The system will retry idempotent operations for a wider list of errors. Disabled by default. - -The `SessionRetryContext` class provides two methods to run operations with retries. - -* `CompletableFuture<Status> supplyStatus`: Run an operation that returns a status. Takes the `Function<Session, CompletableFuture<Status>> fn` lambda as an argument. -* `CompletableFuture<Result<T>> supplyResult`: Run an operation that returns data. Takes the `Function<Session, CompletableFutureResult<T>> fn` lambda as an argument. - -When using the `SessionRetryContext` class, keep in mind that operation retries will be made in the following cases: - -* The lamda function returns the [retryable](../../../error_handling.md) error code. - -* While executing the lambda function, the `UnexpectedResultException` with the [retryable](../../../error_handling.md) error code is raised. - - {% cut "Snippet of code using SessionRetryContext.supplyStatus:" %} - - ```java - private void createTable(TableClient tableClient, String database, String tableName) { - SessionRetryContext retryCtx = SessionRetryContext.create(tableClient).build(); - TableDescription pets = TableDescription.newBuilder() - .addNullableColumn("species", PrimitiveType.Text) - .addNullableColumn("name", PrimitiveType.Text) - .addNullableColumn("color", PrimitiveType.Text) - .addNullableColumn("price", PrimitiveType.Float) - .setPrimaryKeys("species", "name") - .build(); - - String tablePath = database + "/" + tableName; - retryCtx.supplyStatus(session -> session.createTable(tablePath, pets)) - .join().expectSuccess(); - } - ``` - - {% endcut %} - - {% cut "Snippet of code using SessionRetryContext.supplyResult:" %} - - ```java - private void selectData(TableClient tableClient, String tableName) { - SessionRetryContext retryCtx = SessionRetryContext.create(tableClient).build(); - String selectQuery - = "DECLARE $species AS Text;" - + "DECLARE $name AS Text;" - + "SELECT * FROM " + tableName + " " - + "WHERE species = $species AND name = $name;"; - - Params params = Params.of( - "$species", PrimitiveValue.newText("cat"), - "$name", PrimitiveValue.newText("Tom") - ); - - DataQueryResult data = retryCtx - .supplyResult(session -> session.executeDataQuery(selectQuery, TxControl.onlineRo(), params)) - .join().getValue(); - - ResultSetReader rsReader = data.getResultSet(0); - logger.info("Result of select query:"); - while (rsReader.next()) { - logger.info(" species: {}, name: {}, color: {}, price: {}", - rsReader.getColumn("species").getText(), - rsReader.getColumn("name").getText(), - rsReader.getColumn("color").getText(), - rsReader.getColumn("price").getFloat() - ); - } - } - ``` - - {% endcut %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/index.md deleted file mode 100644 index f56238fcc19..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/retry/index.md +++ /dev/null @@ -1,24 +0,0 @@ -# Retrying - -{% include [work in progress message](../_includes/addition.md) %} - -{{ ydb-short-name }} is a distributed database management system with automatic load scaling. -Routine maintenance can be carried out on the server side, with server racks or entire data centers temporarily shut down. -This may result in errors arising from {{ ydb-short-name }} operation. -There are different response scenarios depending on the error type. -{{ ydb-short-name }} To ensure high database availability, SDKs provide built-in tools for retries, accounting for error types and responses to them. - -Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for retries: - -{% list tabs %} - -- Go - - {% include [go.md](_includes/go.md) %} - -- Java - - {% include [java.md](_includes/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/session-pool-limit.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/session-pool-limit.md new file mode 100644 index 00000000000..bf0fd6e9f61 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/session-pool-limit.md @@ -0,0 +1,78 @@ +# Setting the session pool size + +{% include [work in progress message](_includes/addition.md) %} + +The client's session pool size affects resource consumption (RAM, CPU) on the server side of {{ ydb-short-name }}. + +Simple math: if `1000` clients of the same DB have `1000` sessions each, `1000000` actors (workers, session performers) are created on the server side. If you don't limit the number of sessions on the client, this may result in a slow cluster that is close to a failure. + +By default, the {{ ydb-short-name }} SDK limits the number of sessions to `50`. +A good recommendation is to set the limit on the number of client sessions to the minimum required for the normal operation of the client app. Keep in mind that sessions are single-threaded both on the server and client side. So if the application needs to make `1000` simultaneous (`inflight`) requests to {{ ydb-short-name }} for its estimated load, the limit should be set to `1000` sessions. +Here it's necessary to distinguish between the estimated `RPS` (requests per second) and `inflight`. In the first case, this is the total number of requests to {{ ydb-short-name }} completed within `1` second. For example, if `RPS`=`10000` and the average `latency` is `100`ms, it's sufficient to set the session limit to `1000`. This means that each session will perform an average of `10` consecutive requests for the estimated second. + +Below are examples of the code for setting the session pool limit in different {{ ydb-short-name }} SDKs. + +{% list tabs %} + +- Go (native) + + ```golang + package main + + import ( + "context" + + "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithSessionPoolSizeLimit(500), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + ... + } + ``` + +- Go (database/sql) + + The `database/sql` library has its own connection pool. Each `database/sql`connection corresponds to a specific {{ ydb-short-name }} session. A `database/sql` connection pool is managed by the `sql.DB.SetMaxOpenConns` and `sql.DB.SetMaxIdleConns` functions. Learn more in the `database/sql` [documentation](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). + + Example of the code that uses the size of `database/sql` connection pool: + ```golang + package main + + import ( + "context" + "database/sql" + + _ "github.com/ydb-platform/ydb-go-sdk/v3" + ) + + func main() { + db, err := ydb.Open("ydb", os.Getenv("YDB_CONNECTION_STRING")) + if err != nil { + panic(err) + } + defer db.Close(ctx) + db.SetMaxOpenConns(100) + db.SetMaxIdleConns(100) + ... + } + ``` + +- Java + + ```java + this.tableClient = TableClient.newClient(rpc) + // 10 - minimum number of active sessions to keep in the pool during the cleanup + // 500 - maximum number of sessions in the pool + .sessionPoolSize(10, 500) + .build(); + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/_includes/go.md deleted file mode 100644 index 653b08c1037..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/_includes/go.md +++ /dev/null @@ -1,22 +0,0 @@ -```go -package main - -import ( - "context" - - "github.com/ydb-platform/ydb-go-sdk/v3" -) - -func main() { - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithSessionPoolSizeLimit(500), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/_includes/java.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/_includes/java.md deleted file mode 100644 index 04f4bbba62c..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/_includes/java.md +++ /dev/null @@ -1,7 +0,0 @@ -```java -this.tableClient = TableClient.newClient(transport) - // 10 - minimum number of active sessions to keep in the pool during the cleanup - // 500 - maximum number of sessions in the pool - .sessionPoolSize(10, 500) - .build(); -```
\ No newline at end of file diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/index.md deleted file mode 100644 index 7cf9390b2d6..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/session_pool_limit/index.md +++ /dev/null @@ -1,28 +0,0 @@ -# Setting the session pool size - -{% include [work in progress message](../_includes/addition.md) %} - -The client's session pool size affects resource consumption (RAM, CPU) on the server side of {{ ydb-short-name }}. - -Simple math: if `1000` clients of the same DB have `1000` sessions each, `1000000` actors (workers, session performers) are created on the server side. If you don't limit the number of sessions on the client, this may result in a slow cluster that is close to a failure. - -By default, the {{ ydb-short-name }} SDK has a limit of `50` sessions when using native drivers. There is no limit for third-party libraries, such as Go database/sql. - -A good recommendation is to set the limit on the number of client sessions to the minimum required for the normal operation of the client app. Keep in mind that sessions are single-threaded both on the server and client side. So if the application needs to make `1000` simultaneous (`inflight`) requests to {{ ydb-short-name }} for its estimated load, the limit should be set to `1000` sessions. - -Here it's necessary to distinguish between the estimated `RPS` (requests per second) and `inflight`. In the first case, this is the total number of requests to {{ ydb-short-name }} completed within `1` second. For example, if `RPS`=`10000` and the average `latency` is `100`ms, it's sufficient to set the session limit to `1000`. This means that each session will perform an average of `10` consecutive requests for the estimated second. - -Below are examples of the code for setting the session pool limit in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% include [go.md](_includes/go.md) %} - -- Java - - {% include [java.md](_includes/java.md) %} - -{% endlist %} - diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml b/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml index 02069df58d1..5aaa0fe252d 100644 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/toc_i.yaml @@ -1,17 +1,49 @@ items: - name: Overview href: index.md +- name: Initialize the driver + href: init.md - name: Authentication - include: { mode: link, path: auth/toc_p.yaml } + items: + - name: Overview + href: auth.md + - name: Using a token + href: auth-access-token.md + - name: Anonymous + href: auth-anonymous.md + - name: Service account file + href: auth-service-account.md + - name: Metadata service + href: auth-metadata.md + - name: Using environment variables + href: auth-env.md + - name: Username and password based + href: auth-static.md - name: Balancing - include: { mode: link, path: balancing/toc_p.yaml } + items: + - name: Overview + href: balancing.md + - name: Random choice + href: balancing-random-choice.md + - name: Prefer the nearest data center + href: balancing-prefer-local.md + - name: Prefer the specific availability zone + href: balancing-prefer-location.md - name: Running repeat queries - href: retry/index.md -- name: Troubleshooting - include: { mode: link, path: debug/toc_p.yaml } + href: retry.md - name: Setting the session pool size - href: session_pool_limit/index.md + href: session-pool-limit.md - name: Upserting data - href: upsert/index.md + href: upsert.md - name: Bulk-upserting data - href: bulk_upsert/index.md + href: bulk-upsert.md +- name: Troubleshooting + items: + - name: Overview + href: debug.md + - name: Enable logging + href: debug-logs.md + - name: Enable metrics in Prometheus + href: debug-prometheus.md + - name: Enable tracing in Jaeger + href: debug-jaeger.md diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert.md new file mode 100644 index 00000000000..0649137b498 --- /dev/null +++ b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert.md @@ -0,0 +1,133 @@ +# Inserting data + +{% include [work in progress message](_includes/addition.md) %} + +Below are code examples showing the {{ ydb-short-name }} SDK built-in tools for batch insert: + +{% list tabs %} + +- Go (native) + + ```go + package main + + import ( + "context" + "os" + + "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/table" + "github.com/ydb-platform/ydb-go-sdk/v3/table/types" + ) + + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + db, err := ydb.Open(ctx, + os.Getenv("YDB_CONNECTION_STRING"), + ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), + ) + if err != nil { + panic(err) + } + defer db.Close(ctx) + // execute upsert with native ydb data + err = db.Table().DoTx( // Do retry operation on errors with best effort + ctx, // context manages exiting from Do + func(ctx context.Context, tx table.TransactionActor) (err error) { // retry operation + res, err = tx.Execute(ctx, ` + PRAGMA TablePathPrefix("/path/to/table"); + DECLARE $seriesID AS Uint64; + DECLARE $seasonID AS Uint64; + DECLARE $episodeID AS Uint64; + DECLARE $views AS Uint64; + UPSERT INTO episodes ( series_id, season_id, episode_id, views ) + VALUES ( $seriesID, $seasonID, $episodeID, $views ); + `, + table.NewQueryParameters( + table.ValueParam("$seriesID", types.Uint64Value(1)), + table.ValueParam("$seasonID", types.Uint64Value(1)), + table.ValueParam("$episodeID", types.Uint64Value(1)), + table.ValueParam("$views", types.Uint64Value(1)), // increment views + ), + ) + if err != nil { + return err + } + if err = res.Err(); err != nil { + return err + } + return res.Close() + }, table.WithIdempotent(), + ) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } + } + ``` + +- Go (database/sql) + + ```go + package main + + import ( + "context" + "database/sql" + "os" + + _ "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/retry" + "github.com/ydb-platform/ydb-go-sdk/v3/types" + ) + + func main() { + db, err := sql.Open("ydb", os.Getenv("YDB_CONNECTION_STRING")) + if err != nil { + panic(err) + } + defer db.Close(ctx) + // execute upsert with native ydb data + err = retry.DoTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + if _, err = tx.ExecContext(ctx,` + PRAGMA TablePathPrefix("/local"); + REPLACE INTO series + SELECT + series_id, + title, + series_info, + comment + FROM AS_TABLE($seriesData); + `, + sql.Named("seriesData", types.ListValue( + types.StructValue( + types.StructFieldValue("series_id", types.Uint64Value(1)), + types.StructFieldValue("title", types.TextValue("IT Crowd")), + types.StructFieldValue("series_info", types.TextValue( + "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, produced by "+ + "Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, and Matt Berry.", + )), + types.StructFieldValue("comment", types.NullValue(types.TypeText)), + ), + types.StructValue( + types.StructFieldValue("series_id", types.Uint64Value(2)), + types.StructFieldValue("title", types.TextValue("Silicon Valley")), + types.StructFieldValue("series_info", types.TextValue( + "Silicon Valley is an American comedy television series created by Mike Judge, John Altschuler and "+ + "Dave Krinsky. The series focuses on five young men who founded a startup company in Silicon Valley.", + )), + types.StructFieldValue("comment", types.TextValue("lorem ipsum")), + ), + )), + ); err != nil { + return err + } + return nil + }, retry.WithDoTxRetryOptions(retry.WithIdempotent(true))) + if err != nil { + fmt.Printf("unexpected error: %v", err) + } + } + ``` + +{% endlist %} diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md deleted file mode 100644 index d8c158cb4a1..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/_includes/go.md +++ /dev/null @@ -1,59 +0,0 @@ -```go -package main - -import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/table" - "github.com/ydb-platform/ydb-go-sdk/v3/table/types" -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithAccessTokenCredentials(os.Getenv("YDB_TOKEN")), - ) - if err != nil { - panic(err) - } - defer func() { - _ = db.Close(ctx) - }() - // execute bulk upsert with native ydb data - err = db.Table().DoTx( // Do retry operation on errors with best effort - ctx, // context manages exiting from Do - func(ctx context.Context, tx table.TransactionActor) (err error) { // retry operation - res, err = tx.Execute(ctx, ` - PRAGMA TablePathPrefix("/path/to/table"); - DECLARE $seriesID AS Uint64; - DECLARE $seasonID AS Uint64; - DECLARE $episodeID AS Uint64; - DECLARE $views AS Uint64; - UPSERT INTO episodes ( series_id, season_id, episode_id, views ) - VALUES ( $seriesID, $seasonID, $episodeID, $views ); - `, - table.NewQueryParameters( - table.ValueParam("$seriesID", types.Uint64Value(1)), - table.ValueParam("$seasonID", types.Uint64Value(1)), - table.ValueParam("$episodeID", types.Uint64Value(1)), - table.ValueParam("$views", types.Uint64Value(1)), // increment views - ), - ) - if err != nil { - return err - } - if err = res.Err(); err != nil { - return err - } - return res.Close() - }, - ) - if err != nil { - fmt.Printf("unexpected error: %v", err) - } -} -``` diff --git a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md b/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md deleted file mode 100644 index d50715e3944..00000000000 --- a/ydb/docs/en/core/reference/ydb-sdk/recipes/upsert/index.md +++ /dev/null @@ -1,14 +0,0 @@ -# Inserting data - -{% include [work in progress message](../_includes/addition.md) %} - -Below are code examples showing the {{ ydb-short-name }} SDK methods for upserting data: - -{% list tabs %} - -- Go - - {% include [go.md](_includes/go.md) %} - -{% endlist %} - |