1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
syntax = "proto3";
package yandex.cloud.datatransfer.v1;
option go_package = "github.com/yandex-cloud/go-genproto/yandex/cloud/datatransfer/v1;datatransfer";
option java_package = "yandex.cloud.api.datatransfer.v1";
import "yandex/cloud/datatransfer/v1/endpoint.proto";
enum TransferType {
TRANSFER_TYPE_UNSPECIFIED = 0;
// Snapshot and increment
SNAPSHOT_AND_INCREMENT = 1;
// Snapshot
SNAPSHOT_ONLY = 2;
// Increment
INCREMENT_ONLY = 3;
}
enum TransferStatus {
TRANSFER_STATUS_UNSPECIFIED = 0;
// Transfer does some work before running
CREATING = 1;
// Transfer created but not started by user
CREATED = 2;
// Transfer currently doing replication work
RUNNING = 3;
// Transfer shutdown
STOPPING = 4;
// Transfer stopped by user
STOPPED = 5;
// Transfer stopped by system
ERROR = 6;
// Transfer copy snapshot
SNAPSHOTTING = 7;
// Transfer reach terminal phase
DONE = 8;
}
// Transfer core entity
message Transfer {
reserved 3, 11, 13 to 14, 16, 18 to 21;
string id = 1;
string folder_id = 2;
string name = 4;
string description = 5;
map<string,string> labels = 6;
Endpoint source = 7;
Endpoint target = 8;
Runtime runtime = 9;
TransferStatus status = 10;
TransferType type = 12;
string warning = 15;
Transformation transformation = 17;
bool prestable = 22;
}
message Runtime {
reserved 1 to 3;
oneof runtime {
YcRuntime yc_runtime = 4;
}
}
message ShardingUploadParams {
int64 job_count = 1;
int64 process_count = 2;
}
message YcRuntime {
reserved 2 to 7;
int64 job_count = 1;
ShardingUploadParams upload_shard_params = 8;
}
// Mask function
message MaskFunction {
oneof mask_function {
// Hash mask function
MaskFunctionHash mask_function_hash = 1;
}
}
// Hash data using HMAC
message MaskFunctionHash {
// This string will be used in the HMAC(sha256, salt) function applied to the
// column data.
string user_defined_salt = 1;
}
// Filter tables using lists of included and excluded tables.
message TablesFilter {
// List of tables that will be included to transfer
repeated string include_tables = 1;
// List of tables that will be excluded to transfer
repeated string exclude_tables = 2;
}
// Filter columns using lists of included and excluded columns.
message ColumnsFilter {
// List of columns that will be included to transfer
repeated string include_columns = 1;
// List of columns that will be excluded to transfer
repeated string exclude_columns = 2;
}
// Mask field transformer allows you to hash data
message MaskFieldTransformer {
// List of included and excluded tables
TablesFilter tables = 1;
// Specify the name of the column for data masking (a regular expression).
repeated string columns = 2;
// Mask function
MaskFunction function = 3;
}
// Set up a list of table columns to transfer
message FilterColumnsTransformer {
// List of the tables to filter using lists of included and excluded tables.
TablesFilter tables = 1;
// List of the columns to transfer to the target tables using lists of included and
// excluded columns.
ColumnsFilter columns = 2;
}
message Table {
string name_space = 1;
string name = 2;
}
// Specify rule for renaming table
message RenameTable {
// Specify the current names of the table in the source
Table original_name = 1;
// Specify the new names for this table in the target
Table new_name = 2;
}
// Set rules for renaming tables by specifying the current names of the tables in
// the source and new names for these tables in the target.
message RenameTablesTransformer {
// List of renaming rules
repeated RenameTable rename_tables = 1;
}
// Override primary keys
message ReplacePrimaryKeyTransformer {
// List of included and excluded tables
TablesFilter tables = 1;
// List of columns to be used as primary keys
repeated string keys = 2;
}
// Convert column values to strings.
// The values will be converted depending on the source type.
// Conversion rules are described in [documentation](/docs/data-transfer/concepts/data-transformation#convert-to-string).
message ToStringTransformer {
// List of included and excluded tables
TablesFilter tables = 1;
// List of included and excluded columns
ColumnsFilter columns = 2;
}
// Set the number of shards for particular tables and a list of columns whose
// values will be used for calculating a hash to determine a shard.
message SharderTransformer {
// List of included and excluded tables
TablesFilter tables = 1;
// List of included and excluded columns
ColumnsFilter columns = 2;
// Number of shards
int64 shards_count = 3;
}
// A transfer splits the X table into multiple tables (X_1, X_2, ..., X_n) based on
// data.
// If a row was located in the X table before it was split, it is now in the X_i
// table,
// where i is determined by the column list and split string parameters.
// Example:
// If the column list has two columns, month of birth and gender, specified and the
// split string states @,
// information about an employee whose name is John and who was born on February
// 11, 1984,
// from the Employees table will get to a new table named Employees@February@male.
message TableSplitterTransformer {
// List of included and excluded tables
TablesFilter tables = 1;
// Specify the columns in the tables to be partitioned.
repeated string columns = 2;
// Specify the split string to be used for merging components in a new table name.
string splitter = 3;
}
// This filter only applies to transfers with queues (Logbroker or Apache Kafka®)
// as a data source.
// When running a transfer, only the strings meeting the specified criteria remain
// in a changefeed.
message FilterRowsTransformer {
// List of included and excluded tables
TablesFilter tables = 1;
// Filtering criterion. This can be comparison operators for numeric, string, and
// Boolean values,
// comparison to NULL, and checking whether a substring is part of a string.
// For more details see [documentation](/docs/data-transfer/concepts/data-transformation#append-only-sources).
string filter = 2;
}
// Some transformers may have limitations and only apply to some source-target
// pairs.
message Transformer {
reserved 3, 5, 8, 10 to 12;
oneof transformer {
MaskFieldTransformer mask_field = 1;
FilterColumnsTransformer filter_columns = 2;
RenameTablesTransformer rename_tables = 4;
ReplacePrimaryKeyTransformer replace_primary_key = 6;
ToStringTransformer convert_to_string = 7;
SharderTransformer sharder_transformer = 9;
TableSplitterTransformer table_splitter_transformer = 13;
FilterRowsTransformer filter_rows = 14;
}
}
// Transformation is converting data using special transformer functions.
// These functions are executed on a data stream, applied to each data change item,
// and transform them.
// A transformer can be run at both the metadata and data levels.
// Data can only be transformed if the source and target are of different types.
message Transformation {
// Transformers are set as a list.
// When activating a transfer, a transformation plan is made for the tables that
// match the specified criteria.
// Transformers are applied to the tables in the sequence specified in the list.
repeated Transformer transformers = 1;
}
|
