aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNikolay Shestakov <tesseract@ydb.tech>2024-04-23 16:36:02 +0500
committerGitHub <noreply@github.com>2024-04-23 16:36:02 +0500
commitac08084a6aa7ad3b7471565db0521d32f4aa86ef (patch)
tree875a6ddaf21cc93c4a082d2f69a83675ece1b22a
parent74a626e8fcc1ca89e697dd3b55808e68cddff6c5 (diff)
downloadydb-ac08084a6aa7ad3b7471565db0521d32f4aa86ef.tar.gz
Topic API changes for split merge (version 2) (#1502)
-rw-r--r--ydb/public/api/protos/ydb_persqueue_v1.proto57
-rw-r--r--ydb/public/api/protos/ydb_topic.proto92
2 files changed, 144 insertions, 5 deletions
diff --git a/ydb/public/api/protos/ydb_persqueue_v1.proto b/ydb/public/api/protos/ydb_persqueue_v1.proto
index e4eecdbdee2..f2b58f65a50 100644
--- a/ydb/public/api/protos/ydb_persqueue_v1.proto
+++ b/ydb/public/api/protos/ydb_persqueue_v1.proto
@@ -6,6 +6,8 @@ import "ydb/public/api/protos/ydb_issue_message.proto";
import "ydb/public/api/protos/annotations/sensitive.proto";
import "ydb/public/api/protos/annotations/validation.proto";
+import "google/protobuf/duration.proto";
+
package Ydb.PersQueue.V1;
option java_package = "com.yandex.ydb.persqueue";
@@ -1080,6 +1082,18 @@ message Credentials {
}
}
+enum AutoscalingStrategy {
+ // The autoscaling algorithm is not specified. The default value will be used.
+ AUTOSCALING_STRATEGY_UNSPECIFIED = 0;
+ // The autoscaling is disabled.
+ AUTOSCALING_STRATEGY_DISABLED = 1;
+ // The autoscaling algorithm will increase partitions count depending on the load characteristics.
+ // The autoscaling algorithm will never decrease the number of partitions.
+ AUTOSCALING_STRATEGY_SCALE_UP = 2;
+ // The autoscaling algorithm will both increase and decrease partitions count depending on the load characteristics.
+ AUTOSCALING_STRATEGY_SCALE_UP_AND_DOWN = 3;
+}
+
/**
* Message for describing topic internals.
*/
@@ -1089,8 +1103,13 @@ message TopicSettings {
FORMAT_BASE = 1;
}
- // How many partitions in topic. Must less than database limit. Default limit - 10.
- int32 partitions_count = 1 [(value) = "> 0"];
+ oneof partitioning {
+ // How many partitions in topic. Must less than database limit. Default limit - 10.
+ int32 partitions_count = 1 [(value) = "> 0"];
+ // Settings for the partitions count autoscaling.
+ AutoscalingSettings autoscaling_settings = 15;
+ };
+
oneof retention {
// How long data in partition should be stored. Must be greater than 0 and less than limit for this database.
// Default limit - 36 hours.
@@ -1165,6 +1184,40 @@ message TopicSettings {
RemoteMirrorRule remote_mirror_rule = 11;
}
+message AutoscalingSettings {
+ // Strategy of autoscaling.
+ AutoscalingStrategy strategy = 1;
+
+ // Auto merge would stop working when the partitions count reaches min_active_partitions.
+ // Zero value means default - 1.
+ int64 min_active_partitions = 2 [(Ydb.value) = ">= 0"];
+ // Auto split would stop working when the partitions count reaches max_active_partitions.
+ // Zero value means default - 1.
+ int64 max_active_partitions = 3 [(Ydb.value) = ">= 0"];
+ // Limit for total partition count, including active (open for write) and read-only partitions.
+ // Zero value means default - 100.
+ int64 partition_count_limit = 4 [(Ydb.value) = ">= 0", deprecated = true];
+
+ // Partition write speed autoscaling options.
+ AutoscalingPartitionWriteSpeedStrategy partition_write_speed = 5;
+}
+
+message AutoscalingPartitionWriteSpeedStrategy {
+ //Partition will be autoscaled up (divided into 2 partitions)
+ //after write speed to the partition exceeds scale_up_threshold_percent (in percentage of maximum write speed to the partition) for the period of time threshold_time_seconds
+
+ //Partition will become a candidate to the autoscaling down
+ //after write speed doesn’t reach scale_down_threshold_percent (in percentage of maximum write speed to the partition) for the period of time threshold_time_seconds
+ //This candidate partition will be autoscaled down when other neighbour partition will become a candidate to the autoscaling down and not earlier than a retention period.
+
+ // Zero value means default - 300.
+ google.protobuf.Duration threshold_time = 1;
+ // Zero value means default - 90.
+ int32 scale_up_threshold_percent = 2 [(Ydb.value) = ">= 0"];
+ // Zero value means default - 30.
+ int32 scale_down_threshold_percent = 3 [(Ydb.value) = ">= 0"];
+}
+
/**
* Create topic request sent from client to server.
*/
diff --git a/ydb/public/api/protos/ydb_topic.proto b/ydb/public/api/protos/ydb_topic.proto
index 127c9605a7e..79a2376af37 100644
--- a/ydb/public/api/protos/ydb_topic.proto
+++ b/ydb/public/api/protos/ydb_topic.proto
@@ -307,6 +307,7 @@ message StreamReadMessage {
StopPartitionSessionRequest stop_partition_session_request = 9;
UpdatePartitionSession update_partition_session = 10;
+ EndPartitionSession end_partition_session = 11;
}
}
@@ -321,6 +322,8 @@ message StreamReadMessage {
string reader_name = 3;
// Direct reading from a partition node.
bool direct_read = 4;
+ // Indicates that the SDK supports autoscaling.
+ bool autoscaling_support = 5;
message TopicReadSettings {
// Topic path.
@@ -559,6 +562,20 @@ message StreamReadMessage {
int64 direct_read_id = 2;
}
+ // Signal from server that client has finished reading the partition and all messages have been read.
+ // Once a partition has been finished no further messages will ever arrive to that partition.
+ // This command is a hint to the client to commit offsets, after which the child partitions will be balanced independently in different reading sessions.
+ // Unlike StopPartitionSessionRequest, the client does not have to close the reading session.
+ // Client should not send a response to the command.
+ message EndPartitionSession {
+ // Partition session identifier.
+ int64 partition_session_id = 1;
+
+ // Ids of partitions which were merged with the ended partition.
+ repeated int64 adjacent_partition_ids = 2;
+ // Ids of partitions which was formed when the ended partition was split or merged.
+ repeated int64 child_partition_ids = 3;
+ }
}
// Messages for bidirectional streaming rpc StreamDirectRead
@@ -797,14 +814,55 @@ message AlterConsumer {
map<string, string> alter_attributes = 6;
}
+enum AutoscalingStrategy {
+ // The autoscaling algorithm is not specified. The default value will be used.
+ AUTOSCALING_STRATEGY_UNSPECIFIED = 0;
+ // The autoscaling is disabled.
+ AUTOSCALING_STRATEGY_DISABLED = 1;
+ // The autoscaling algorithm will increase partitions count depending on the load characteristics.
+ // The autoscaling algorithm will never decrease the number of partitions.
+ AUTOSCALING_STRATEGY_SCALE_UP = 2;
+ // The autoscaling algorithm will both increase and decrease partitions count depending on the load characteristics.
+ AUTOSCALING_STRATEGY_SCALE_UP_AND_DOWN = 3;
+}
+
// Partitioning settings for topic.
message PartitioningSettings {
- // Minimum partition count auto merge would stop working at.
+ // Auto merge would stop working when the partitions count reaches min_active_partitions
// Zero value means default - 1.
int64 min_active_partitions = 1 [(Ydb.value) = ">= 0"];
+ // Auto split would stop working when the partitions count reaches max_active_partitions
+ // Zero value means default - 1.
+ int64 max_active_partitions = 3 [(Ydb.value) = ">= 0"];
// Limit for total partition count, including active (open for write) and read-only partitions.
// Zero value means default - 100.
- int64 partition_count_limit = 2 [(Ydb.value) = ">= 0"];
+ // Use max_active_partitions
+ int64 partition_count_limit = 2 [(Ydb.value) = ">= 0", deprecated = true];
+ // Settings for the partitions count autoscaling.
+ AutoscalingSettings autoscaling_settings = 4;
+}
+
+message AutoscalingSettings {
+ // Strategy of autoscaling.
+ AutoscalingStrategy strategy = 1;
+ // Partition write speed autoscaling options.
+ AutoscalingPartitionWriteSpeedStrategy partition_write_speed = 2;
+}
+
+message AutoscalingPartitionWriteSpeedStrategy {
+ //Partition will be autoscaled up (divided into 2 partitions)
+ //after write speed to the partition exceeds scale_up_threshold_percent (in percentage of maximum write speed to the partition) for the period of time threshold_time_seconds
+
+ //Partition will become a candidate to the autoscaling down
+ //after write speed doesn’t reach scale_down_threshold_percent (in percentage of maximum write speed to the partition) for the period of time threshold_time_seconds
+ //This candidate partition will be autoscaled down when other neighbour partition will become a candidate to the autoscaling down and not earlier than a retention period.
+
+ // Zero value means default - 300.
+ google.protobuf.Duration threshold_time = 1;
+ // Zero value means default - 90.
+ int32 scale_up_threshold_percent = 2 [(Ydb.value) = ">= 0"];
+ // Zero value means default - 30.
+ int32 scale_down_threshold_percent = 3 [(Ydb.value) = ">= 0"];
}
// Partitioning settings for topic.
@@ -812,9 +870,37 @@ message AlterPartitioningSettings {
// Minimum partition count auto merge would stop working at.
// Zero value means default - 1.
optional int64 set_min_active_partitions = 1 [(Ydb.value) = ">= 0"];
+ // Maximum partition count auto merge would stop working at.
+ // Zero value means default - 1.
+ optional int64 max_active_partitions = 3 [(Ydb.value) = ">= 0"];
// Limit for total partition count, including active (open for write) and read-only partitions.
// Zero value means default - 100.
- optional int64 set_partition_count_limit = 2 [(Ydb.value) = ">= 0"];
+ // Use set_max_active_partitions
+ optional int64 set_partition_count_limit = 2 [(Ydb.value) = ">= 0", deprecated = true];
+ // Settings for autoscaling the partition number
+ optional AlterAutoscalingSettings alter_autoscaling_settings = 4;
+}
+
+message AlterAutoscalingSettings {
+ // Strategy of autoscaling
+ optional AutoscalingStrategy set_strategy = 1;
+ // Autoscaling partition write speed options.
+ optional AlterAutoscalingPartitionWriteSpeedStrategy set_partition_write_speed = 2;
+}
+
+message AlterAutoscalingPartitionWriteSpeedStrategy {
+ // The time of exceeding the threshold value, after which the partition will be
+ // autoscaling.
+ // Zero value means default - 300.
+ optional google.protobuf.Duration set_threshold_time = 1;
+ // The threshold value of the write speed to the partition as a percentage, when exceeded,
+ // the partition will be auto split.
+ // Zero value means default - 90.
+ optional int32 set_scale_up_threshold_percent = 2 [(Ydb.value) = ">= 0"];
+ // The threshold value of the write speed to the partition as a percentage, if it is not reached,
+ // the partition will be auto merged.
+ // Zero value means default - 30.
+ optional int32 set_scale_down_threshold_percent = 3 [(Ydb.value) = ">= 0"];
}
// Metering mode specifies the method used to determine consumption of resources by the topic.