From d7e581e57e495934ad2ddff2324f9d2a38f97e4d Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Sat, 25 Jul 2026 13:48:10 +0200 Subject: [PATCH] *: re-vendor kvproto at b41e863 (2026-06-22) and record the vendored revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vendored protos were an unrecorded late-2023/early-2024 vintage (errorpb and pdpb untouched since #324, one hand-patched addition from #519). Nothing recorded which kvproto revision they came from, and the staleness now blocks real features. Re-vendor everything at pingcap/kvproto b41e86365ce0 — the revision client-go currently pins — and add proto/VERSION so the vintage is never again guesswork. Regenerated with the existing tonic-build pipeline (21 files). One mechanical source fix: pdpb.RequestHeader gained caller_component/caller_id, so the TSO stream's header construction now defaults the new fields. This commit is the re-vendor and nothing else. Newly available surface includes KvFlush / KvBufferBatchGet (pipelined DML), HealthFeedback, and pdpb.BatchScanRegions. Two new protocol semantics become VISIBLE with the refresh — errorpb.UndeterminedResult (an unknown raft apply outcome) and shared locks (SharedLock / SharedPessimisticLock, whose real holders live in shared_lock_infos). Handling them is deliberately left to follow-up PRs so this one stays a pure re-vendor. Neither is a regression introduced here: - UndeterminedResult falls through to the generic region-error arm, which backs off and retries — the same default the follow-up keeps for every plan that is not a commit point. - A shared-lock wrapper carries no key bytes on the wire regardless of which proto vintage parses it, so the keyspace truncation hazard predates this commit; the refresh only makes the wrapper identifiable. Tests: make check green; 67 lib tests; txn/raw/failpoint integration suites green against a local api-v2 cluster. Signed-off-by: Eduard R. --- proto/VERSION | 3 + proto/autoid.proto | 45 +- proto/brpb.proto | 1318 +++++++++----- proto/cdcpb.proto | 271 +-- proto/configpb.proto | 155 +- proto/coprocessor.proto | 259 ++- proto/deadlock.proto | 96 +- proto/debugpb.proto | 370 ++-- proto/diagnosticspb.proto | 106 +- proto/disaggregated.proto | 189 +- proto/disk_usage.proto | 16 +- proto/encryptionpb.proto | 175 +- proto/enginepb.proto | 94 +- proto/errorpb.proto | 221 +-- proto/gcpb.proto | 8 +- proto/import_kvpb.proto | 153 +- proto/import_sstpb.proto | 633 +++---- proto/include/eraftpb.proto | 202 +-- proto/include/gogoproto/gogo.proto | 8 + proto/include/rustproto.proto | 66 +- proto/keyspacepb.proto | 29 +- proto/kvrpcpb.proto | 2296 ++++++++++++++----------- proto/logbackuppb.proto | 55 +- proto/meta_storagepb.proto | 121 +- proto/metapb.proto | 197 ++- proto/mpp.proto | 109 +- proto/pdpb.proto | 1576 ++++++++++------- proto/raft_cmdpb.proto | 442 ++--- proto/raft_serverpb.proto | 408 ++--- proto/recoverdatapb.proto | 81 +- proto/replication_modepb.proto | 86 +- proto/resource_manager.proto | 156 +- proto/resource_usage_agent.proto | 47 +- proto/schedulingpb.proto | 274 +-- proto/tikvpb.proto | 498 +++--- proto/tracepb.proto | 64 +- proto/tsopb.proto | 150 +- src/generated/backup.rs | 566 +++++- src/generated/cdcpb.rs | 14 + src/generated/coprocessor.rs | 132 ++ src/generated/deadlock.rs | 25 +- src/generated/encryptionpb.rs | 85 + src/generated/errorpb.rs | 11 + src/generated/import_sstpb.rs | 169 +- src/generated/keyspacepb.rs | 33 + src/generated/kvrpcpb.rs | 381 +++- src/generated/logbackup.rs | 44 + src/generated/meta_storagepb.rs | 55 + src/generated/metapb.rs | 10 + src/generated/mpp.rs | 7 + src/generated/pdpb.rs | 665 ++++++- src/generated/raft_cmdpb.rs | 10 + src/generated/raft_serverpb.rs | 21 + src/generated/resource_manager.rs | 96 +- src/generated/resource_usage_agent.rs | 21 +- src/generated/schedulingpb.rs | 52 + src/generated/tikvpb.rs | 310 +++- src/generated/tsopb.rs | 10 + src/pd/timestamp.rs | 2 +- 59 files changed, 8931 insertions(+), 4765 deletions(-) create mode 100644 proto/VERSION diff --git a/proto/VERSION b/proto/VERSION new file mode 100644 index 00000000..6af27a69 --- /dev/null +++ b/proto/VERSION @@ -0,0 +1,3 @@ +pingcap/kvproto b41e86365ce0e0fba482005c0dfa5e2ca967d74c (2026-06-22) +Matches client-go v2.0.8-0.20260708122311-01bd8f99f4da (go.mod: kvproto v0.0.0-20260622063236-b41e86365ce0). +Re-vendor: copy proto/*.proto (matching names only) and include/ from that revision, then `make generate`. diff --git a/proto/autoid.proto b/proto/autoid.proto index 252ae58e..869b640d 100644 --- a/proto/autoid.proto +++ b/proto/autoid.proto @@ -4,44 +4,45 @@ package autoid; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - message AutoIDRequest { - int64 dbID = 1; - int64 tblID = 2; - bool isUnsigned = 3; - uint64 n = 4; - int64 increment = 5; - int64 offset = 6; - uint32 keyspaceID = 7; + int64 dbID = 1; + int64 tblID = 2; + bool isUnsigned = 3; + uint64 n = 4; + int64 increment = 5; + int64 offset = 6; + uint32 keyspaceID = 7; } message AutoIDResponse { - int64 min = 1; - int64 max = 2; + int64 min = 1; + int64 max = 2; - bytes errmsg = 3; + bytes errmsg = 3; } message RebaseRequest { - int64 dbID = 1; - int64 tblID = 2; - bool isUnsigned = 3; - int64 base = 4; - bool force = 5; + int64 dbID = 1; + int64 tblID = 2; + bool isUnsigned = 3; + int64 base = 4; + bool force = 5; } message RebaseResponse { - bytes errmsg = 1; + bytes errmsg = 1; } service AutoIDAlloc { - rpc AllocAutoID(AutoIDRequest) returns (AutoIDResponse) {} - rpc Rebase(RebaseRequest) returns (RebaseResponse) {} + rpc AllocAutoID(AutoIDRequest) returns (AutoIDResponse) {} + rpc Rebase(RebaseRequest) returns (RebaseResponse) {} } - diff --git a/proto/brpb.proto b/proto/brpb.proto index 13acb75c..f8c47d1d 100644 --- a/proto/brpb.proto +++ b/proto/brpb.proto @@ -1,325 +1,466 @@ syntax = "proto3"; package backup; -import "kvrpcpb.proto"; -import "errorpb.proto"; -import "metapb.proto"; import "encryptionpb.proto"; +import "errorpb.proto"; import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; +enum PrepareSnapshotBackupRequestType { + // Update the lease of suspending some commands (Which may impact the + // disk snapshot backup) to be proposed. + // As long as the lease is kept, those commands may not be proposed. + // If the client has gone, the lease will be destroyed after its TTL. + UpdateLease = 0; + // Wait a region apply to the last index. + WaitApply = 1; + // Hint before closing the stream. Server will go back to normal after this. + // This should also return a final "UpdateLeaseResult" to the client, which + // indices whether the last lease is valid. + Finish = 2; +} + +message PrepareSnapshotBackupRequest { + // The type of the request. + PrepareSnapshotBackupRequestType ty = 1; + // The regions related to the request. + // If the request is "UpdateLease" or "Finish", it should be empty. + repeated metapb.Region regions = 2; + // The lease duration for "UpdateLease" request. + uint64 lease_in_seconds = 3; +} + +enum PrepareSnapshotBackupEventType { + // A region has finished wait apply. + WaitApplyDone = 0; + // A lease has been updated. + UpdateLeaseResult = 1; +} + +message PrepareSnapshotBackupResponse { + // The type of the event. + PrepareSnapshotBackupEventType ty = 1; + // The region bind to the type. + // For "WaitApplyDone". + metapb.Region region = 2; + // The error of the execution. + errorpb.Error error = 3; + // Whether the last lease is valid. + // For "UpdateLease" and "Finish". + bool last_lease_is_valid = 4; +} // The message save the metadata of a backup. message BackupMeta { - // ID and version of backuped cluster. - uint64 cluster_id = 1; - string cluster_version = 2; - // Save the version of BR running backup jobs. - string br_version = 11; - // The backupmeta scheme version. - int32 version = 12; - - // path field is no longer used. - reserved 3; reserved "path"; - // A set of files that compose a backup. - // Note: `files` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - repeated File files = 4; - // An index to files contains data files. - MetaFile file_index = 13; - - // A pair of timestamp specifies a time range of a backup. - // For full backup, the start_version equals to the end_version, - // it means point in time. - // For incremental backup, the time range is specified as - // (start_version, end_version]. - uint64 start_version = 5; - uint64 end_version = 6; - - // Table metadata describes database and table info. - // Note: `schemas` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - repeated Schema schemas = 7; - // An index to files contains Schemas. - MetaFile schema_index = 14; - - // If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be - // ignored, and the backup data's range is represented by raw_ranges. - bool is_raw_kv = 8; - // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - repeated RawRange raw_ranges = 9; - - // An index to files contains RawRanges. - MetaFile raw_range_index = 15; - - // In incremental backup, DDLs which are completed in - // (lastBackupTS, backupTS] will be stored here. - // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for - // compatibility, so new BR can restore older backups. - bytes ddls = 10; - // An index to files contains DDLs. - MetaFile ddl_indexes = 16; - // the backup result into `backupmeta` file - string backup_result = 17; - - // API version implies the encode of the key and value. - kvrpcpb.APIVersion api_version = 18; - - // the placement policy info in backup cluster. we assume the policy won't be too much for one cluster. - repeated PlacementPolicy policies = 19; - - // new_collations_enabled specifies the config `new_collations_enabled_on_first_bootstrap` in tidb. - string new_collations_enabled = 20; - - // If in txn kv mode, `schemas` will be ignored, the backup data's range is as same as normal backup. - bool is_txn_kv = 21; - - // maintain the id mapping from upstream cluster to downstream cluster. - repeated PitrDBMap db_maps = 22; - - BackupMode mode = 23; - - // record the backup range and the correspond SST files when using file-copy backup. - repeated BackupRange ranges = 24; - - // record the size of the backup data files and meta files - uint64 backup_size = 25; + option (gogoproto.goproto_unrecognized) = true; + + // ID and version of backuped cluster. + uint64 cluster_id = 1; + string cluster_version = 2; + // Save the version of BR running backup jobs. + string br_version = 11; + // The backupmeta scheme version. + int32 version = 12; + + // path field is no longer used. + reserved 3; + reserved "path"; + // A set of files that compose a backup. + // Note: `files` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + repeated File files = 4; + // An index to files contains data files. + MetaFile file_index = 13; + + // A pair of timestamp specifies a time range of a backup. + // For full backup, the start_version equals to the end_version, + // it means point in time. + // For incremental backup, the time range is specified as + // (start_version, end_version]. + uint64 start_version = 5; + uint64 end_version = 6; + + // Table metadata describes database and table info. + // Note: `schemas` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + repeated Schema schemas = 7; + // An index to files contains Schemas. + MetaFile schema_index = 14; + // Backup metadata compatibility version for restore. + // 0 means the backup was created before this field existed. + uint32 backup_schema_version = 26; + + // If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be + // ignored, and the backup data's range is represented by raw_ranges. + bool is_raw_kv = 8; + // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + repeated RawRange raw_ranges = 9; + + // An index to files contains RawRanges. + MetaFile raw_range_index = 15; + + // In incremental backup, DDLs which are completed in + // (lastBackupTS, backupTS] will be stored here. + // Note: `raw_ranges` is deprecated, as it bloats backupmeta. It is kept for + // compatibility, so new BR can restore older backups. + bytes ddls = 10; + // An index to files contains DDLs. + MetaFile ddl_indexes = 16; + // the backup result into `backupmeta` file + string backup_result = 17; + + // API version implies the encode of the key and value. + kvrpcpb.APIVersion api_version = 18; + + // the placement policy info in backup cluster. we assume the policy won't be too much for one cluster. + repeated PlacementPolicy policies = 19; + + // new_collations_enabled specifies the config `new_collations_enabled_on_first_bootstrap` in tidb. + string new_collations_enabled = 20; + + // If in txn kv mode, `schemas` will be ignored, the backup data's range is as same as normal backup. + bool is_txn_kv = 21; + + // maintain the id mapping from upstream cluster to downstream cluster. + repeated PitrDBMap db_maps = 22; + + BackupMode mode = 23; + + // record the backup range and the correspond SST files when using file-copy backup. + repeated BackupRange ranges = 24; + + // record the size of the backup data files and meta files + uint64 backup_size = 25; } message BackupRange { - bytes start_key = 1; - bytes end_key = 2; + bytes start_key = 1; + bytes end_key = 2; + + repeated File files = 3; +} - repeated File files = 3; +// The message saves the physical table checksum +message TableMeta { + // physical id is either table id or partition id + int64 physical_id = 1; + uint64 crc64xor = 2; + uint64 total_kvs = 3; + uint64 total_bytes = 4; } message File { - string name = 1; - bytes sha256 = 2; + string name = 1; + bytes sha256 = 2; - bytes start_key = 3; - bytes end_key = 4; - uint64 start_version = 5; - uint64 end_version = 6; + bytes start_key = 3; + bytes end_key = 4; + uint64 start_version = 5; + uint64 end_version = 6; - uint64 crc64xor = 7; - uint64 total_kvs = 8; - uint64 total_bytes = 9; + // recorded the total statistics across multiple tables + uint64 crc64xor = 7; + uint64 total_kvs = 8; + uint64 total_bytes = 9; + // recorded when the SST kvs are from different tables + repeated TableMeta table_metas = 13; - string cf = 10; + string cf = 10; - uint64 size = 11; - // cipher_iv is used for AES cipher - bytes cipher_iv = 12; + uint64 size = 11; + // cipher_iv is used for AES cipher + bytes cipher_iv = 12; + + // Reserved for a downstream fork + reserved 1000 to 1199; } // MetaFile describes a multi-level index of data used in backup. message MetaFile { - // A set of files that contains a MetaFile. - // It is used as a multi-level index. - repeated File meta_files = 1; - // A set of files that contains user data. - repeated File data_files = 2; - // A set of files that contains Schemas. - repeated Schema schemas = 3; - // A set of files that contains RawRanges. - repeated RawRange raw_ranges = 4; - // A set of files that contains BackupRanges. - repeated BackupRange backup_ranges = 6; - // A set of files that contains DDLs. - repeated bytes ddls = 5; + // A set of files that contains a MetaFile. + // It is used as a multi-level index. + repeated File meta_files = 1; + // A set of files that contains user data. + repeated File data_files = 2; + // A set of files that contains Schemas. + repeated Schema schemas = 3; + // A set of files that contains RawRanges. + repeated RawRange raw_ranges = 4; + // A set of files that contains BackupRanges. + repeated BackupRange backup_ranges = 6; + // A set of files that contains DDLs. + repeated bytes ddls = 5; } message PlacementPolicy { - bytes info = 1; + bytes info = 1; +} + +message StatsBlock { + // The physical id which the stats json data belongs to. + int64 physical_id = 1; + + // The marshal data of the part of json_table. Currently it is the partition table. + bytes json_table = 2; +} + +message StatsFile { + // A set of files that contains json_tables. + repeated StatsBlock blocks = 1; +} + +message StatsFileIndex { + string name = 1; + bytes sha256 = 2; + + // the size of the file content encrypted + uint64 size_enc = 3; + // the size of the original file content + uint64 size_ori = 4; + // cipher_iv is used for AES cipher + bytes cipher_iv = 5; + + // If the size of the stats is very small, save the data into the metafile + bytes inline_data = 6; } message Schema { - bytes db = 1; - bytes table = 2; + bytes db = 1; + bytes table = 2; + + uint64 crc64xor = 3; + uint64 total_kvs = 4; + uint64 total_bytes = 5; + + uint32 tiflash_replicas = 6; - uint64 crc64xor = 3; - uint64 total_kvs = 4; - uint64 total_bytes = 5; + // stats represents the dump stats for a analyzed table, which generate by DumpStatsToJSON + // https://github.com/pingcap/tidb/blob/e136429d8dc5d70f43cd3f94179b0b9f47595097/statistics/handle/dump.go#L116 + bytes stats = 7; - uint32 tiflash_replicas = 6; + // An index of files that contains stats data + repeated StatsFileIndex stats_index = 8; - // stats represents the dump stats for a analyzed table, which generate by DumpStatsToJSON - // https://github.com/pingcap/tidb/blob/e136429d8dc5d70f43cd3f94179b0b9f47595097/statistics/handle/dump.go#L116 - bytes stats = 7; + // is_merge_option_allowed indicates whether merge option is allowed for this schema. + // For backward compatibility, this field defaults to false when not present. + bool is_merge_option_allowed = 9; + + // partition_merge_option_allowed indicates whether merge option is allowed for each partition. + // is_merge_option_allowed is table level, and the partition_merge_option_allowed is partition level. + map partition_merge_option_allowed = 10; } message IDMap { - int64 upstream_id = 1; - int64 downstream_id = 2; + int64 upstream_id = 1; + int64 downstream_id = 2; } -message PitrTableMap{ - string name = 1; - IDMap id_map = 2; - repeated IDMap partitions = 3; +message PitrTableMap { + string name = 1; + IDMap id_map = 2; + repeated IDMap partitions = 3; + // whether this table has been filtered out + bool filtered_out = 4; } -message PitrDBMap{ - string name = 1; - IDMap id_map = 2; - repeated PitrTableMap tables = 3; +message PitrDBMap { + string name = 1; + IDMap id_map = 2; + repeated PitrTableMap tables = 3; + // whether this db has been filtered out + bool filtered_out = 4; + // whether this db has been reused + // skip restoring the metakv of database in log restore if it is true + bool reused = 5; } message RawRange { - bytes start_key = 1; - bytes end_key = 2; - string cf = 3; + bytes start_key = 1; + bytes end_key = 2; + string cf = 3; } message ClusterIDError { - uint64 current = 1; - uint64 request = 2; + uint64 current = 1; + uint64 request = 2; } message Error { - string msg = 1; - oneof detail { - ClusterIDError cluster_id_error = 3; - kvrpcpb.KeyError kv_error = 4; - errorpb.Error region_error = 5; - } + string msg = 1; + oneof detail { + ClusterIDError cluster_id_error = 3; + kvrpcpb.KeyError kv_error = 4; + errorpb.Error region_error = 5; + } } // sst files or log files compression algorithm // for log files, unknown means not use compression algorithm enum CompressionType { - UNKNOWN = 0; - LZ4 = 1; - SNAPPY = 2; - ZSTD = 3; + UNKNOWN = 0; + LZ4 = 1; + SNAPPY = 2; + ZSTD = 3; +} + +message StreamBackupTaskSecurityConfig { + oneof encryption { + // not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig + CipherInfo plaintext_data_key = 1; + MasterKeyConfig master_key_config = 2; + } } // BackupMpde represents the mode of this whole backup request to the cluster. // and we need to store it in `backupmeta`. enum BackupMode { - // scan and generate precise SST file of a backup range. - // then we don't need to do anything in future restore. - // Note: this mode will cost the CPU resource of TiKV. - SCAN = 0; - // check and upload the coarse overlap SST files of a backup range. - // then we need to use a merge iterator to filter unexpected kv in future restore. - // Note: this mode will save the CPU resource of TiKV. - FILE = 1; + // scan and generate precise SST file of a backup range. + // then we don't need to do anything in future restore. + // Note: this mode will cost the CPU resource of TiKV. + SCAN = 0; + // check and upload the coarse overlap SST files of a backup range. + // then we need to use a merge iterator to filter unexpected kv in future restore. + // Note: this mode will save the CPU resource of TiKV. + FILE = 1; } message CipherInfo { - encryptionpb.EncryptionMethod cipher_type = 1; - bytes cipher_key = 2; + encryptionpb.EncryptionMethod cipher_type = 1; + bytes cipher_key = 2; +} + +message MasterKeyConfig { + encryptionpb.EncryptionMethod encryption_type = 1; + repeated encryptionpb.MasterKey master_keys = 2; +} + +message SortedSubRanges { + repeated kvrpcpb.KeyRange sub_ranges = 1; } message BackupRequest { - uint64 cluster_id = 1; - - bytes start_key = 2; - bytes end_key = 3; - uint64 start_version = 4; - uint64 end_version = 5; - - // path field is deprecated, use storage_backend instead - reserved 6; reserved "path"; - - // The I/O rate limit for backup request. - uint64 rate_limit = 7; - // The concurrency for executing the backup request in every tikv node. - uint32 concurrency = 8; - - StorageBackend storage_backend = 9; - - // If raw kv mode is enabled, `start_version` and `end_version` will be ignored, and `cf` - // specifies which cf to backup. - bool is_raw_kv = 10; - string cf = 11; - // algorithm used for compress sst files - CompressionType compression_type = 12; - // sst compression level, some algorithms support negative compression levels - int32 compression_level = 13; - // The cipher_info is Used to encrypt sst - CipherInfo cipher_info = 14; - - // dst_api_version indicates the key-value encoding version used by the - // generated SST file. Accepted values: - // 1. "v1": the generated SST files are encoded with api-v1, can be restored - // to TiKV clusters whose api version is set to v1. - // 2. "v2": the generated SST files are encoded with api-v2, can be restored - // to TiKV clusters whose api version is set to v2. - kvrpcpb.APIVersion dst_api_version = 15; - - // with checkpoint, some subintervals of the range have been backed up and recorded. - // only the remaining sub ranges of the range need to be backed up this time. - repeated kvrpcpb.KeyRange sub_ranges = 16; - - // replica_read indicates whether to use replica read for backup. - // If it is false, the backup will only read data from leader. - // If it is true, the backup will read data from both leader and follower. - bool replica_read = 17; - BackupMode mode = 18; - - // unique_id represents the handle of this backup. after we implement file-copy backup. - // we need generate some internal states during the whole backup precedure. - // this unique id is help to find the state effictively. - string unique_id = 19; - kvrpcpb.Context context = 20; + uint64 cluster_id = 1; + + bytes start_key = 2; + bytes end_key = 3; + uint64 start_version = 4; + uint64 end_version = 5; + + // path field is deprecated, use storage_backend instead + reserved 6; + reserved "path"; + + // The I/O rate limit for backup request. + uint64 rate_limit = 7; + // The concurrency for executing the backup request in every tikv node. + uint32 concurrency = 8; + + StorageBackend storage_backend = 9; + + // If raw kv mode is enabled, `start_version` and `end_version` will be ignored, and `cf` + // specifies which cf to backup. + bool is_raw_kv = 10; + string cf = 11; + // algorithm used for compress sst files + CompressionType compression_type = 12; + // sst compression level, some algorithms support negative compression levels + int32 compression_level = 13; + // The cipher_info is Used to encrypt sst + CipherInfo cipher_info = 14; + + // dst_api_version indicates the key-value encoding version used by the + // generated SST file. Accepted values: + // 1. "v1": the generated SST files are encoded with api-v1, can be restored + // to TiKV clusters whose api version is set to v1. + // 2. "v2": the generated SST files are encoded with api-v2, can be restored + // to TiKV clusters whose api version is set to v2. + kvrpcpb.APIVersion dst_api_version = 15; + + // with checkpoint, some subintervals of the range have been backed up and recorded. + // only the remaining sub ranges of the range need to be backed up this time. + repeated kvrpcpb.KeyRange sub_ranges = 16; + + // only ranges in the same groups can be merged into one SST file. + // the ranges in each groups are sorted. + repeated SortedSubRanges sorted_sub_ranges_groups = 21; + + // replica_read indicates whether to use replica read for backup. + // If it is false, the backup will only read data from leader. + // If it is true, the backup will read data from both leader and follower. + bool replica_read = 17; + BackupMode mode = 18; + + // unique_id represents the handle of this backup. after we implement file-copy backup. + // we need generate some internal states during the whole backup precedure. + // this unique id is help to find the state effictively. + string unique_id = 19; + kvrpcpb.Context context = 20; + + // Reserved for a downstream fork + reserved 1000 to 1199; } message StreamBackupTaskInfo { - // The storage for backup, parsed by BR. - StorageBackend storage = 1; - // The time range for backing up. - uint64 start_ts = 2; - uint64 end_ts = 3; - - // Misc meta datas. - // The name of the task, also the ID of the task. - string name = 4; - // The table filter of the task. - // Only for displaying the task info. - repeated string table_filter = 5; - - // compression type - CompressionType compression_type = 6; - - // The last timestamp of the task has been updated. - // This is a simple solution for unfrequent config changing: - // When we watched a config change(via polling or etcd watching), - // We perform a incremental scan between [last_update_ts, now), - // for filling the diff data during conf changing. - // The current implementation scan [0, now) for every task ranges newly added, - // So this field is reserved for future usage. - reserved "last_update_ts"; - - // We use '/tidb/br-stream/ranges// -> ' - // for storing all the target ranges of the task, the "ranges" field will not be saved in the taskinfo. - reserved "ranges"; - - // The status field allow us mark the task as 'paused'. - // We use '/tidb/br-stream/pause/ -> ()' for pausing a task for now. - // If we need more complexed status transforming, maybe add the status field back would be fine. - reserved "status"; + // The storage for backup, parsed by BR. + StorageBackend storage = 1; + // The time range for backing up. + uint64 start_ts = 2; + uint64 end_ts = 3; + + // Misc meta data. + // The name of the task, also the ID of the task. + string name = 4; + // The table filter of the task. + // Only for displaying the task info. + repeated string table_filter = 5; + + // compression type + CompressionType compression_type = 6; + + // security config for backup files + StreamBackupTaskSecurityConfig security_config = 7; + + // The last timestamp of the task has been updated. + // This is a simple solution for unfrequent config changing: + // When we watched a config change(via polling or etcd watching), + // We perform a incremental scan between [last_update_ts, now), + // for filling the diff data during conf changing. + // The current implementation scan [0, now) for every task ranges newly added, + // So this field is reserved for future usage. + reserved "last_update_ts"; + + // We use '/tidb/br-stream/ranges// -> ' + // for storing all the target ranges of the task, the "ranges" field will not be saved in the taskinfo. + reserved "ranges"; + + // The status field allow us mark the task as 'paused'. + // We use '/tidb/br-stream/pause/ -> ()' for pausing a task for now. + // If we need more complexed status transforming, maybe add the status field back would be fine. + reserved "status"; } message StorageBackend { - oneof backend { - Noop noop = 1; - Local local = 2; - S3 s3 = 3; - GCS gcs = 4; - CloudDynamic cloud_dynamic = 5; - HDFS hdfs = 6; - AzureBlobStorage azure_blob_storage = 7; - } + oneof backend { + Noop noop = 1; + Local local = 2; + S3 s3 = 3; + GCS gcs = 4; + CloudDynamic cloud_dynamic = 5; + HDFS hdfs = 6; + AzureBlobStorage azure_blob_storage = 7; + } } // Noop storage backend saves files into void. @@ -327,319 +468,546 @@ message Noop {} // Local storage backend saves files into local disk message Local { - string path = 1; + string path = 1; } // S3 storage backend saves files into S3 compatible storages // For non-aws providers, endpoint must be provided message S3 { - string endpoint = 1; - string region = 2; - string bucket = 3; - string prefix = 4; - string storage_class = 5; - // server side encryption - string sse = 6; - string acl = 7; - string access_key = 8; - string secret_access_key = 9; - bool force_path_style = 10; - string sse_kms_key_id = 11; - string role_arn = 12; - string external_id = 13; - bool object_lock_enabled = 14; - string session_token = 15; - string provider = 16; - string profile = 17; + string endpoint = 1; + string region = 2; + string bucket = 3; + string prefix = 4; + string storage_class = 5; + // server side encryption + string sse = 6; + string acl = 7; + string access_key = 8; + string secret_access_key = 9; + bool force_path_style = 10; + string sse_kms_key_id = 11; + string role_arn = 12; + string external_id = 13; + bool object_lock_enabled = 14; + string session_token = 15; + string provider = 16; + string profile = 17; } // GCS storage backend saves files into google cloud storage. message GCS { - string endpoint = 1; - string bucket = 2; - string prefix = 3; - string storage_class = 4; - // If not empty, applies a predefined set of access controls. - // See https://cloud.google.com/storage/docs/access-control/lists#predefined-acl - // for valid values. - string predefined_acl = 5; - // Service Account Credentials JSON blob - // You can get one from https://console.cloud.google.com/apis/credentials, and - // copy the content, set it as string here. - string credentials_blob = 6; + string endpoint = 1; + string bucket = 2; + string prefix = 3; + string storage_class = 4; + // If not empty, applies a predefined set of access controls. + // See https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + // for valid values. + string predefined_acl = 5; + // Service Account Credentials JSON blob + // You can get one from https://console.cloud.google.com/apis/credentials, and + // copy the content, set it as string here. + string credentials_blob = 6; } // The encryption algorithm must be AES256. message AzureCustomerKey { - // A Base64-encoded AES-256 encryption key value. - string encryption_key = 1; - // The Base64-encoded SHA256 of the encryption key. - string encryption_key_sha256 = 2; + // A Base64-encoded AES-256 encryption key value. + string encryption_key = 1; + // The Base64-encoded SHA256 of the encryption key. + string encryption_key_sha256 = 2; } // AzureBlobStorage storage backend saves files into azure blob storage. message AzureBlobStorage { - string endpoint = 1; - // Alias: container - string bucket = 2; - // Notice: prefix starts without `/`, otherwise the first directory's name is empty. - string prefix = 3; - // Alias: access_tier. - // See https://docs.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview - string storage_class = 4; - - // if empty, try to read account_name from the node's environment variable $AZURE_STORAGE_ACCOUNT. - string account_name = 5; - - // There are 3 kinds of credentials, and the priority order is - // `SAS > Shared key > Azure AD (env) > Shared key (env)`. - // - // 1. Use shared key to access the azure blob - // If the node's environment variables($AZURE_CLIENT_ID, $AZURE_TENANT_ID, $AZURE_CLIENT_SECRET) exist, - // prefer to use token to access the azure blob. - // - // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key - // - // Otherwise, if empty, try to read shared key from the node's environment variable $AZURE_STORAGE_KEY. - string shared_key = 6; - - // 2. Use Azure AD (Azure Active Directory) to access the azure blob - // - // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-azure-active-directory - // - // The Azure AD would generate the token, which tasks some time. - // So it is not recommanded to generate the token in each request. - // // AzureActiveDirectory azure_ad = #; - // - // 3. Use SAS (shared access signature) - // - // See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature - // - string access_sig = 8; - - // Server Side Encryption, 2 types in total: - // - // 1. Specify an encryption scope for uploaded blobs. - // - // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope - // - string encryption_scope = 9; - - // 2. Provide an encryption key on a request to blob storage. - // - // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys - // - AzureCustomerKey encryption_key = 10; + string endpoint = 1; + // Alias: container + string bucket = 2; + // Notice: prefix starts without `/`, otherwise the first directory's name is empty. + string prefix = 3; + // Alias: access_tier. + // See https://docs.microsoft.com/en-us/azure/storage/blobs/access-tiers-overview + string storage_class = 4; + + // if empty, try to read account_name from the node's environment variable $AZURE_STORAGE_ACCOUNT. + string account_name = 5; + + // There are 3 kinds of credentials, and the priority order is + // `SAS > Shared key > Azure AD (env) > Shared key (env)`. + // + // 1. Use shared key to access the azure blob + // If the node's environment variables($AZURE_CLIENT_ID, $AZURE_TENANT_ID, $AZURE_CLIENT_SECRET) exist, + // prefer to use token to access the azure blob. + // + // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key + // + // Otherwise, if empty, try to read shared key from the node's environment variable $AZURE_STORAGE_KEY. + string shared_key = 6; + + // 2. Use Azure AD (Azure Active Directory) to access the azure blob + // + // See https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-azure-active-directory + // + // The Azure AD would generate the token, which tasks some time. + // So it is not recommanded to generate the token in each request. + // // AzureActiveDirectory azure_ad = #; + // + // 3. Use SAS (shared access signature) + // + // See https://learn.microsoft.com/en-us/rest/api/storageservices/delegate-access-with-shared-access-signature + // + string access_sig = 8; + + // Server Side Encryption, 2 types in total: + // + // 1. Specify an encryption scope for uploaded blobs. + // + // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope + // + string encryption_scope = 9; + + // 2. Provide an encryption key on a request to blob storage. + // + // See https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys + // + AzureCustomerKey encryption_key = 10; } message Bucket { - string endpoint = 1; - string region = 3; - string bucket = 4; - string prefix = 5; - string storage_class = 6; + string endpoint = 1; + string region = 3; + string bucket = 4; + string prefix = 5; + string storage_class = 6; } // CloudDynamic allows testing new cloud providers and new fields without changing protobuf definitions message CloudDynamic { - Bucket bucket = 1; - string provider_name = 2; // s3, gcs and azureBlobStorage are supported - map attrs = 3; + Bucket bucket = 1; + string provider_name = 2; // s3, gcs and azureBlobStorage are supported + map attrs = 3; } // HDFS storage backend saves file into HDFS compatible storages message HDFS { - // a URL: hdfs:///some/path or hdfs://host:port/some/path - string remote = 1; + // a URL: hdfs:///some/path or hdfs://host:port/some/path + string remote = 1; } - message BackupResponse { - Error error = 1; - bytes start_key = 2; - bytes end_key = 3; - repeated File files = 4; + Error error = 1; + bytes start_key = 2; + bytes end_key = 3; + repeated File files = 4; - // API version implies the encode of the key and value. - kvrpcpb.APIVersion api_version = 5; + // API version implies the encode of the key and value. + kvrpcpb.APIVersion api_version = 5; } -message CleanupRequest { - // unique_id represents the unique handle of the whole backup predecure. - // it generated in prepare request and corrosponed to one specific backup. - string unique_id = 1; +message CleanupRequest { + // unique_id represents the unique handle of the whole backup predecure. + // it generated in prepare request and corrosponed to one specific backup. + string unique_id = 1; } message CleanupResponse { - Error error = 1; - bool success = 2; + Error error = 1; + bool success = 2; } -message PrepareRequest { - // whether save state to the storage. - bool save_to_storage = 1; +message PrepareRequest { + // whether save state to the storage. + bool save_to_storage = 1; } message PrepareResponse { - Error error = 1; - // unique_id represents the unique handle of the whole backup predecure. - // if unique_id = 0 means prepare failed. - // if unique_id > 0 means prepare success and all states saved with this unique info. - string unique_id = 2; - uint64 collect_file_count = 3; - uint64 collect_file_size = 4; + Error error = 1; + // unique_id represents the unique handle of the whole backup predecure. + // if unique_id = 0 means prepare failed. + // if unique_id > 0 means prepare success and all states saved with this unique info. + string unique_id = 2; + uint64 collect_file_count = 3; + uint64 collect_file_size = 4; } -message CheckAdminRequest { -} +message CheckAdminRequest {} message CheckAdminResponse { - Error error = 1; - metapb.Region region = 2; - bool has_pending_admin = 3; + Error error = 1; + metapb.Region region = 2; + bool has_pending_admin = 3; } service Backup { - rpc backup(BackupRequest) returns (stream BackupResponse) {} - // CheckPendingAdminOp used for snapshot backup. before we start snapshot for a TiKV. - // we need stop all schedule first and make sure all in-flight schedule has finished. - // this rpc check all pending conf change for leader. - rpc CheckPendingAdminOp(CheckAdminRequest) returns (stream CheckAdminResponse) {} + rpc backup(BackupRequest) returns (stream BackupResponse) {} + + // CheckPendingAdminOp used for snapshot backup. before we start snapshot for a TiKV. + // we need stop all schedule first and make sure all in-flight schedule has finished. + // this rpc check all pending conf change for leader. + rpc CheckPendingAdminOp(CheckAdminRequest) returns (stream CheckAdminResponse) {} - // prepare is used for file-copy backup. before we start the backup for a TiKV. - // we need invoke this function to generate the SST files map. or we get nothing to backup. - rpc prepare(PrepareRequest) returns (PrepareResponse) {} + // PrepareSnapshotBackup is an advanced version of preparing snapshot backup. + // Check the defination of `PrepareSnapshotBackupRequest` for more. + rpc PrepareSnapshotBackup(stream PrepareSnapshotBackupRequest) returns (stream PrepareSnapshotBackupResponse); - // cleanup used for file-copy backup. after we finish the backup for a TiKV. - // we need clean some internel state. e.g. checkpoint, SST File maps - rpc cleanup(CleanupRequest) returns (CleanupResponse) {} + // prepare is used for file-copy backup. before we start the backup for a TiKV. + // we need invoke this function to generate the SST files map. or we get nothing to backup. + rpc prepare(PrepareRequest) returns (PrepareResponse) {} + + // cleanup used for file-copy backup. after we finish the backup for a TiKV. + // we need clean some internel state. e.g. checkpoint, SST File maps + rpc cleanup(CleanupRequest) returns (CleanupResponse) {} } // ExternalStorage is a service for using a cloud backend from StorageBackend to store files. // This can be used to backup and restore SST files. service ExternalStorage { - // Restore to a file - rpc restore(ExternalStorageRestoreRequest) returns (ExternalStorageRestoreResponse) {} - // Save a file to storage - rpc save(ExternalStorageSaveRequest) returns (ExternalStorageSaveResponse) {} + // Restore to a file + rpc restore(ExternalStorageRestoreRequest) returns (ExternalStorageRestoreResponse) {} + // Save a file to storage + rpc save(ExternalStorageSaveRequest) returns (ExternalStorageSaveResponse) {} } message ExternalStorageRestoreRequest { - StorageBackend storage_backend = 1; - string object_name = 2; - string restore_name = 3; - uint64 content_length = 4; + StorageBackend storage_backend = 1; + string object_name = 2; + string restore_name = 3; + uint64 content_length = 4; } -message ExternalStorageRestoreResponse { -} +message ExternalStorageRestoreResponse {} message ExternalStorageSaveRequest { - StorageBackend storage_backend = 1; - string object_name = 2; - uint64 content_length = 3; + StorageBackend storage_backend = 1; + string object_name = 2; + uint64 content_length = 3; } -message ExternalStorageSaveResponse { -} +message ExternalStorageSaveResponse {} enum MetaVersion { - V1 = 0; - V2 = 1; + V1 = 0; + V2 = 1; } message Metadata { - // deprecated after(in) v6.3.0 TiKV cluster - repeated DataFileInfo files = 1; - repeated DataFileGroup file_groups = 6; - int64 store_id = 2; - uint64 resolved_ts = 3; - uint64 max_ts = 4; - uint64 min_ts = 5; - MetaVersion meta_version = 7; + // deprecated after(in) v6.3.0 TiKV cluster + repeated DataFileInfo files = 1; + repeated DataFileGroup file_groups = 6; + int64 store_id = 2; + uint64 resolved_ts = 3; + uint64 max_ts = 4; + uint64 min_ts = 5; + MetaVersion meta_version = 7; } enum FileType { - Delete = 0; - Put = 1; + Delete = 0; + Put = 1; } // DataFileGroup is the merged file info in log-backup message DataFileGroup { - // Path of the file. - string path = 1; + option (rustproto.carllerche_bytes_for_bytes) = true; + option (rustproto.carllerche_bytes_for_string) = true; + + // Path of the file. + string path = 1; - // Partitions of the file. - repeated DataFileInfo data_files_info = 2; + // Partitions of the file. + repeated DataFileInfo data_files_info = 2; - /// Below are extra information of the file, for better filtering files. - // The min ts of the keys in the file. - uint64 min_ts = 3; - // The max ts of the keys in the file. - uint64 max_ts = 4; - // The resolved ts of the region when saving the file. - uint64 min_resolved_ts = 5; + /// Below are extra information of the file, for better filtering files. + // The min ts of the keys in the file. + uint64 min_ts = 3; + // The max ts of the keys in the file. + uint64 max_ts = 4; + // The resolved ts of the region when saving the file. + uint64 min_resolved_ts = 5; - // The file length after compressed. - uint64 length = 6; + // The file length after compressed. + uint64 length = 6; } message DataFileInfo { - // SHA256 of the file. - bytes sha256 = 1; - // Path of the file. - string path = 2; - int64 number_of_entries = 3; - - /// Below are extra information of the file, for better filtering files. - // The min ts of the keys in the file. - uint64 min_ts = 4; - // The max ts of the keys in the file. - uint64 max_ts = 5; - // The resolved ts of the region when saving the file. - uint64 resolved_ts = 6; - // The region of the file. - int64 region_id = 7; - // The key range of the file. - // Encoded and starts with 'z'(internal key). - bytes start_key = 8; - bytes end_key = 9; - // The column family of the file. - string cf = 10; - // The operation type of the file. - FileType type = 11; - - // Whether the data file contains meta keys(m prefixed keys) only. - bool is_meta = 12; - // The table ID of the file contains, when `is_meta` is true, would be ignored. - int64 table_id = 13; - - // The file length. - uint64 length = 14; - - // The minimal begin ts in default cf if this file is write cf. - uint64 min_begin_ts_in_default_cf = 15; - - // Offset of the partition. compatible with V1 and V2. - uint64 range_offset = 16; - - // The range length of the merged file, if it exists. - uint64 range_length = 17; - - // The compression type for the file. - CompressionType compression_type = 18; - - // It may support encrypting at future. - reserved "iv"; + option (rustproto.carllerche_bytes_for_bytes) = true; + option (rustproto.carllerche_bytes_for_string) = true; + + // Checksum of the plaintext file, i.e., pre-compression, pre-encryption. + bytes sha256 = 1; + // Path of the file. + string path = 2; + int64 number_of_entries = 3; + + /// Below are extra information of the file, for better filtering files. + // The min ts of the keys in the file. + uint64 min_ts = 4; + // The max ts of the keys in the file. + uint64 max_ts = 5; + // The resolved ts of the region when saving the file. + uint64 resolved_ts = 6; + // The region of the file. + int64 region_id = 7; + // The key range of the file. + // Encoded and starts with 'z'(internal key). + bytes start_key = 8; + bytes end_key = 9; + // The column family of the file. + string cf = 10; + // The operation type of the file. + FileType type = 11; + + // Whether the data file contains meta keys(m prefixed keys) only. + bool is_meta = 12; + // The table ID of the file contains, when `is_meta` is true, would be ignored. + int64 table_id = 13; + + // The file length. + uint64 length = 14; + + // The minimal begin ts in default cf if this file is write cf. + uint64 min_begin_ts_in_default_cf = 15; + + // Offset of the partition. compatible with V1 and V2. + uint64 range_offset = 16; + + // The range length of the merged file, if it exists. + uint64 range_length = 17; + + // The compression type for the file. + CompressionType compression_type = 18; + + // The CRC64XOR of all contents of the file. + // In older versions, this might be empty. + uint64 crc64xor = 19; + + // The original region range the log file belongs to. + // In older versions, this might be empty. + // This may not be accurative: file may contain keys less than this. + bytes region_start_key = 20; + // The original region range the log file belongs to. + // In older versions, this might be empty. + // This may not be accurative: file may contain keys greater than this. + bytes region_end_key = 21; + // The region epoch that the log file belongs to. + // In older versions, this might be empty. + // If a region get split or merged during observing, the file may contain multi epoches. + // This may not be complete: file may contain records from other versions. + // + // If there is exactly one epoch, + // `region_start_key` and `region_end_key` must match this epoch. + repeated metapb.RegionEpoch region_epoch = 22; + + // Encryption information of this data file, not set if plaintext. + encryptionpb.FileEncryptionInfo file_encryption_info = 23; } message StreamBackupError { - // the unix epoch time (in millisecs) of the time the error reported. - uint64 happen_at = 1; - // the unified error code of the error. - string error_code = 2; - // the user-friendly error message. - string error_message = 3; - // the store id of who issues the error. - uint64 store_id = 4; + // the unix epoch time (in millisecs) of the time the error reported. + uint64 happen_at = 1; + // the unified error code of the error. + string error_code = 2; + // the user-friendly error message. + string error_message = 3; + // the store id of who issues the error. + uint64 store_id = 4; +} + +message Span { + uint64 offset = 1; + uint64 length = 2; +} + +message SpansOfFile { + string path = 1; + repeated Span spans = 2; +} + +message LogFileSubcompactionMeta { + // The whole size of this subcompaction. + // It should be the physical size of all input logs. + uint64 size = 1; + // The region that we perform the subcompaction. + uint64 region_id = 2; + string cf = 3; + + uint64 input_max_ts = 4; + uint64 input_min_ts = 5; + // The whole compaction's configure. + uint64 compact_from_ts = 6; + uint64 compact_until_ts = 7; + // The key range of the input logs. + // They are encoded user key. (i.e. mem-comparable, no 'z' prefix) + bytes min_key = 8; + // The largest key from the input. + bytes max_key = 9; + // The type of operations in the file. (Write or Delete) + FileType ty = 10; + int64 table_id = 11; + + // The input log files of this. + repeated SpansOfFile sources = 12; +} + +// A subcompaction from a compaction. +// Subcompactions are actions that collect then sort key values from one region. +message LogFileSubcompaction { + // The metadata of the subcompaction. + LogFileSubcompactionMeta meta = 1; + // The generated SSTs. + repeated File sst_outputs = 2; + // The hint for the history of the region that contains the keys being compacted. + // This may be helpful for splitting, but it does not guarantee completeness and accuracy. + repeated RegionMetaHint region_meta_hints = 3; +} + +message RegionMetaHint { + // You may get region id from `LogFileSubcompactionMeta`. + + // The left boundary of the region of this epoch. + // It is an byte-comparable encoded key without TS. + bytes start_key = 1; + // The right boundary of the region of the epoch. + bytes end_key = 2; + // The region epoch. + metapb.RegionEpoch region_epoch = 3; +} + +// Batched version of `LogFileSubcompaction`. +// So we can store many subcompactions to one file, to reduce the number of file. +message LogFileSubcompactions { + repeated LogFileSubcompaction subcompactions = 1; +} + +message LogFileCompaction { + // The selector of compaction source. + // Files contains ANY record greater than this (and less than `until_ts`) will be selected. + // The source and artificats may contain records out of the `from_ts` and `until_ts` range. + // When restoring, please use `input_min_ts` to filter out not needed compactions instead. + uint64 compaction_from_ts = 1; + // The selector of compaction source. + // Files contains ANY record less than this (and not less than `from_ts`) will be selected. + // The source and artificats may contain records out of the `from_ts` and `until_ts` range. + // When restoring, please use `input_max_ts` to filter out not needed compactions instead. + uint64 compaction_until_ts = 2; + // The user provided name for the compaction. + string name = 3; + // The statistics or other useful information for human. + string comments = 4; + + // relative path to the generated subcompaction metas. ("artificates") + string artifacts = 5; + // the XOR of all artificate's hash. + uint64 artifacts_hash = 6; + + // relative path to the subcompaction generated SST files. + // this is a hint for garbage-collecting, so we don't need to read the metas one by one. + // Those SSTs cannot be directly used, what you need is probably in `artificates`. + string generated_files = 7; + + // The smallest TS of record contained among all selected files of this compaction. + uint64 input_min_ts = 8; + // The largest TS of record contained among all selected files of this compaction. + uint64 input_max_ts = 9; + + // Total size of generated compact-log-backup cmeta files. + uint64 generated_cmeta_files_total_size = 10; + // Total size of generated compact-log-backup SST files. + uint64 generated_sst_files_total_size = 11; +} + +message MetaEdit { + // Path to the meta file. + string path = 1; + // Delete the physical files (MetaFileGroup) in the meta file. + repeated string delete_physical_files = 2; + // Delete the logical files (MetaFileInfo) in the meta file. + // Note: Even the operation have been performed in the meta, + // this modification should be kept as long as the corresponding physical + // file not deleted. Or we may cannot know when to delete the physical file. + // Then the file will be leak until truncated. + repeated DeleteSpansOfFile delete_logical_files = 3; + // Whether to delete the metadata itself. + bool destruct_self = 4; + + // Whether all data kv files have been compacted but meta kv file still remains. + // this field used to determine whether we can ignore the physical data kv log files in a compaction. + bool all_data_files_compacted = 5; +} + +// RewrittenTableID records a sort of modification over the SSTs during restoring. +// +// When "backing up" this "restored" SST, we want to backup the original SST before any rewrite. +// But in some cases, SSTs are not restored "as is", TiKV may rewrite or truncate its +// content by the client's request. +message RewrittenTableID { + // The table ID in the backup data. + int64 ancestor_upstream = 1; + // The rewritten table ID during restoring. + int64 upstream = 2; +} + +// IngestedSSTs is created by a client that wants to put data to the cluster by `Ingest` APIs, +// these write cannot be directly recorded by log backup. The client should put a migration +// that contains the SSTs it uploaded. +message IngestedSSTs { + // The table IDs rewritten during restoring. + repeated RewrittenTableID rewritten_tables = 1; + // The SST files restored. + repeated File files = 2; + // Treating the whole batch of SSTs as an huge atomic + // write as this timestamp. + // That is, when user wants to PiTR to sometime after this, + // everything restored will present after the PiTR. + // Otherwise nothing will present. + uint64 as_if_ts = 3; + // The hint of the common prefix of the files. + // Used for speed up garbage collecting. + string files_prefix_hint = 4; + // Whether this batch have finished. + // If not, `as_if_ts` should be ignored and this should only + // be restored when there is a finished backup with the same + // `backup_uuid`. + bool finished = 5; + // When checkpoint enabled, one restoration may be separated to many + // of `IngestedSST`s. + // `IngestedSST`s sharing the same UUID should be treated as one. + // That is: + // - The `as_if_ts` should be the largest one among all finished ingested SST batches. + // - When restoring, all batches sharing the same uuid should be restored or not. + bytes backup_uuid = 6; +} + +// An extended version of `SpansOfFile`, added more metadata for the +// execution of delayed deletion. +message DeleteSpansOfFile { + string path = 1; + repeated Span spans = 2; + + // This is a hint for garbage collecting. + // So we can easily check whether a set of spans consists the whole file. + uint64 whole_file_length = 3; +} + +enum MigrationVersion { + // The initial version. Somehow "UNKNOWN". + M0 = 0; + // Added `creator` and `version`. + M1 = 1; + // Added `ingested_sst_paths`. + M2 = 2; +} + +message Migration { + string creator = 5; + MigrationVersion version = 6; + + repeated MetaEdit edit_meta = 1; + uint64 truncated_to = 2; + repeated LogFileCompaction compactions = 3; + // Delete all files matches this prefix. + repeated string destruct_prefix = 4; + // The ingested SSTs. + // It is mutable during restoring, hence a reference. + repeated string ingested_sst_paths = 7; } diff --git a/proto/cdcpb.proto b/proto/cdcpb.proto index 6d27130f..3a89a204 100644 --- a/proto/cdcpb.proto +++ b/proto/cdcpb.proto @@ -1,191 +1,204 @@ syntax = "proto3"; package cdcpb; -import "raft_cmdpb.proto"; -import "metapb.proto"; import "errorpb.proto"; -import "kvrpcpb.proto"; - import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; +import "raft_cmdpb.proto"; import "rustproto.proto"; -option(gogoproto.sizer_all) = true; -option(gogoproto.marshaler_all) = true; -option(gogoproto.unmarshaler_all) = true; -option(rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message Header { - uint64 cluster_id = 1; - string ticdc_version = 2; + uint64 cluster_id = 1; + string ticdc_version = 2; } message DuplicateRequest { - uint64 region_id = 1; + uint64 region_id = 1; } message Compatibility { - string required_version = 1; + string required_version = 1; } // ClusterIDMismatch is an error variable that // tells people that the cluster ID of the request does not match the TiKV cluster ID. message ClusterIDMismatch { - // The current tikv cluster ID. - uint64 current = 1; - // The cluster ID of the TiCDC request. - uint64 request = 2; + // The current tikv cluster ID. + uint64 current = 1; + // The cluster ID of the TiCDC request. + uint64 request = 2; +} + +// Congested is an error variable that +// tells people that the TiKV-CDC is congested. +message Congested { + // The region ID that triggers the congestion. + uint64 region_id = 1; } message Error { - errorpb.NotLeader not_leader = 1; - errorpb.RegionNotFound region_not_found = 2; - errorpb.EpochNotMatch epoch_not_match = 3; - DuplicateRequest duplicate_request = 4; - Compatibility compatibility = 5; - ClusterIDMismatch cluster_id_mismatch = 6; - errorpb.ServerIsBusy server_is_busy = 7; + errorpb.NotLeader not_leader = 1; + errorpb.RegionNotFound region_not_found = 2; + errorpb.EpochNotMatch epoch_not_match = 3; + DuplicateRequest duplicate_request = 4; + Compatibility compatibility = 5; + ClusterIDMismatch cluster_id_mismatch = 6; + errorpb.ServerIsBusy server_is_busy = 7; + Congested congested = 8; } message TxnInfo { - uint64 start_ts = 1; - bytes primary = 2; + uint64 start_ts = 1; + bytes primary = 2; } message TxnStatus { - uint64 start_ts = 1; - uint64 min_commit_ts = 2; - uint64 commit_ts = 3; - bool is_rolled_back = 4; + uint64 start_ts = 1; + uint64 min_commit_ts = 2; + uint64 commit_ts = 3; + bool is_rolled_back = 4; } message Event { - enum LogType { - UNKNOWN = 0; - PREWRITE = 1; - COMMIT = 2; - ROLLBACK = 3; - COMMITTED = 4; - INITIALIZED = 5; - } - - message Row { - uint64 start_ts = 1; - uint64 commit_ts = 2; - LogType type = 3; - - enum OpType { - UNKNOWN = 0; - PUT = 1; - DELETE = 2; - } - OpType op_type = 4; - bytes key = 5; - bytes value = 6; - bytes old_value = 7; - // expire_ts_unix_secs is used for RawKV (see `ChangeDataRequest.KvApi`), - // and represents the expiration time of this row. - // Absolute time, seconds since Unix epoch. - uint64 expire_ts_unix_secs = 8; - // The source of this row. - uint64 txn_source = 9; - } - - message Entries { - repeated Row entries = 1; - } - - message Admin { - raft_cmdpb.AdminRequest admin_request = 1; - raft_cmdpb.AdminResponse admin_response = 2; - } + enum LogType { + UNKNOWN = 0; + PREWRITE = 1; + COMMIT = 2; + ROLLBACK = 3; + COMMITTED = 4; + INITIALIZED = 5; + } + + message Row { + uint64 start_ts = 1; + uint64 commit_ts = 2; + LogType type = 3; - message LongTxn { - repeated TxnInfo txn_info = 1; - } + // generation is for pipelined DML protocol. See kvrpcpb.FlushRequest.generation. + uint64 generation = 10; - uint64 region_id = 1; - uint64 index = 2; - uint64 request_id = 7; - oneof event { - Entries entries = 3; - Admin admin = 4; - Error error = 5; - uint64 resolved_ts = 6 [deprecated=true]; - // Note that field 7 is taken by request_id. - LongTxn long_txn = 8; - // More region level events ... + enum OpType { + UNKNOWN = 0; + PUT = 1; + DELETE = 2; } + OpType op_type = 4; + + bytes key = 5; + bytes value = 6; + bytes old_value = 7; + // expire_ts_unix_secs is used for RawKV (see `ChangeDataRequest.KvApi`), + // and represents the expiration time of this row. + // Absolute time, seconds since Unix epoch. + uint64 expire_ts_unix_secs = 8; + // The source of this row. + uint64 txn_source = 9; + } + + message Entries { + repeated Row entries = 1; + } + + message Admin { + raft_cmdpb.AdminRequest admin_request = 1; + raft_cmdpb.AdminResponse admin_response = 2; + } + + message LongTxn { + repeated TxnInfo txn_info = 1; + } + + uint64 region_id = 1; + uint64 index = 2; + uint64 request_id = 7; + oneof event { + Entries entries = 3; + Admin admin = 4; + Error error = 5; + uint64 resolved_ts = 6 [deprecated = true]; + // Note that field 7 is taken by request_id. + LongTxn long_txn = 8; + // More region level events ... + } } // NOTE: events and resolved_ts won't appear simultaneously in one ChangeDataEvent. message ChangeDataEvent { - repeated Event events = 1; - ResolvedTs resolved_ts = 2; - // More store level events ... + repeated Event events = 1; + ResolvedTs resolved_ts = 2; + // More store level events ... } message ResolvedTs { - repeated uint64 regions = 1; - uint64 ts = 2; - uint64 request_id = 3; + repeated uint64 regions = 1; + uint64 ts = 2; + uint64 request_id = 3; } message ChangeDataRequest { - message Register {} + message Register {} - message Deregister {} + message Deregister {} - message NotifyTxnStatus { - repeated TxnStatus txn_status = 1; - } + message NotifyTxnStatus { + repeated TxnStatus txn_status = 1; + } - Header header = 1; - uint64 region_id = 2; - metapb.RegionEpoch region_epoch = 3; + Header header = 1; + uint64 region_id = 2; + metapb.RegionEpoch region_epoch = 3; - uint64 checkpoint_ts = 4; - bytes start_key = 5; - bytes end_key = 6; - // Used for CDC to identify events corresponding to different requests. - // Generally in one call stream, a region can be subscribed multiple - // times with different `request_id`s. - uint64 request_id = 7; + uint64 checkpoint_ts = 4; + bytes start_key = 5; + bytes end_key = 6; + // Used for CDC to identify events corresponding to different requests. + // Generally in one call stream, a region can be subscribed multiple + // times with different `request_id`s. + uint64 request_id = 7; - kvrpcpb.ExtraOp extra_op = 8; + kvrpcpb.ExtraOp extra_op = 8; - oneof request { - // A normal request that trying to register change data feed on a region. - Register register = 9; + oneof request { + // A normal request that trying to register change data feed on a region. + Register register = 9; - // Notify the region that some of the running transactions on the region has a pushed - // min_commit_ts so that the resolved_ts can be advanced. - NotifyTxnStatus notify_txn_status = 10; + // Notify the region that some of the running transactions on the region has a pushed + // min_commit_ts so that the resolved_ts can be advanced. + NotifyTxnStatus notify_txn_status = 10; - Deregister deregister = 13; - } + Deregister deregister = 13; + } - // KvAPI specifies to capture data written by different KV API. - // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. - enum KvAPI { - TiDB = 0; - RawKV = 1; - TxnKV = 2; - } + // KvAPI specifies to capture data written by different KV API. + // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. + enum KvAPI { + TiDB = 0; + RawKV = 1; + TxnKV = 2; + } - KvAPI kv_api = 11; + KvAPI kv_api = 11; - // Whether to filter out the value write by cdc itself. - bool filter_loop = 12; + // Whether to filter out the value write by cdc itself. + bool filter_loop = 12; } service ChangeData { - rpc EventFeed(stream ChangeDataRequest) returns(stream ChangeDataEvent); + rpc EventFeed(stream ChangeDataRequest) returns (stream ChangeDataEvent); - // EventFeedV2 is like EventFeed, with some new changes: - // * clients send requested features in HTTP/2 headers; - // * if servers meets unsupported feature request, - // it can fail the stream with an UNIMPLEMENTED error. - rpc EventFeedV2(stream ChangeDataRequest) returns (stream ChangeDataEvent); + // EventFeedV2 is like EventFeed, with some new changes: + // * clients send requested features in HTTP/2 headers; + // * if servers meets unsupported feature request, + // it can fail the stream with an UNIMPLEMENTED error. + rpc EventFeedV2(stream ChangeDataRequest) returns (stream ChangeDataEvent); } diff --git a/proto/configpb.proto b/proto/configpb.proto index 0276328c..951d7e98 100644 --- a/proto/configpb.proto +++ b/proto/configpb.proto @@ -2,47 +2,46 @@ syntax = "proto3"; package configpb; import "gogoproto/gogo.proto"; -import "rustproto.proto"; import "google/api/annotations.proto"; +import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; service Config { - rpc Create(CreateRequest) returns (CreateResponse) {} - rpc GetAll(GetAllRequest) returns (GetAllResponse) {} - rpc Get(GetRequest) returns (GetResponse) { - option (google.api.http) = { - get: "/component" - }; - } - rpc Update(UpdateRequest) returns (UpdateResponse) { - option (google.api.http) = { - post: "/component" - body: "*" - }; - } - rpc Delete(DeleteRequest) returns (DeleteResponse) { - option (google.api.http) = { - delete: "/component" - }; - } + rpc Create(CreateRequest) returns (CreateResponse) {} + rpc GetAll(GetAllRequest) returns (GetAllResponse) {} + rpc Get(GetRequest) returns (GetResponse) { + option (google.api.http) = {get: "/component"}; + } + rpc Update(UpdateRequest) returns (UpdateResponse) { + option (google.api.http) = { + post: "/component" + body: "*" + }; + } + rpc Delete(DeleteRequest) returns (DeleteResponse) { + option (google.api.http) = {delete: "/component"}; + } } enum StatusCode { - UNKNOWN = 0; - OK = 1; - WRONG_VERSION = 2; - NOT_CHANGE = 3; - COMPONENT_NOT_FOUND = 4; - COMPONENT_ID_NOT_FOUND = 5; + UNKNOWN = 0; + OK = 1; + WRONG_VERSION = 2; + NOT_CHANGE = 3; + COMPONENT_NOT_FOUND = 4; + COMPONENT_ID_NOT_FOUND = 5; } message Status { - StatusCode code = 1; - string message = 2; + StatusCode code = 1; + string message = 2; } // The version is used to tell the configuration which can be shared @@ -52,102 +51,102 @@ message Status { // For local version, every component will have one to represent // the version of these configuration which cannot be shared. message Version { - uint64 local = 1; - uint64 global = 2; + uint64 local = 1; + uint64 global = 2; } message Local { - string component_id = 1; + string component_id = 1; } message Global { - string component = 1; + string component = 1; } message ConfigKind { - oneof kind { - Local local = 1; - Global global = 2; - } + oneof kind { + Local local = 1; + Global global = 2; + } } message ConfigEntry { - string name = 1; - string value = 2; + string name = 1; + string value = 2; } message LocalConfig { - Version version = 1; - string component = 2; - string component_id = 3; - string config = 4; + Version version = 1; + string component = 2; + string component_id = 3; + string config = 4; } message Header { - uint64 cluster_id = 1; + uint64 cluster_id = 1; } message CreateRequest { - Header header = 1; - Version version = 2; - string component = 3; - string component_id = 4; - string config = 5; + Header header = 1; + Version version = 2; + string component = 3; + string component_id = 4; + string config = 5; } message CreateResponse { - Header header = 1; - Status status = 2; - Version version = 3; - string config = 4; + Header header = 1; + Status status = 2; + Version version = 3; + string config = 4; } message GetAllRequest { - Header header = 1; + Header header = 1; } message GetAllResponse { - Header header = 1; - Status status = 2; - repeated LocalConfig local_configs = 3; + Header header = 1; + Status status = 2; + repeated LocalConfig local_configs = 3; } message GetRequest { - Header header = 1; - Version version = 2; - string component = 3; - string component_id = 4; + Header header = 1; + Version version = 2; + string component = 3; + string component_id = 4; } message GetResponse { - Header header = 1; - Status status = 2; - Version version = 3; - string config = 4; + Header header = 1; + Status status = 2; + Version version = 3; + string config = 4; } message UpdateRequest { - Header header = 1; - Version version = 2; - ConfigKind kind = 3; - repeated ConfigEntry entries = 4; + Header header = 1; + Version version = 2; + ConfigKind kind = 3; + repeated ConfigEntry entries = 4; } message UpdateResponse { - Header header = 1; - Status status = 2; - Version version = 3; - string config = 4; + Header header = 1; + Status status = 2; + Version version = 3; + string config = 4; } message DeleteRequest { - Header header = 1; - Version version = 2; - ConfigKind kind = 3; + Header header = 1; + Version version = 2; + ConfigKind kind = 3; } message DeleteResponse { - Header header = 1; - Status status = 2; - Version version = 3; + Header header = 1; + Status status = 2; + Version version = 3; } diff --git a/proto/coprocessor.proto b/proto/coprocessor.proto index d0f83a0d..222f0e53 100644 --- a/proto/coprocessor.proto +++ b/proto/coprocessor.proto @@ -2,121 +2,220 @@ syntax = "proto3"; package coprocessor; import "errorpb.proto"; -import "kvrpcpb.proto"; import "gogoproto/gogo.proto"; -import "rustproto.proto"; +import "kvrpcpb.proto"; import "metapb.proto"; +import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - - // [start, end) message KeyRange { - bytes start = 1; - bytes end = 2; + bytes start = 1; + bytes end = 2; +} + +// KeyRange with an attached read_ts (version). +// It is used by TiCI versioned lookup. Callers must ensure `range` is a point range. +message VersionedKeyRange { + KeyRange range = 1; + uint64 read_ts = 2; } message Request { - kvrpcpb.Context context = 1; - int64 tp = 2; - bytes data = 3; - uint64 start_ts = 7; - repeated KeyRange ranges = 4; - - // If cache is enabled, TiKV returns cache hit instead of data if - // its last version matches this `cache_if_match_version`. - bool is_cache_enabled = 5; - uint64 cache_if_match_version = 6; - // Any schema-ful storage to validate schema correctness if necessary. - int64 schema_ver = 8; - bool is_trace_enabled = 9; - // paging_size is 0 when it's disabled, otherwise, it should be a positive number. - uint64 paging_size = 10; - // tasks stores the batched coprocessor tasks sent to the same tikv store. - repeated StoreBatchTask tasks = 11; - uint64 connection_id = 12; // This is the session id between a client and tidb - string connection_alias = 13; // This is the session alias between a client and tidb + kvrpcpb.Context context = 1; + int64 tp = 2; + bytes data = 3; + uint64 start_ts = 7; + repeated KeyRange ranges = 4; + + // If cache is enabled, TiKV returns cache hit instead of data if + // its last version matches this `cache_if_match_version`. + bool is_cache_enabled = 5; + uint64 cache_if_match_version = 6; + // Any schema-ful storage to validate schema correctness if necessary. + int64 schema_ver = 8; + bool is_trace_enabled = 9; + // paging_size is 0 when it's disabled, otherwise, it should be a positive number. + uint64 paging_size = 10; + // tasks stores the batched coprocessor tasks sent to the same tikv store. + repeated StoreBatchTask tasks = 11; + uint64 connection_id = 12; // This is the session id between a client and tidb + string connection_alias = 13; // This is the session alias between a client and tidb + + repeated TableShardInfos table_shard_infos = 14; // Shard infos for FTS index, used by TiFlash reading TiCI. + // Versioned point ranges for TiCI lookup. + // When `versioned_ranges` is non-empty, all `versioned_ranges[i].range` must be point range. + repeated VersionedKeyRange versioned_ranges = 15; + // max_keys_read is 0 when disabled, otherwise limits the number of storage + // engine keys scanned per coprocessor task. It is a per-task hard ceiling + // applied uniformly to any coprocessor request, used to bound worst-case + // scan amplification regardless of how the request is paginated. + uint64 max_keys_read = 16; + // paging_size_bytes is 0 when disabled, otherwise it should be a positive number. + // When set, within a paged coprocessor request, TiKV stops scanning the current + // page once accumulated scanned bytes reach this limit and returns the page + // boundary so the next page can resume from there. + // + // Unlike max_keys_read, which is a per-task hard ceiling on keys scanned, + // paging_size_bytes is an independent rate-control dimension intended for + // Resource Control: it bounds the byte volume of a single page so that RU + // pre-charging in PD's resource controller can be performed at byte + // granularity, complementing the existing row-count-based paging + // (paging_size). The two fields target different scenarios and may be set + // independently. + uint64 paging_size_bytes = 17; } message Response { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; - errorpb.Error region_error = 2; - kvrpcpb.LockInfo locked = 3; - string other_error = 4; - KeyRange range = 5; - - // This field is always filled for compatibility consideration. However - // newer TiDB should respect `exec_details_v2` field instead. - kvrpcpb.ExecDetails exec_details = 6; - // This field is provided in later versions, containing more detailed - // information. - kvrpcpb.ExecDetailsV2 exec_details_v2 = 11; + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; + errorpb.Error region_error = 2; + kvrpcpb.LockInfo locked = 3; + string other_error = 4; + KeyRange range = 5; + + // This field is always filled for compatibility consideration. However + // newer TiDB should respect `exec_details_v2` field instead. + kvrpcpb.ExecDetails exec_details = 6; + // This field is provided in later versions, containing more detailed + // information. + kvrpcpb.ExecDetailsV2 exec_details_v2 = 11; + + bool is_cache_hit = 7; + uint64 cache_last_version = 8; + bool can_be_cached = 9; + + reserved 10; + + // Contains the latest buckets version of the region. + // Clients should query PD to update buckets in cache if its is stale. + uint64 latest_buckets_version = 12; + + // StoreBatchTaskResponse is the collection of batch task responses. + repeated StoreBatchTaskResponse batch_responses = 13; +} - bool is_cache_hit = 7; - uint64 cache_last_version = 8; - bool can_be_cached = 9; +message RegionInfo { + uint64 region_id = 1; + metapb.RegionEpoch region_epoch = 2; + repeated KeyRange ranges = 3; +} - reserved 10; +message ShardInfo { + uint64 shard_id = 1; + uint64 shard_epoch = 2; + repeated KeyRange ranges = 3; +} - // Contains the latest buckets version of the region. - // Clients should query PD to update buckets in cache if its is stale. - uint64 latest_buckets_version = 12; +message TableShardInfos { + // The executor ID is used to identify the tici executor. + string executor_id = 1; + // The shard_infos contains the shard information for each tici executor. + repeated ShardInfo shard_infos = 2; +} - // StoreBatchTaskResponse is the collection of batch task responses. - repeated StoreBatchTaskResponse batch_responses = 13; +message TiCIEstimateCountRequest { + kvrpcpb.Context context = 1; + uint64 start_ts = 2; + int64 table_id = 3; + int64 index_id = 4; + bytes fts_query_info = 5; + string time_zone_name = 6; + int64 time_zone_offset = 7; + repeated ShardInfo shard_infos = 8; } -message RegionInfo { - uint64 region_id = 1; - metapb.RegionEpoch region_epoch = 2; - repeated KeyRange ranges = 3; +message TiCIEstimateCountResponse { + uint64 est_count = 1; + string other_error = 2; } message TableRegions { - int64 physical_table_id = 1; - repeated RegionInfo regions = 2; + int64 physical_table_id = 1; + repeated RegionInfo regions = 2; } message BatchRequest { - kvrpcpb.Context context = 1; - int64 tp = 2; - bytes data = 3; - repeated RegionInfo regions = 4; - uint64 start_ts = 5; - // Any schema-ful storage to validate schema correctness if necessary. - int64 schema_ver = 6; - // Used for partition table scan - repeated TableRegions table_regions = 7; - string log_id = 8; - uint64 connection_id = 9; // This is the session id between a client and tidb - string connection_alias = 10; // This is the session alias between a client and tidb + kvrpcpb.Context context = 1; + int64 tp = 2; + bytes data = 3; + repeated RegionInfo regions = 4; + uint64 start_ts = 5; + // Any schema-ful storage to validate schema correctness if necessary. + int64 schema_ver = 6; + // Used for partition table scan + repeated TableRegions table_regions = 7; + string log_id = 8; + uint64 connection_id = 9; // This is the session id between a client and tidb + string connection_alias = 10; // This is the session alias between a client and tidb + repeated TableShardInfos table_shard_infos = 11; // Shard infos for FTS index, used by TiFlash reading TiCI. } message BatchResponse { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; - string other_error = 2; - kvrpcpb.ExecDetails exec_details = 3; - repeated metapb.Region retry_regions = 4; + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; + string other_error = 2; + kvrpcpb.ExecDetails exec_details = 3; + repeated metapb.Region retry_regions = 4; + repeated ShardInfo retry_shards = 5; } message StoreBatchTask { - uint64 region_id = 1; - metapb.RegionEpoch region_epoch = 2; - metapb.Peer peer = 3; - repeated KeyRange ranges = 4; - uint64 task_id = 5; + uint64 region_id = 1; + metapb.RegionEpoch region_epoch = 2; + metapb.Peer peer = 3; + repeated KeyRange ranges = 4; + uint64 task_id = 5; + // Versioned point ranges for TiCI lookup. + // When `versioned_ranges` is non-empty, all `versioned_ranges[i].range` must be point range. + repeated VersionedKeyRange versioned_ranges = 6; + // Bucket metadata version used to validate this child task. + uint64 buckets_version = 7; } message StoreBatchTaskResponse { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; - errorpb.Error region_error = 2; - kvrpcpb.LockInfo locked = 3; - string other_error = 4; - uint64 task_id = 5; - kvrpcpb.ExecDetailsV2 exec_details_v2 = 6; + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; + errorpb.Error region_error = 2; + kvrpcpb.LockInfo locked = 3; + string other_error = 4; + uint64 task_id = 5; + kvrpcpb.ExecDetailsV2 exec_details_v2 = 6; +} + +message DelegateRequest { + kvrpcpb.Context context = 1; + uint64 start_ts = 2; + repeated KeyRange ranges = 3; + // Used for avoid redundant mem-table copying. + // If the sequence is the same, tikv-server will not return the mem-table. + uint64 mem_table_sequence = 4; + + // Used for avoid redundant snapshot copying. + // If the sequence is the same, tikv-server will not return the snapshot. + uint64 snapshot_sequence = 5; +} + +message DelegateResponse { + bytes mem_table_data = 1; + bytes snapshot = 2; + errorpb.Error region_error = 3; + kvrpcpb.LockInfo locked = 4; + string other_error = 5; + // Used for avoid redundant mem-table copying. + uint64 mem_table_sequence = 6; } diff --git a/proto/deadlock.proto b/proto/deadlock.proto index da66a8c2..056c23cd 100644 --- a/proto/deadlock.proto +++ b/proto/deadlock.proto @@ -2,59 +2,81 @@ syntax = "proto3"; package deadlock; import "gogoproto/gogo.proto"; +import "rustproto.proto"; -message WaitForEntriesRequest { -} +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; + +message WaitForEntriesRequest {} message WaitForEntriesResponse { - repeated WaitForEntry entries = 1 [(gogoproto.nullable) = false]; + repeated WaitForEntry entries = 1 [(gogoproto.nullable) = false]; } message WaitForEntry { - // The transaction id that is waiting. - uint64 txn = 1; - // The transaction id that is being waited for. - uint64 wait_for_txn = 2; - // The hash value of the key is being waited for. - uint64 key_hash = 3; - // The key the current txn is trying to lock. - bytes key = 4; - // The tag came from the lock request's context. - bytes resource_group_tag = 5; - // Milliseconds it has been waits. - uint64 wait_time = 6; + // The transaction id that is waiting. + uint64 txn = 1; + // The transaction id that is being waited for. + uint64 wait_for_txn = 2; + // The hash value of the key is being waited for. + uint64 key_hash = 3; + // The key the current txn is trying to lock. + bytes key = 4; + // The tag came from the lock request's context. + bytes resource_group_tag = 5; + // Milliseconds it has been waits. + uint64 wait_time = 6; } enum DeadlockRequestType { - Detect = 0; - // CleanUpWaitFor cleans a single entry the transaction is waiting. - CleanUpWaitFor = 1; - // CleanUp cleans all entries the transaction is waiting. - CleanUp = 2; + Detect = 0; + // CleanUpWaitFor cleans a single entry the transaction is waiting. + CleanUpWaitFor = 1; + // CleanUp cleans all entries the transaction is waiting. + CleanUp = 2; +} + +message ReplaceLockByKeyItem { + uint64 key_hash = 1; + bytes key = 2; + uint64 old_lock_ts = 3; + uint64 new_lock_ts = 4; +} + +message ReplaceLocksByKeysRequest { + repeated ReplaceLockByKeyItem items = 1; } message DeadlockRequest { - DeadlockRequestType tp = 1; - WaitForEntry entry = 2 [(gogoproto.nullable) = false]; + DeadlockRequestType tp = 1; + WaitForEntry entry = 2 [(gogoproto.nullable) = false]; + ReplaceLocksByKeysRequest replace_locks_by_keys = 3; } message DeadlockResponse { - // The same entry sent by DeadlockRequest, identifies the sender. - WaitForEntry entry = 1 [(gogoproto.nullable) = false]; - // The key hash of the lock that is hold by the waiting transaction. - uint64 deadlock_key_hash = 2; - // The other entries of the dead lock circle. The current entry is in `entry` field and not - // included in this field. - repeated WaitForEntry wait_chain = 3; + // The same entry sent by DeadlockRequest, identifies the sender. + WaitForEntry entry = 1 [(gogoproto.nullable) = false]; + // The key hash of the lock that is hold by the waiting transaction. The hash of the `deadlock_key` field. + uint64 deadlock_key_hash = 2; + // The other entries of the dead lock circle. The current entry is in `entry` field and not + // included in this field. + repeated WaitForEntry wait_chain = 3; + // The key of the lock that is hold by the waiting transaction. + bytes deadlock_key = 4; } service Deadlock { - // Get local wait for entries, should be handle by every node. - // The owner should sent this request to all members to build the complete wait for graph. - rpc GetWaitForEntries(WaitForEntriesRequest) returns (WaitForEntriesResponse) {} - - // Detect should only sent to the owner. only be handled by the owner. - // The DeadlockResponse is sent back only if there is deadlock detected. - // CleanUpWaitFor and CleanUp doesn't return responses. - rpc Detect(stream DeadlockRequest) returns (stream DeadlockResponse) {} + // Get local wait for entries, should be handle by every node. + // The owner should sent this request to all members to build the complete wait for graph. + rpc GetWaitForEntries(WaitForEntriesRequest) returns (WaitForEntriesResponse) {} + + // Detect should only sent to the owner. only be handled by the owner. + // The DeadlockResponse is sent back only if there is deadlock detected. + // CleanUpWaitFor and CleanUp doesn't return responses. + rpc Detect(stream DeadlockRequest) returns (stream DeadlockResponse) {} } diff --git a/proto/debugpb.proto b/proto/debugpb.proto index dc612d14..65b3d492 100644 --- a/proto/debugpb.proto +++ b/proto/debugpb.proto @@ -2,18 +2,20 @@ syntax = "proto3"; package debugpb; import "eraftpb.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; import "raft_serverpb.proto"; -import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Debug service for TiKV. // // Errors are defined as follow: @@ -24,318 +26,308 @@ option java_package = "org.tikv.kvproto"; // reason can be found in grpc message. // Note: It bypasses raft layer. service Debug { - // Read a value arbitrarily for a key. - // Note: Server uses key directly w/o any encoding. - rpc Get(GetRequest) returns (GetResponse) {} + // Read a value arbitrarily for a key. + // Note: Server uses key directly w/o any encoding. + rpc Get(GetRequest) returns (GetResponse) {} - // Read raft info. - rpc RaftLog(RaftLogRequest) returns (RaftLogResponse) {} - rpc RegionInfo(RegionInfoRequest) returns (RegionInfoResponse) {} + // Read raft info. + rpc RaftLog(RaftLogRequest) returns (RaftLogResponse) {} + rpc RegionInfo(RegionInfoRequest) returns (RegionInfoResponse) {} - // Calculate size of a region. - // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. - rpc RegionSize(RegionSizeRequest) returns (RegionSizeResponse) {} + // Calculate size of a region. + // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. + rpc RegionSize(RegionSizeRequest) returns (RegionSizeResponse) {} - // Scan a specific range. - // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. - // Server uses keys directly w/o any encoding. - rpc ScanMvcc(ScanMvccRequest) returns (stream ScanMvccResponse) {} + // Scan a specific range. + // Note: DO NOT CALL IT IN PRODUCTION, it's really expensive. + // Server uses keys directly w/o any encoding. + rpc ScanMvcc(ScanMvccRequest) returns (stream ScanMvccResponse) {} - // Compact a column family in a specified range. - // Note: Server uses keys directly w/o any encoding. - rpc Compact(CompactRequest) returns (CompactResponse) {} + // Compact a column family in a specified range. + // Note: Server uses keys directly w/o any encoding. + rpc Compact(CompactRequest) returns (CompactResponse) {} - // Inject a fail point. Currently, it's only used in tests. - // Note: DO NOT CALL IT IN PRODUCTION. - rpc InjectFailPoint(InjectFailPointRequest) returns (InjectFailPointResponse) {} - // Recover from a fail point. - rpc RecoverFailPoint(RecoverFailPointRequest) returns (RecoverFailPointResponse) {} - // List all fail points. - rpc ListFailPoints(ListFailPointsRequest) returns (ListFailPointsResponse) {} + // Inject a fail point. Currently, it's only used in tests. + // Note: DO NOT CALL IT IN PRODUCTION. + rpc InjectFailPoint(InjectFailPointRequest) returns (InjectFailPointResponse) {} + // Recover from a fail point. + rpc RecoverFailPoint(RecoverFailPointRequest) returns (RecoverFailPointResponse) {} + // List all fail points. + rpc ListFailPoints(ListFailPointsRequest) returns (ListFailPointsResponse) {} - // Get Metrics - rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse){} + // Get Metrics + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} - // Do a consistent check for a region. - rpc CheckRegionConsistency(RegionConsistencyCheckRequest) returns (RegionConsistencyCheckResponse) {} + // Do a consistent check for a region. + rpc CheckRegionConsistency(RegionConsistencyCheckRequest) returns (RegionConsistencyCheckResponse) {} - // dynamically modify tikv's config - rpc ModifyTikvConfig(ModifyTikvConfigRequest) returns (ModifyTikvConfigResponse) {} + // dynamically modify tikv's config + rpc ModifyTikvConfig(ModifyTikvConfigRequest) returns (ModifyTikvConfigResponse) {} - // Get region properties - rpc GetRegionProperties(GetRegionPropertiesRequest) returns (GetRegionPropertiesResponse) {} + // Get region properties + rpc GetRegionProperties(GetRegionPropertiesRequest) returns (GetRegionPropertiesResponse) {} - // Get store ID - rpc GetStoreInfo(GetStoreInfoRequest) returns (GetStoreInfoResponse) {} + // Get store ID + rpc GetStoreInfo(GetStoreInfoRequest) returns (GetStoreInfoResponse) {} - // Get cluster ID - rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} + // Get cluster ID + rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} - // Get all region IDs in the store - rpc GetAllRegionsInStore(GetAllRegionsInStoreRequest) returns (GetAllRegionsInStoreResponse) {} + // Get all region IDs in the store + rpc GetAllRegionsInStore(GetAllRegionsInStoreRequest) returns (GetAllRegionsInStoreResponse) {} - // Make this TiKV node return to the status on this node to certain ts. - rpc ResetToVersion(ResetToVersionRequest) returns (ResetToVersionResponse) {} + // Make this TiKV node return to the status on this node to certain ts. + rpc ResetToVersion(ResetToVersionRequest) returns (ResetToVersionResponse) {} - // Get range properties - rpc GetRangeProperties(GetRangePropertiesRequest) returns (GetRangePropertiesResponse) {} + // Get range properties + rpc GetRangeProperties(GetRangePropertiesRequest) returns (GetRangePropertiesResponse) {} - // Flashback given key range to a specified version. - rpc FlashbackToVersion(FlashbackToVersionRequest) returns (FlashbackToVersionResponse) {} - - // GetRegionReadProgress returns the some useful info in RegionReadProgress - rpc GetRegionReadProgress(GetRegionReadProgressRequest) returns (GetRegionReadProgressResponse) {} + // Flashback given key range to a specified version. + rpc FlashbackToVersion(FlashbackToVersionRequest) returns (FlashbackToVersionResponse) {} + + // GetRegionReadProgress returns the some useful info in RegionReadProgress + rpc GetRegionReadProgress(GetRegionReadProgressRequest) returns (GetRegionReadProgressResponse) {} } enum DB { - INVALID = 0; - KV = 1; - RAFT = 2; + INVALID = 0; + KV = 1; + RAFT = 2; } enum MODULE { - UNUSED = 0; - KVDB = 1; - RAFTDB = 2; - READPOOL = 3; - SERVER = 4; - STORAGE = 5; - PD = 6; - METRIC = 7; - COPROCESSOR = 8; - SECURITY = 9; - IMPORT = 10; + UNUSED = 0; + KVDB = 1; + RAFTDB = 2; + READPOOL = 3; + SERVER = 4; + STORAGE = 5; + PD = 6; + METRIC = 7; + COPROCESSOR = 8; + SECURITY = 9; + IMPORT = 10; } message GetRequest { - DB db = 1; - string cf = 2; - bytes key = 3; + DB db = 1; + string cf = 2; + bytes key = 3; } message GetResponse { - bytes value = 1; + bytes value = 1; } message RaftLogRequest { - uint64 region_id = 1; - uint64 log_index = 2; + uint64 region_id = 1; + uint64 log_index = 2; } message RaftLogResponse { - eraftpb.Entry entry = 1; + eraftpb.Entry entry = 1; } message RegionInfoRequest { - uint64 region_id = 1; + uint64 region_id = 1; } message RegionInfoResponse { - raft_serverpb.RaftLocalState raft_local_state = 1; - raft_serverpb.RaftApplyState raft_apply_state = 2; - raft_serverpb.RegionLocalState region_local_state = 3; + raft_serverpb.RaftLocalState raft_local_state = 1; + raft_serverpb.RaftApplyState raft_apply_state = 2; + raft_serverpb.RegionLocalState region_local_state = 3; } message RegionSizeRequest { - uint64 region_id = 1; - repeated string cfs = 2; + uint64 region_id = 1; + repeated string cfs = 2; } message RegionSizeResponse { - message Entry { - string cf = 1; - uint64 size = 2; - } + message Entry { + string cf = 1; + uint64 size = 2; + } - repeated Entry entries = 1; + repeated Entry entries = 1; } message ScanMvccRequest { - bytes from_key = 1; - bytes to_key = 2; - uint64 limit = 3; + bytes from_key = 1; + bytes to_key = 2; + uint64 limit = 3; } message ScanMvccResponse { - bytes key = 1; - kvrpcpb.MvccInfo info = 2; + bytes key = 1; + kvrpcpb.MvccInfo info = 2; } enum BottommostLevelCompaction { - // Skip bottommost level compaction - Skip = 0; - // Force bottommost level compaction - Force = 1; - // Compact bottommost level if there is a compaction filter. - IfHaveCompactionFilter = 2; + // Skip bottommost level compaction + Skip = 0; + // Force bottommost level compaction + Force = 1; + // Compact bottommost level if there is a compaction filter. + IfHaveCompactionFilter = 2; } message CompactRequest { - DB db = 1; - string cf = 2; - bytes from_key = 3; - bytes to_key = 4; - uint32 threads = 5; - BottommostLevelCompaction bottommost_level_compaction = 6; + DB db = 1; + string cf = 2; + bytes from_key = 3; + bytes to_key = 4; + uint32 threads = 5; + BottommostLevelCompaction bottommost_level_compaction = 6; } -message CompactResponse { -} +message CompactResponse {} message InjectFailPointRequest { - string name = 1; - string actions = 2; + string name = 1; + string actions = 2; } -message InjectFailPointResponse { -} +message InjectFailPointResponse {} message RecoverFailPointRequest { - string name = 1; + string name = 1; } -message RecoverFailPointResponse { -} +message RecoverFailPointResponse {} -message ListFailPointsRequest { -} +message ListFailPointsRequest {} message ListFailPointsResponse { - message Entry { - string name = 1; - string actions = 2; - } + message Entry { + string name = 1; + string actions = 2; + } - repeated Entry entries = 1; + repeated Entry entries = 1; } message GetMetricsRequest { - bool all = 1; + bool all = 1; } message GetMetricsResponse { - string prometheus = 1; - string rocksdb_kv = 2; - string rocksdb_raft = 3; - string jemalloc = 4; - uint64 store_id = 5; + string prometheus = 1; + string rocksdb_kv = 2; + string rocksdb_raft = 3; + string jemalloc = 4; + uint64 store_id = 5; } message RegionConsistencyCheckRequest { - uint64 region_id = 1; + uint64 region_id = 1; } -message RegionConsistencyCheckResponse { -} +message RegionConsistencyCheckResponse {} message ModifyTikvConfigRequest { - MODULE module = 1; - string config_name = 2; - string config_value = 3; + MODULE module = 1; + string config_name = 2; + string config_value = 3; } -message ModifyTikvConfigResponse { -} +message ModifyTikvConfigResponse {} message Property { - string name = 1; - string value = 2; + string name = 1; + string value = 2; } message GetRegionPropertiesRequest { - uint64 region_id = 1; + uint64 region_id = 1; } message GetRegionPropertiesResponse { - repeated Property props = 1; + repeated Property props = 1; } -message GetStoreInfoRequest { -} +message GetStoreInfoRequest {} message GetStoreInfoResponse { - uint64 store_id = 1; - kvrpcpb.APIVersion api_version = 2; + uint64 store_id = 1; + kvrpcpb.APIVersion api_version = 2; } -message GetClusterInfoRequest { -} +message GetClusterInfoRequest {} message GetClusterInfoResponse { - uint64 cluster_id = 1; + uint64 cluster_id = 1; } -message GetAllRegionsInStoreRequest { -} +message GetAllRegionsInStoreRequest {} message GetAllRegionsInStoreResponse { - repeated uint64 regions = 1; + repeated uint64 regions = 1; } message ResetToVersionRequest { - uint64 ts = 1; + uint64 ts = 1; } -message ResetToVersionResponse { -} +message ResetToVersionResponse {} message GetRangePropertiesRequest { - bytes start_key = 1; - bytes end_key = 2; + bytes start_key = 1; + bytes end_key = 2; } message GetRangePropertiesResponse { - message RangeProperty { - string key = 1; - string value = 2; - } + message RangeProperty { + string key = 1; + string value = 2; + } - repeated RangeProperty properties = 1; + repeated RangeProperty properties = 1; } message FlashbackToVersionRequest { - kvrpcpb.Context context = 1; - uint64 version = 2; - uint64 region_id = 3; - bytes start_key = 4; - bytes end_key = 5; - uint64 start_ts = 6; - uint64 commit_ts = 7; + kvrpcpb.Context context = 1; + uint64 version = 2; + uint64 region_id = 3; + bytes start_key = 4; + bytes end_key = 5; + uint64 start_ts = 6; + uint64 commit_ts = 7; } message FlashbackToVersionResponse { - string error = 1; + string error = 1; } message GetRegionReadProgressRequest { - uint64 region_id = 1; - bool log_locks = 2; // when set to true, print a log of the locks with min start_ts in the resolver. - uint64 min_start_ts = 3; // only print locks whose start_ts >= min_start_ts. Can be used to find certain transaction. + uint64 region_id = 1; + bool log_locks = 2; // when set to true, print a log of the locks with min start_ts in the resolver. + uint64 min_start_ts = 3; // only print locks whose start_ts >= min_start_ts. Can be used to find certain transaction. } message GetRegionReadProgressResponse { - // below are from region_read_progress module - uint64 safe_ts = 1; - uint64 applied_index = 2; - uint64 pending_front_applied_index = 3; - uint64 pending_front_ts = 4; - uint64 pending_back_applied_index = 5; - uint64 pending_back_ts = 6; - bool region_read_progress_paused = 7; - uint64 duration_to_last_update_safe_ts_ms = 8; - uint64 duration_to_last_consume_leader_ms = 9; - bool region_read_progress_exist = 10; - uint64 read_state_ts = 18; - uint64 read_state_apply_index = 19; - bool discard = 20; - - // below are from resolved-ts module - uint64 resolved_ts = 11; - uint64 resolver_tracked_index = 12; - bool resolver_exist = 13; - bool resolver_stopped = 14; - uint64 num_locks = 16; - uint64 num_transactions = 17; - - string error = 15; -} \ No newline at end of file + // below are from region_read_progress module + uint64 safe_ts = 1; + uint64 applied_index = 2; + uint64 pending_front_applied_index = 3; + uint64 pending_front_ts = 4; + uint64 pending_back_applied_index = 5; + uint64 pending_back_ts = 6; + bool region_read_progress_paused = 7; + uint64 duration_to_last_update_safe_ts_ms = 8; + uint64 duration_to_last_consume_leader_ms = 9; + bool region_read_progress_exist = 10; + uint64 read_state_ts = 18; + uint64 read_state_apply_index = 19; + bool discard = 20; + + // below are from resolved-ts module + uint64 resolved_ts = 11; + uint64 resolver_tracked_index = 12; + bool resolver_exist = 13; + bool resolver_stopped = 14; + uint64 num_locks = 16; + uint64 num_transactions = 17; + + string error = 15; +} diff --git a/proto/diagnosticspb.proto b/proto/diagnosticspb.proto index 548b63ad..dc63d936 100644 --- a/proto/diagnosticspb.proto +++ b/proto/diagnosticspb.proto @@ -4,89 +4,91 @@ package diagnosticspb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Diagnostics service for TiDB cluster components. service Diagnostics { - // Searchs log in the target node - rpc search_log(SearchLogRequest) returns (stream SearchLogResponse) {}; - // Retrieves server info in the target node - rpc server_info(ServerInfoRequest) returns (ServerInfoResponse) {}; + // Searchs log in the target node + rpc search_log(SearchLogRequest) returns (stream SearchLogResponse) {} + // Retrieves server info in the target node + rpc server_info(ServerInfoRequest) returns (ServerInfoResponse) {} } enum LogLevel { - UNKNOWN = 0; - Debug = 1; - Info = 2; - Warn = 3; - Trace = 4; - Critical = 5; - Error = 6; + UNKNOWN = 0; + Debug = 1; + Info = 2; + Warn = 3; + Trace = 4; + Critical = 5; + Error = 6; } message SearchLogRequest { - enum Target { - Normal = 0; - Slow = 1; - } - int64 start_time = 1; - int64 end_time = 2; - repeated LogLevel levels = 3; - // We use a string array to represent multiple CNF pattern sceniaor like: - // SELECT * FROM t WHERE c LIKE '%s%' and c REGEXP '.*a.*' because - // Golang and Rust don't support perl-like (?=re1)(?=re2) - repeated string patterns = 4; - Target target = 5; + enum Target { + Normal = 0; + Slow = 1; + } + int64 start_time = 1; + int64 end_time = 2; + repeated LogLevel levels = 3; + // We use a string array to represent multiple CNF pattern sceniaor like: + // SELECT * FROM t WHERE c LIKE '%s%' and c REGEXP '.*a.*' because + // Golang and Rust don't support perl-like (?=re1)(?=re2) + repeated string patterns = 4; + Target target = 5; } message SearchLogResponse { - repeated LogMessage messages = 1; + repeated LogMessage messages = 1; } message LogMessage { - int64 time = 1; - LogLevel level = 2; - string message = 3; + int64 time = 1; + LogLevel level = 2; + string message = 3; } enum ServerInfoType { - All = 0; - HardwareInfo = 1; - SystemInfo = 2; - LoadInfo = 3; + All = 0; + HardwareInfo = 1; + SystemInfo = 2; + LoadInfo = 3; } message ServerInfoRequest { - ServerInfoType tp = 1; + ServerInfoType tp = 1; } message ServerInfoPair { - string key = 1; - string value = 2; + string key = 1; + string value = 2; } message ServerInfoItem { - // cpu, memory, disk, network ... - string tp = 1; - // eg. network: lo1/eth0, cpu: core1/core2, disk: sda1/sda2 - string name = 2; - // all key-value pairs for specified item, e.g: - // ServerInfoItem { - // tp = "network" - // name = "eth0" - // paris = [ - // ServerInfoPair { key = "readbytes", value = "4k"}, - // ServerInfoPair { key = "writebytes", value = "1k"}, - // ] - // } - repeated ServerInfoPair pairs = 3; + // cpu, memory, disk, network ... + string tp = 1; + // eg. network: lo1/eth0, cpu: core1/core2, disk: sda1/sda2 + string name = 2; + // all key-value pairs for specified item, e.g: + // ServerInfoItem { + // tp = "network" + // name = "eth0" + // paris = [ + // ServerInfoPair { key = "readbytes", value = "4k"}, + // ServerInfoPair { key = "writebytes", value = "1k"}, + // ] + // } + repeated ServerInfoPair pairs = 3; } message ServerInfoResponse { - repeated ServerInfoItem items = 1; + repeated ServerInfoItem items = 1; } diff --git a/proto/disaggregated.proto b/proto/disaggregated.proto index adaa8f61..445e8ffd 100644 --- a/proto/disaggregated.proto +++ b/proto/disaggregated.proto @@ -2,168 +2,177 @@ syntax = "proto3"; package disaggregated; import "coprocessor.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; +import "rustproto.proto"; option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; /// For S3 lock service /// message S3LockResult { - oneof error { - Success success = 1; - NotOwner not_owner = 2; - Conflict conflict = 3; - } + oneof error { + Success success = 1; + NotOwner not_owner = 2; + Conflict conflict = 3; + } } message Success {} + // Error caused by S3GC owner changed // client should retry -message NotOwner{ -} +message NotOwner {} + // Error caused by concurrency conflict, // request cancel message Conflict { - string reason = 1; + string reason = 1; } message TryAddLockRequest { - // The data file key to add lock - bytes data_file_key = 1; - // The lock store id - uint64 lock_store_id = 3; - // The upload sequence number of lock store - uint64 lock_seq = 4; + // The data file key to add lock + bytes data_file_key = 1; + // The lock store id + uint64 lock_store_id = 3; + // The upload sequence number of lock store + uint64 lock_seq = 4; } message TryAddLockResponse { - S3LockResult result = 1; + S3LockResult result = 1; } message TryMarkDeleteRequest { - // The data file key to be marked as deleted - bytes data_file_key = 1; + // The data file key to be marked as deleted + bytes data_file_key = 1; } message TryMarkDeleteResponse { - S3LockResult result = 1; + S3LockResult result = 1; } /// For disagg compute node init its disagg configuration /// -message GetDisaggConfigRequest { -} +message GetDisaggConfigRequest {} message DisaggS3Config { - string bucket = 1; - string root = 2; - string endpoint = 3; + string bucket = 1; + string root = 2; + string endpoint = 3; } message GetDisaggConfigResponse { - DisaggS3Config s3_config = 1; + DisaggS3Config s3_config = 1; } /// For compute task dispatch and data exchange /// message DisaggTaskMeta { - uint64 start_ts = 1; // start ts of a query - // gather_id + query_ts + server_id + local_query_id to represent a global unique query. - int64 gather_id = 9; // used to distinguish different gathers in the mpp query - uint64 query_ts = 2; // timestamp when start to execute query, used for TiFlash miniTSO schedule. - uint64 server_id = 3; // TiDB server id - uint64 local_query_id = 4; // unique local query_id if tidb don't restart. - int64 task_id = 5; // if task id is -1 , it indicates a tidb task. - string executor_id = 6; // the exectuor id - uint32 keyspace_id = 7; // keyspace id of the request - kvrpcpb.APIVersion api_version = 8; // API version of the request - uint64 connection_id = 10; // This is the session id between a client and tidb - string connection_alias = 11; // This is the session alias between a client and tidb + uint64 start_ts = 1; // start ts of a query + // gather_id + query_ts + server_id + local_query_id to represent a global unique query. + int64 gather_id = 9; // used to distinguish different gathers in the mpp query + uint64 query_ts = 2; // timestamp when start to execute query, used for TiFlash miniTSO schedule. + uint64 server_id = 3; // TiDB server id + uint64 local_query_id = 4; // unique local query_id if tidb don't restart. + int64 task_id = 5; // if task id is -1 , it indicates a tidb task. + string executor_id = 6; // the exectuor id + uint32 keyspace_id = 7; // keyspace id of the request + kvrpcpb.APIVersion api_version = 8; // API version of the request + uint64 connection_id = 10; // This is the session id between a client and tidb + string connection_alias = 11; // This is the session alias between a client and tidb } message DisaggReadError { - int32 code = 1; - string msg = 2; + int32 code = 1; + string msg = 2; } message EstablishDisaggTaskError { - oneof errors { - ErrorRegion error_region = 1; - ErrorLocked error_locked = 2; + oneof errors { + ErrorRegion error_region = 1; + ErrorLocked error_locked = 2; - ErrorOther error_other = 99; - } + ErrorOther error_other = 99; + } } message ErrorRegion { - string msg = 1; - // The read node needs to update its region cache about these regions. - repeated uint64 region_ids = 2; + string msg = 1; + // The read node needs to update its region cache about these regions. + repeated uint64 region_ids = 2; } message ErrorLocked { - string msg = 1; - // The read node needs to resolve these locks. - repeated kvrpcpb.LockInfo locked = 2; + string msg = 1; + // The read node needs to resolve these locks. + repeated kvrpcpb.LockInfo locked = 2; } message ErrorOther { - int32 code = 1; - string msg = 2; + int32 code = 1; + string msg = 2; } message EstablishDisaggTaskRequest { - DisaggTaskMeta meta = 1; - string address = 2; // target address of this task. - // The write node needs to ensure that subsequent - // FetchDisaggPagesRequest can be processed within timeout_s. - // unit: seconds - int64 timeout_s = 3; - // The key ranges, Region meta that read node need to execute TableScan - repeated coprocessor.RegionInfo regions = 4; - int64 schema_ver = 5; - // Used for PartitionTableScan - repeated coprocessor.TableRegions table_regions = 6; - // The encoded TableScan/PartitionTableScan + Selection. - bytes encoded_plan = 7; + DisaggTaskMeta meta = 1; + string address = 2; // target address of this task. + // The write node needs to ensure that subsequent + // FetchDisaggPagesRequest can be processed within timeout_s. + // unit: seconds + int64 timeout_s = 3; + // The key ranges, Region meta that read node need to execute TableScan + repeated coprocessor.RegionInfo regions = 4; + int64 schema_ver = 5; + // Used for PartitionTableScan + repeated coprocessor.TableRegions table_regions = 6; + // The encoded TableScan/PartitionTableScan + Selection. + bytes encoded_plan = 7; } message EstablishDisaggTaskResponse { - EstablishDisaggTaskError error = 1; + EstablishDisaggTaskError error = 1; - // Write node maintains a snapshot with a lease time. - // Read node should read the delta pages - // (ColumnFileInMemory and ColumnFileTiny) - // along with this store_id and snapshot_id. - uint64 store_id = 3; // metapb.Store.id - DisaggTaskMeta snapshot_id = 4; - // Serialized disaggregated tasks (per physical table) - repeated bytes tables = 5; + // Write node maintains a snapshot with a lease time. + // Read node should read the delta pages + // (ColumnFileInMemory and ColumnFileTiny) + // along with this store_id and snapshot_id. + uint64 store_id = 3; // metapb.Store.id + DisaggTaskMeta snapshot_id = 4; + // Serialized disaggregated tasks (per physical table) + repeated bytes tables = 5; } message CancelDisaggTaskRequest { - DisaggTaskMeta meta = 1; + DisaggTaskMeta meta = 1; } message CancelDisaggTaskResponse {} message FetchDisaggPagesRequest { - // The snapshot id to fetch pages - DisaggTaskMeta snapshot_id = 1; - int64 table_id = 2; - uint64 segment_id = 3; - // It must be a subset of the delta pages ids returned - // in EstablishDisaggTaskResponse.segments - repeated uint64 page_ids = 4; + // The snapshot id to fetch pages + DisaggTaskMeta snapshot_id = 1; + int64 table_id = 2; + uint64 segment_id = 3; + // It must be a subset of the delta pages ids returned + // in EstablishDisaggTaskResponse.segments + repeated uint64 page_ids = 4; } message PagesPacket { - DisaggReadError error = 1; + DisaggReadError error = 1; - // Serialized column file data - // * ColumnFilePersisted alone with its schema, page data, field offsets - repeated bytes pages = 2; - // * ColumnFileInMemory alone with its serialized block - repeated bytes chunks = 3; + // Serialized column file data + // * ColumnFilePersisted alone with its schema, page data, field offsets + repeated bytes pages = 2; + // * ColumnFileInMemory alone with its serialized block + repeated bytes chunks = 3; - // Return tipb.SelectResponse.execution_summaries in the - // last packet - repeated bytes summaries = 4; + // Return tipb.SelectResponse.execution_summaries in the + // last packet + repeated bytes summaries = 4; } diff --git a/proto/disk_usage.proto b/proto/disk_usage.proto index 8670f172..2c49ff3e 100644 --- a/proto/disk_usage.proto +++ b/proto/disk_usage.proto @@ -1,14 +1,20 @@ syntax = "proto3"; package disk_usage; +import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; enum DiskUsage { - Normal = 0; - AlmostFull = 1; - AlreadyFull = 2; + Normal = 0; + AlmostFull = 1; + AlreadyFull = 2; } diff --git a/proto/encryptionpb.proto b/proto/encryptionpb.proto index 2328088c..7eb2978c 100644 --- a/proto/encryptionpb.proto +++ b/proto/encryptionpb.proto @@ -8,71 +8,73 @@ package encryptionpb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // General encryption metadata for any data type. message EncryptionMeta { - // ID of the key used to encrypt the data. - uint64 key_id = 1; - // Initialization vector (IV) of the data. - bytes iv = 2; + // ID of the key used to encrypt the data. + uint64 key_id = 1; + // Initialization vector (IV) of the data. + bytes iv = 2; } // Information about an encrypted file. message FileInfo { - // ID of the key used to encrypt the file. - uint64 key_id = 1; - // Initialization vector (IV) of the file. - bytes iv = 2; - // Method of encryption algorithm used to encrypted the file. - EncryptionMethod method = 3; + // ID of the key used to encrypt the file. + uint64 key_id = 1; + // Initialization vector (IV) of the file. + bytes iv = 2; + // Method of encryption algorithm used to encrypted the file. + EncryptionMethod method = 3; } message FileDictionary { - // A map of file name to file info. - map files = 1; + // A map of file name to file info. + map files = 1; } enum EncryptionMethod { - UNKNOWN = 0; - PLAINTEXT = 1; - AES128_CTR = 2; - AES192_CTR = 3; - AES256_CTR = 4; - SM4_CTR = 5; + UNKNOWN = 0; + PLAINTEXT = 1; + AES128_CTR = 2; + AES192_CTR = 3; + AES256_CTR = 4; + SM4_CTR = 5; } // The key used to encrypt the user data. message DataKey { - // A sequence of secret bytes used to encrypt data. - bytes key = 1; - // Method of encryption algorithm used to encrypted data. - EncryptionMethod method = 2; - // Creation time of the key. - uint64 creation_time = 3; - // A flag for the key have ever been exposed. - bool was_exposed = 4; + // A sequence of secret bytes used to encrypt data. + bytes key = 1; + // Method of encryption algorithm used to encrypted data. + EncryptionMethod method = 2; + // Creation time of the key. + uint64 creation_time = 3; + // A flag for the key have ever been exposed. + bool was_exposed = 4; } message KeyDictionary { - // A map of key ID to dat key. - map keys = 1; - // ID of a key currently in use. - uint64 current_key_id = 2; + // A map of key ID to dat key. + map keys = 1; + // ID of a key currently in use. + uint64 current_key_id = 2; } // Master key config. message MasterKey { - oneof backend { - MasterKeyPlaintext plaintext = 1; - MasterKeyFile file = 2; - MasterKeyKms kms = 3; - } + oneof backend { + MasterKeyPlaintext plaintext = 1; + MasterKeyFile file = 2; + MasterKeyKms kms = 3; + } } // MasterKeyPlaintext indicates content is stored as plaintext. @@ -81,35 +83,86 @@ message MasterKeyPlaintext {} // MasterKeyFile is a master key backed by a file containing encryption key in human-readable // hex format. message MasterKeyFile { - // Local file path. - string path = 1; + // Local file path. + string path = 1; } // MasterKeyKms is a master key backed by KMS service that manages the encryption key, // and provide API to encrypt and decrypt a data key, which is used to encrypt the content. message MasterKeyKms { - // KMS vendor. - string vendor = 1; - // KMS key id. - string key_id = 2; - // KMS region. - string region = 3; - // KMS endpoint. Normally not needed. - string endpoint = 4; + // KMS vendor. + string vendor = 1; + // KMS key id. + string key_id = 2; + // KMS region. + string region = 3; + // KMS endpoint. Normally not needed. + string endpoint = 4; + // optional, used to set up azure master key backend + AzureKms azure_kms = 5; + // optional, used to set up gcp master key backend + GcpKms gcp_kms = 6; + // optional, used to set up aws master key backend + AwsKms aws_kms = 7; +} + +message AzureKms { + string tenant_id = 1; + string client_id = 2; + string client_secret = 3; + // Key vault to encrypt/decrypt data key. + string key_vault_url = 4; + // optional hsm used to generate data key + string hsm_name = 5; + string hsm_url = 6; + string client_certificate = 7; + string client_certificate_path = 8; + string client_certificate_password = 9; +} + +message GcpKms { + string credential = 1; +} + +message AwsKms { + string access_key = 1; + string secret_access_key = 2; } message EncryptedContent { - // Metadata of the encrypted content. - // Eg. IV, method and KMS key ID - // It is preferred to define new fields for extra metadata than using this metadata map. - map metadata = 1; - // Encrypted content. - bytes content = 2; - // Master key used to encrypt the content. - MasterKey master_key = 3; - // Initilization vector (IV) used. - bytes iv = 4; - // Encrypted data key generated by KMS and used to actually encrypt data. - // Valid only when KMS is used. - bytes ciphertext_key = 5; + // Metadata of the encrypted content. + // Eg. IV, method and KMS key ID + // It is preferred to define new fields for extra metadata than using this metadata map. + map metadata = 1; + // Encrypted content. + bytes content = 2; + // Master key used to encrypt the content. + MasterKey master_key = 3; + // Initilization vector (IV) used. + bytes iv = 4; + // Encrypted data key generated by KMS and used to actually encrypt data. + // Valid only when KMS is used. + bytes ciphertext_key = 5; +} + +message FileEncryptionInfo { + oneof mode { + PlainTextDataKey plain_text_data_key = 1; + MasterKeyBased master_key_based = 2; + } + // file encryption method + encryptionpb.EncryptionMethod encryption_method = 3; + // iv to encrypt the file by data key + bytes file_iv = 4; + // file checksum after encryption, optional if using GCM + bytes checksum = 5; +} + +// not recommended in production. +// user needs to pass back the same data key for restore. +message PlainTextDataKey {} + +message MasterKeyBased { + // encrypted data key with metadata + repeated encryptionpb.EncryptedContent data_key_encrypted_content = 1; } diff --git a/proto/enginepb.proto b/proto/enginepb.proto index c0eb17c7..68358d4e 100644 --- a/proto/enginepb.proto +++ b/proto/enginepb.proto @@ -1,86 +1,96 @@ syntax = "proto3"; package enginepb; +import "gogoproto/gogo.proto"; import "metapb.proto"; import "raft_cmdpb.proto"; import "raft_serverpb.proto"; +import "rustproto.proto"; + +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message CommandRequestHeader { - uint64 region_id = 1; - uint64 index = 2; - uint64 term = 3; + uint64 region_id = 1; + uint64 index = 2; + uint64 term = 3; - // Flush in-memory data to disk. - bool sync_log = 4; - // Destroy the region. - bool destroy = 5; + // Flush in-memory data to disk. + bool sync_log = 4; + // Destroy the region. + bool destroy = 5; - // Additional information for the request. - bytes context = 6; + // Additional information for the request. + bytes context = 6; } message CommandRequest { - CommandRequestHeader header = 1; + CommandRequestHeader header = 1; - // We don't enclose normal requests and administrator request - // at same time. + // We don't enclose normal requests and administrator request + // at same time. - // kv put / delete - repeated raft_cmdpb.Request requests = 2; + // kv put / delete + repeated raft_cmdpb.Request requests = 2; - // region metadata manipulation command. - raft_cmdpb.AdminRequest admin_request = 3; - // region metadata manipulation result. - raft_cmdpb.AdminResponse admin_response = 4; + // region metadata manipulation command. + raft_cmdpb.AdminRequest admin_request = 3; + // region metadata manipulation result. + raft_cmdpb.AdminResponse admin_response = 4; } message CommandRequestBatch { - repeated CommandRequest requests = 1; + repeated CommandRequest requests = 1; } message CommandResponseHeader { - uint64 region_id = 1; - // Region is destroyed. - bool destroyed = 2; + uint64 region_id = 1; + // Region is destroyed. + bool destroyed = 2; } message CommandResponse { - CommandResponseHeader header = 1; + CommandResponseHeader header = 1; - raft_serverpb.RaftApplyState apply_state = 2; - uint64 applied_term = 3; + raft_serverpb.RaftApplyState apply_state = 2; + uint64 applied_term = 3; } message CommandResponseBatch { - repeated CommandResponse responses = 1; + repeated CommandResponse responses = 1; } message SnapshotState { - metapb.Region region = 1; - metapb.Peer peer = 2; - raft_serverpb.RaftApplyState apply_state = 3; + metapb.Region region = 1; + metapb.Peer peer = 2; + raft_serverpb.RaftApplyState apply_state = 3; } message SnapshotData { - string cf = 1; - uint32 checksum = 2; - repeated raft_serverpb.KeyValue data = 3; + string cf = 1; + uint32 checksum = 2; + repeated raft_serverpb.KeyValue data = 3; } message SnapshotRequest { - oneof chunk { - // The first message for snapshots. - // It contains the latest region information after applied snapshot. - SnapshotState state = 1; - - // Following messages are always data. - SnapshotData data = 2; - } + oneof chunk { + // The first message for snapshots. + // It contains the latest region information after applied snapshot. + SnapshotState state = 1; + + // Following messages are always data. + SnapshotData data = 2; + } } message SnapshotDone {} service Engine { - rpc ApplyCommandBatch(stream CommandRequestBatch) returns (stream CommandResponseBatch) {} - rpc ApplySnapshot(stream SnapshotRequest) returns (SnapshotDone) {} + rpc ApplyCommandBatch(stream CommandRequestBatch) returns (stream CommandResponseBatch) {} + rpc ApplySnapshot(stream SnapshotRequest) returns (SnapshotDone) {} } diff --git a/proto/errorpb.proto b/proto/errorpb.proto index 11628cdf..f05368e0 100644 --- a/proto/errorpb.proto +++ b/proto/errorpb.proto @@ -1,208 +1,217 @@ syntax = "proto3"; package errorpb; -import "metapb.proto"; import "gogoproto/gogo.proto"; +import "metapb.proto"; import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - -// NotLeader is the error variant that tells a request be handle by raft leader +// NotLeader is the error variant that tells a request be handle by raft leader // is sent to raft follower or learner. message NotLeader { - // The requested region ID - uint64 region_id = 1; - // Region leader of the requested region - metapb.Peer leader = 2; + // The requested region ID + uint64 region_id = 1; + // Region leader of the requested region + metapb.Peer leader = 2; } // IsWitness is the error variant that tells a request be handle by witness // which should be forbidden and retry. message IsWitness { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } // BucketVersionNotMatch is the error variant that tells the request buckets version is not match. // client should update the buckets version and retry. -message BucketVersionNotMatch{ - uint64 version = 1; - repeated bytes keys = 2; +message BucketVersionNotMatch { + uint64 version = 1; + repeated bytes keys = 2; } message DiskFull { - // The requested store ID - repeated uint64 store_id = 1; - // The detailed info - string reason = 2; + // The requested store ID + repeated uint64 store_id = 1; + // The detailed info + string reason = 2; } -// StoreNotMatch is the error variant that tells the request is sent to wrong store. +// StoreNotMatch is the error variant that tells the request is sent to wrong store. // (i.e. inconsistency of the store ID that request shows and the real store ID of this server.) message StoreNotMatch { - // Store id in request - uint64 request_store_id = 1; - // Actual store id - uint64 actual_store_id = 2; + // Store id in request + uint64 request_store_id = 1; + // Actual store id + uint64 actual_store_id = 2; } // RegionNotFound is the error variant that tells there isn't any region in this TiKV // matches the requested region ID. message RegionNotFound { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } // RegionNotInitialized is the error variant that tells there isn't any initialized peer // matchesthe request region ID. -message RegionNotInitialized { - // The request region ID - uint64 region_id = 1; +message RegionNotInitialized { + // The request region ID + uint64 region_id = 1; } // KeyNotInRegion is the error variant that tells the key the request requires isn't present in -// this region. +// this region. message KeyNotInRegion { - // The requested key - bytes key = 1; - // The requested region ID - uint64 region_id = 2; - // Start key of the requested region - bytes start_key = 3; - // Snd key of the requested region - bytes end_key = 4; + // The requested key + bytes key = 1; + // The requested region ID + uint64 region_id = 2; + // Start key of the requested region + bytes start_key = 3; + // Snd key of the requested region + bytes end_key = 4; } // EpochNotMatch is the error variant that tells a region has been updated. // (e.g. by splitting / merging, or raft Confchange.) // Hence, a command is based on a stale version of a region. message EpochNotMatch { - // Available regions that may be siblings of the requested one. - repeated metapb.Region current_regions = 1; + // Available regions that may be siblings of the requested one. + repeated metapb.Region current_regions = 1; } // ServerIsBusy is the error variant that tells the server is too busy to response. message ServerIsBusy { - string reason = 1; - // The suggested backoff time - uint64 backoff_ms = 2; - uint32 estimated_wait_ms = 3; - // Current applied_index at the leader, may be used in replica read. - uint64 applied_index = 4; + string reason = 1; + // The suggested backoff time + uint64 backoff_ms = 2; + uint32 estimated_wait_ms = 3; + // Current applied_index at the leader, may be used in replica read. + uint64 applied_index = 4; } // StaleCommand is the error variant that tells the command is stale, that is, // the current request term is lower than current raft term. // This can be retried at most time. -message StaleCommand { -} +message StaleCommand {} // RaftEntryTooLarge is the error variant that tells the request is too large to be serialized to a // reasonable small raft entry. // (i.e. greater than the configured value `raft_entry_max_size` in `raftstore`) message RaftEntryTooLarge { - // The requested region ID - uint64 region_id = 1; - // Size of the raft entry - uint64 entry_size = 2; + // The requested region ID + uint64 region_id = 1; + // Size of the raft entry + uint64 entry_size = 2; } // MaxTimestampNotSynced is the error variant that tells the peer has just become a leader and // updating the max timestamp in the concurrency manager from PD TSO is ongoing. In this case, // the prewrite of an async commit transaction cannot succeed. The client can backoff and // resend the request. -message MaxTimestampNotSynced { -} +message MaxTimestampNotSynced {} // ReadIndexNotReady is the error variant that tells the read index request is not ready, that is, // the current region is in a status that not ready to serve the read index request. For example, // region is in splitting or merging status. // This can be retried at most time. message ReadIndexNotReady { - // The reason why the region is not ready to serve read index request - string reason = 1; - // The requested region ID - uint64 region_id = 2; + // The reason why the region is not ready to serve read index request + string reason = 1; + // The requested region ID + uint64 region_id = 2; } // ProposalInMergingMode is the error variant that tells the proposal is rejected because raft is // in the merging mode. This may happen when BR/Lightning try to ingest SST. // This can be retried at most time. message ProposalInMergingMode { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } message DataIsNotReady { - // The requested region ID - uint64 region_id = 1; - uint64 peer_id = 2; - uint64 safe_ts = 3; + // The requested region ID + uint64 region_id = 1; + uint64 peer_id = 2; + uint64 safe_ts = 3; } message RecoveryInProgress { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } message FlashbackInProgress { - // The requested region ID - uint64 region_id = 1; - uint64 flashback_start_ts = 2; + // The requested region ID + uint64 region_id = 1; + uint64 flashback_start_ts = 2; } message FlashbackNotPrepared { - // The requested region ID - uint64 region_id = 1; + // The requested region ID + uint64 region_id = 1; } // MismatchPeerId is the error variant that tells the request is sent to wrong peer. // Client receives this error should reload the region info and retry. message MismatchPeerId { - uint64 request_peer_id = 1; - uint64 store_peer_id = 2; + uint64 request_peer_id = 1; + uint64 store_peer_id = 2; +} + +// UndeterminedResult is the error variant that tells the result is not determined yet. +// For example, the raft protocol timed out and the apply result is unknown. +message UndeterminedResult { + string message = 1; } // Error wraps all region errors, indicates an error encountered by a request. message Error { - reserved "stale_epoch"; - - // The error message - string message = 1; - NotLeader not_leader = 2; - RegionNotFound region_not_found = 3; - KeyNotInRegion key_not_in_region = 4; - EpochNotMatch epoch_not_match = 5; - ServerIsBusy server_is_busy = 6; - StaleCommand stale_command = 7; - StoreNotMatch store_not_match = 8; - RaftEntryTooLarge raft_entry_too_large = 9; - MaxTimestampNotSynced max_timestamp_not_synced = 10; - ReadIndexNotReady read_index_not_ready = 11; - ProposalInMergingMode proposal_in_merging_mode = 12; - DataIsNotReady data_is_not_ready = 13; - RegionNotInitialized region_not_initialized = 14; - DiskFull disk_full = 15; - // Online recovery is still in performing, reject writes to avoid potential issues - RecoveryInProgress RecoveryInProgress = 16; - // Flashback is still in performing, reject any read or write to avoid potential issues. - // NOTICE: this error is non-retryable, the request should fail ASAP when it meets this error. - FlashbackInProgress FlashbackInProgress = 17; - // If the second phase flashback request is sent to a region that is not prepared for the flashback, - // this error will be returned. - // NOTICE: this error is non-retryable, the client should retry the first phase flashback request when it meets this error. - FlashbackNotPrepared FlashbackNotPrepared = 18; - // IsWitness is the error variant that tells a request be handle by witness - // which should be forbidden and retry. - IsWitness is_witness = 19; - - MismatchPeerId mismatch_peer_id = 20; - - // BucketVersionNotMatch is the error variant that tells the request buckets version is not match. - BucketVersionNotMatch bucket_version_not_match = 21; + reserved "stale_epoch"; + + // The error message + string message = 1; + NotLeader not_leader = 2; + RegionNotFound region_not_found = 3; + KeyNotInRegion key_not_in_region = 4; + EpochNotMatch epoch_not_match = 5; + ServerIsBusy server_is_busy = 6; + StaleCommand stale_command = 7; + StoreNotMatch store_not_match = 8; + RaftEntryTooLarge raft_entry_too_large = 9; + MaxTimestampNotSynced max_timestamp_not_synced = 10; + ReadIndexNotReady read_index_not_ready = 11; + ProposalInMergingMode proposal_in_merging_mode = 12; + DataIsNotReady data_is_not_ready = 13; + RegionNotInitialized region_not_initialized = 14; + DiskFull disk_full = 15; + // Online recovery is still in performing, reject writes to avoid potential issues + RecoveryInProgress RecoveryInProgress = 16; + // Flashback is still in performing, reject any read or write to avoid potential issues. + // NOTICE: this error is non-retryable, the request should fail ASAP when it meets this error. + FlashbackInProgress FlashbackInProgress = 17; + // If the second phase flashback request is sent to a region that is not prepared for the flashback, + // this error will be returned. + // NOTICE: this error is non-retryable, the client should retry the first phase flashback request when it meets this error. + FlashbackNotPrepared FlashbackNotPrepared = 18; + // IsWitness is the error variant that tells a request be handle by witness + // which should be forbidden and retry. + IsWitness is_witness = 19; + + MismatchPeerId mismatch_peer_id = 20; + + // BucketVersionNotMatch is the error variant that tells the request buckets version is not match. + BucketVersionNotMatch bucket_version_not_match = 21; + + // UndeterminedResult is the error variant that tells the result is not determined yet. + UndeterminedResult undetermined_result = 22; } diff --git a/proto/gcpb.proto b/proto/gcpb.proto index 053872b8..8af33908 100644 --- a/proto/gcpb.proto +++ b/proto/gcpb.proto @@ -4,13 +4,15 @@ package gcpb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service GC { rpc ListKeySpaces(ListKeySpacesRequest) returns (ListKeySpacesResponse) {} diff --git a/proto/import_kvpb.proto b/proto/import_kvpb.proto index fc72adf1..6495867b 100644 --- a/proto/import_kvpb.proto +++ b/proto/import_kvpb.proto @@ -2,17 +2,19 @@ syntax = "proto3"; package import_kvpb; -import "import_sstpb.proto"; import "gogoproto/gogo.proto"; +import "import_sstpb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // ImportKV provides a service to import key-value pairs to TiKV. // // In order to import key-value pairs to TiKV, the user should: @@ -32,137 +34,130 @@ option java_package = "org.tikv.kvproto"; // in the engine. An engine can not be cleaned up when it is // writing or importing. service ImportKV { - // Switch the target cluster to normal/import mode. - rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} - // Open an engine. - rpc OpenEngine(OpenEngineRequest) returns (OpenEngineResponse) {} - // Open a write stream to the engine. - rpc WriteEngine(stream WriteEngineRequest) returns (WriteEngineResponse) {} - // Write to engine, single message version - rpc WriteEngineV3(WriteEngineV3Request) returns (WriteEngineResponse) {} - // Close the engine. - rpc CloseEngine(CloseEngineRequest) returns (CloseEngineResponse) {} - // Import the engine to the target cluster. - rpc ImportEngine(ImportEngineRequest) returns (ImportEngineResponse) {} - // Clean up the engine. - rpc CleanupEngine(CleanupEngineRequest) returns (CleanupEngineResponse) {} - // Compact the target cluster for better performance. - rpc CompactCluster(CompactClusterRequest) returns (CompactClusterResponse) {} - // Get current version and commit hash - rpc GetVersion(GetVersionRequest) returns (GetVersionResponse) {} - // Get importer metrics - rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} + // Switch the target cluster to normal/import mode. + rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} + // Open an engine. + rpc OpenEngine(OpenEngineRequest) returns (OpenEngineResponse) {} + // Open a write stream to the engine. + rpc WriteEngine(stream WriteEngineRequest) returns (WriteEngineResponse) {} + // Write to engine, single message version + rpc WriteEngineV3(WriteEngineV3Request) returns (WriteEngineResponse) {} + // Close the engine. + rpc CloseEngine(CloseEngineRequest) returns (CloseEngineResponse) {} + // Import the engine to the target cluster. + rpc ImportEngine(ImportEngineRequest) returns (ImportEngineResponse) {} + // Clean up the engine. + rpc CleanupEngine(CleanupEngineRequest) returns (CleanupEngineResponse) {} + // Compact the target cluster for better performance. + rpc CompactCluster(CompactClusterRequest) returns (CompactClusterResponse) {} + // Get current version and commit hash + rpc GetVersion(GetVersionRequest) returns (GetVersionResponse) {} + // Get importer metrics + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} } message SwitchModeRequest { - string pd_addr = 1; - import_sstpb.SwitchModeRequest request = 2; + string pd_addr = 1; + import_sstpb.SwitchModeRequest request = 2; } -message SwitchModeResponse { -} +message SwitchModeResponse {} message OpenEngineRequest { - bytes uuid = 1; - bytes key_prefix = 2; + bytes uuid = 1; + bytes key_prefix = 2; } -message OpenEngineResponse { -} +message OpenEngineResponse {} message WriteHead { - bytes uuid = 1; + bytes uuid = 1; } message Mutation { - enum OP { - Put = 0; - } - OP op = 1; - bytes key = 2; - bytes value = 3; + enum OP { + Put = 0; + } + OP op = 1; + bytes key = 2; + bytes value = 3; } message WriteBatch { - uint64 commit_ts = 1; - repeated Mutation mutations = 2; + uint64 commit_ts = 1; + repeated Mutation mutations = 2; } message WriteEngineRequest { - oneof chunk { - WriteHead head = 1; - WriteBatch batch = 2; - } + oneof chunk { + WriteHead head = 1; + WriteBatch batch = 2; + } } message KVPair { - bytes key = 1; - bytes value = 2; + bytes key = 1; + bytes value = 2; } message WriteEngineV3Request { - bytes uuid = 1; - uint64 commit_ts = 2; - repeated KVPair pairs = 3; + bytes uuid = 1; + uint64 commit_ts = 2; + repeated KVPair pairs = 3; } message WriteEngineResponse { - Error error = 1; + Error error = 1; } message CloseEngineRequest { - bytes uuid = 1; + bytes uuid = 1; } message CloseEngineResponse { - Error error = 1; + Error error = 1; } message ImportEngineRequest { - bytes uuid = 1; - string pd_addr = 2; + bytes uuid = 1; + string pd_addr = 2; } -message ImportEngineResponse { -} +message ImportEngineResponse {} message CleanupEngineRequest { - bytes uuid = 1; + bytes uuid = 1; } -message CleanupEngineResponse { -} +message CleanupEngineResponse {} message CompactClusterRequest { - string pd_addr = 1; - import_sstpb.CompactRequest request = 2; + string pd_addr = 1; + import_sstpb.CompactRequest request = 2; } -message CompactClusterResponse { -} +message CompactClusterResponse {} -message GetVersionRequest { -} +message GetVersionRequest {} message GetVersionResponse { - string version = 1; - string commit = 2; + string version = 1; + string commit = 2; } -message GetMetricsRequest { -} +message GetMetricsRequest {} message GetMetricsResponse { - string prometheus = 1; + string prometheus = 1; } message Error { - message EngineNotFound { - bytes uuid = 1; - } - // This can happen if the client hasn't opened the engine, or the server - // restarts while the client is writing or closing. An unclosed engine will - // be removed on server restart, so the client should not continue but - // restart the previous job in that case. - EngineNotFound engine_not_found = 1; + message EngineNotFound { + bytes uuid = 1; + } + // This can happen if the client hasn't opened the engine, or the server + // restarts while the client is writing or closing. An unclosed engine will + // be removed on server restart, so the client should not continue but + // restart the previous job in that case. + EngineNotFound engine_not_found = 1; } diff --git a/proto/import_sstpb.proto b/proto/import_sstpb.proto index 089dd35f..b230d31b 100644 --- a/proto/import_sstpb.proto +++ b/proto/import_sstpb.proto @@ -2,20 +2,23 @@ syntax = "proto3"; package import_sstpb; -import "metapb.proto"; +import "brpb.proto"; +import "encryptionpb.proto"; import "errorpb.proto"; -import "kvrpcpb.proto"; import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "rustproto.proto"; -import "brpb.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // ImportSST provides a service to import a generated SST file to a region in TiKV. // // In order to import an SST file to a region, the user should: @@ -32,417 +35,465 @@ option java_package = "org.tikv.kvproto"; // file's metadata, to guarantee that the region's epoch must be the same // between the SST file is uploaded and ingested later. service ImportSST { - // Switch to normal/import mode. - rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} - // Get import mode(normal/import). - rpc GetMode(GetModeRequest) returns (GetModeResponse) {} - // Upload an SST file to a server. - rpc Upload(stream UploadRequest) returns (UploadResponse) {} - // Ingest an uploaded SST file to a region. - rpc Ingest(IngestRequest) returns (IngestResponse) {} - // Compact the specific range for better performance. - rpc Compact(CompactRequest) returns (CompactResponse) {} - - rpc SetDownloadSpeedLimit(SetDownloadSpeedLimitRequest) returns (SetDownloadSpeedLimitResponse) {} - // Download an SST file from an external storage, and performs key-rewrite - // after downloading. - rpc Download(DownloadRequest) returns (DownloadResponse) {} - - // Open a write stream to generate sst files - rpc Write(stream WriteRequest) returns (WriteResponse) {} - rpc RawWrite(stream RawWriteRequest) returns (RawWriteResponse) {} - - // Ingest Multiple files in one request - rpc MultiIngest(MultiIngestRequest) returns (IngestResponse) {} - - // Collect duplicate data from TiKV. - rpc DuplicateDetect(DuplicateDetectRequest) returns (stream DuplicateDetectResponse) {} - - // Apply download & apply increment kv files to TiKV. - rpc Apply(ApplyRequest) returns (ApplyResponse) {} - - // ClearFiles clear applied file after restore succeed. - rpc ClearFiles(ClearRequest) returns (ClearResponse) {} - - // Suspend ingest for data listeners don't support catching import data. - rpc SuspendImportRPC(SuspendImportRPCRequest) returns (SuspendImportRPCResponse) {} - + // Switch to normal/import mode. + rpc SwitchMode(SwitchModeRequest) returns (SwitchModeResponse) {} + // Get import mode(normal/import). + rpc GetMode(GetModeRequest) returns (GetModeResponse) {} + // Upload an SST file to a server. + rpc Upload(stream UploadRequest) returns (UploadResponse) {} + // Ingest an uploaded SST file to a region. + rpc Ingest(IngestRequest) returns (IngestResponse) {} + // Compact the specific range for better performance. + rpc Compact(CompactRequest) returns (CompactResponse) {} + + rpc SetDownloadSpeedLimit(SetDownloadSpeedLimitRequest) returns (SetDownloadSpeedLimitResponse) {} + // Download an SST file from an external storage, and performs key-rewrite + // after downloading. + rpc Download(DownloadRequest) returns (DownloadResponse) {} + // Download SST files in batch from external storage, perform key rewrite, + // and merge them into one SST after downloading. + rpc BatchDownload(DownloadRequest) returns (DownloadResponse) {} + // Download SST files in batch from external storage, perform key rewrite, + // merge them into one SST, and keep only the latest MVCC version after downloading. + rpc BatchDownloadLatestMVCC(DownloadRequest) returns (DownloadResponse) {} + + // Open a write stream to generate sst files + rpc Write(stream WriteRequest) returns (WriteResponse) {} + rpc RawWrite(stream RawWriteRequest) returns (RawWriteResponse) {} + + // Ingest Multiple files in one request + rpc MultiIngest(MultiIngestRequest) returns (IngestResponse) {} + + // Collect duplicate data from TiKV. + rpc DuplicateDetect(DuplicateDetectRequest) returns (stream DuplicateDetectResponse) {} + + // Apply download & apply increment kv files to TiKV. + rpc Apply(ApplyRequest) returns (ApplyResponse) {} + + // ClearFiles clear applied file after restore succeed. + rpc ClearFiles(ClearRequest) returns (ClearResponse) {} + + // Suspend ingest for data listeners don't support catching import data. + rpc SuspendImportRPC(SuspendImportRPCRequest) returns (SuspendImportRPCResponse) {} + + // AddForcePartitionRange marks a range in tikv that any compact overlaps with this range + // should generates SST files partitioned at region boundaries as well as this range boundary. + // TiKV will also try to do manual compact(if needed) after setting this range to eusure + // any incoming SST under this range can be ingested into the bottom level if there is no real kv overlap. + rpc AddForcePartitionRange(AddPartitionRangeRequest) returns (AddPartitionRangeResponse) {} + + // Remove the force partition range after the task is finished. If this function is not called, + // tikv will cleanup the range after TTL to ensure it can be cleaned eventually. + rpc RemoveForcePartitionRange(RemovePartitionRangeRequest) returns (RemovePartitionRangeResponse) {} } message SuspendImportRPCRequest { - // whether to suspend new imports. - bool should_suspend_imports = 1; + // whether to suspend new imports. + bool should_suspend_imports = 1; - // the duration of import service suspension - // when should_deny_imports is false, - // this won't take effect. - uint64 duration_in_secs = 2; - // The identifier for the caller. - string caller = 3; + // the duration of import service suspension + // when should_deny_imports is false, + // this won't take effect. + uint64 duration_in_secs = 2; + // The identifier for the caller. + string caller = 3; } message SuspendImportRPCResponse { - // The last state before this RPC. - bool already_suspended = 1; + // The last state before this RPC. + bool already_suspended = 1; } enum SwitchMode { - Normal = 0; - Import = 1; + Normal = 0; + Import = 1; } message SwitchModeRequest { - SwitchMode mode = 1; - repeated Range ranges = 2; + SwitchMode mode = 1; + repeated Range ranges = 2; } -message SwitchModeResponse { -} +message SwitchModeResponse {} -message GetModeRequest { -} +message GetModeRequest {} message GetModeResponse { - SwitchMode mode = 1; + SwitchMode mode = 1; } message Range { - bytes start = 1; - bytes end = 2; + bytes start = 1; + bytes end = 2; } message SSTMeta { - bytes uuid = 1; - Range range = 2; - uint32 crc32 = 3; - uint64 length = 4; - string cf_name = 5; - uint64 region_id = 6; - metapb.RegionEpoch region_epoch = 7; - bool end_key_exclusive = 8; - // total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size - // set these values can save time from tikv upload keys and size to PD through Heartbeat. - uint64 total_kvs = 9; - uint64 total_bytes = 10; - // API version implies the encode of the key and value. - kvrpcpb.APIVersion api_version = 11; - // cipher_iv is used to encrypt/decrypt sst - bytes cipher_iv = 12; + bytes uuid = 1; + Range range = 2; + uint32 crc32 = 3; + uint64 length = 4; + string cf_name = 5; + uint64 region_id = 6; + metapb.RegionEpoch region_epoch = 7; + bool end_key_exclusive = 8; + // total_kvs and total_bytes is equivalent to PD's approximate_keys and approximate_size + // set these values can save time from tikv upload keys and size to PD through Heartbeat. + uint64 total_kvs = 9; + uint64 total_bytes = 10; + // API version implies the encode of the key and value. + kvrpcpb.APIVersion api_version = 11; + // cipher_iv is used to encrypt/decrypt sst + bytes cipher_iv = 12; + + // Reserved for a downstream fork + reserved 1000 to 1199; } // A rewrite rule is applied on the *encoded* keys (the internal storage // representation). message RewriteRule { - bytes old_key_prefix = 1; - bytes new_key_prefix = 2; - uint64 new_timestamp = 3; + bytes old_key_prefix = 1; + bytes new_key_prefix = 2; + // (Optional) Rewrite all keys in the range to use this timestamp. + uint64 new_timestamp = 3; + // (Optional) Skip keys with timestamps greater than this during download, useful for compacted SST backups. + uint64 ignore_after_timestamp = 4; + // (Optional) Skip write CF keys with timestamps less than this during download. Default CF keys are preserved. + uint64 ignore_before_timestamp = 5; } message UploadRequest { - oneof chunk { - SSTMeta meta = 1; - bytes data = 2; - } + oneof chunk { + SSTMeta meta = 1; + bytes data = 2; + } } -message UploadResponse { -} +message UploadResponse {} message IngestRequest { - kvrpcpb.Context context = 1; - SSTMeta sst = 2; + kvrpcpb.Context context = 1; + SSTMeta sst = 2; } message MultiIngestRequest { - kvrpcpb.Context context = 1; - repeated SSTMeta ssts = 2; + kvrpcpb.Context context = 1; + repeated SSTMeta ssts = 2; } message IngestResponse { - errorpb.Error error = 1; + errorpb.Error error = 1; } message CompactRequest { - // Compact files in the range and above the output level. - // Compact all files if the range is not specified. - // Compact all files to the bottommost level if the output level is -1. - Range range = 1; - int32 output_level = 2; - kvrpcpb.Context context = 3; + // Compact files in the range and above the output level. + // Compact all files if the range is not specified. + // Compact all files to the bottommost level if the output level is -1. + Range range = 1; + int32 output_level = 2; + kvrpcpb.Context context = 3; } -message CompactResponse { -} +message CompactResponse {} message DownloadRequest { - // Map represents the map of . - // We'll generate all SSTMeta into one SST File. - map ssts = 1; - // resolved_ts is used to merge related SST Files. - uint64 resolved_ts = 3; - - // The SST meta used to identify the downloaded file. - // Must be the same among all nodes in the same Raft group. - // Note: the "crc32" and "cf_name" fields are ignored in this request, - // and the "range" field represents the closed key range after rewrite - // (as origin keys in encoded representation). - SSTMeta sst = 2 [(gogoproto.nullable) = false]; - - // The url field is deprecated, use storage_backend instead - reserved 8; reserved "url"; - - // The file name of the SST file. - string name = 9; - - // Performs a key prefix rewrite after downloading the SST file. - // All keys in the SST will be rewritten as: - // - // new_key = new_key_prefix + old_key[len(old_key_prefix)..] - // - // When used for TiDB, rewriting the prefix changes the table ID. Please - // note that key-rewrite is applied on the origin keys in encoded - // representation (the SST itself should still use data keys in encoded - // representation). - // - // You need to ensure that the keys before and after rewriting are in the - // same order, otherwise the RPC request will fail. - RewriteRule rewrite_rule = 13 [(gogoproto.nullable) = false]; - - backup.StorageBackend storage_backend = 14; - // The identity for the stroage backend. - // When this field presents, the storage would be cached. - // If there is a cached storage, TiKV would use it driectly. - string storage_cache_id = 17; - - bool is_raw_kv = 15; - // cipher_info is used to decrypt sst when download sst - backup.CipherInfo cipher_info = 16; - - // The type of the download request. - DownloadRequestType request_type = 18; - kvrpcpb.Context context = 19; + // Map represents the map of . + // We'll generate all SSTMeta into one SST File. + map ssts = 1; + // resolved_ts is used to merge related SST Files. + uint64 resolved_ts = 3; + + // The SST meta used to identify the downloaded file. + // Must be the same among all nodes in the same Raft group. + // Note: the "crc32" and "cf_name" fields are ignored in this request, + // and the "range" field represents the closed key range after rewrite + // (as origin keys in encoded representation). + SSTMeta sst = 2 [(gogoproto.nullable) = false]; + + // The url field is deprecated, use storage_backend instead + reserved 8; + reserved "url"; + + // The file name of the SST file. + string name = 9; + + // Performs a key prefix rewrite after downloading the SST file. + // All keys in the SST will be rewritten as: + // + // new_key = new_key_prefix + old_key[len(old_key_prefix)..] + // + // When used for TiDB, rewriting the prefix changes the table ID. Please + // note that key-rewrite is applied on the origin keys in encoded + // representation (the SST itself should still use data keys in encoded + // representation). + // + // You need to ensure that the keys before and after rewriting are in the + // same order, otherwise the RPC request will fail. + RewriteRule rewrite_rule = 13 [(gogoproto.nullable) = false]; + repeated RewriteRule sorted_rewrite_rules = 20; + + backup.StorageBackend storage_backend = 14; + // The identity for the stroage backend. + // When this field presents, the storage would be cached. + // If there is a cached storage, TiKV would use it driectly. + string storage_cache_id = 17; + + bool is_raw_kv = 15; + // cipher_info is used to decrypt sst when download sst + backup.CipherInfo cipher_info = 16; + + // The type of the download request. + DownloadRequestType request_type = 18; + kvrpcpb.Context context = 19; } enum DownloadRequestType { - // For the compatibility with old version of TiDBs - Legacy = 0; - // For the TiDBs with newer versions that support keyspace feature. - Keyspace = 1; + // For the compatibility with old version of TiDBs + Legacy = 0; + // For the TiDBs with newer versions that support keyspace feature. + Keyspace = 1; } // For now it is just used for distinguishing the error of the request with the error // of gRPC, add more concrete types if it is necessary later. message Error { - string message = 1; - - // We meet some internal errors of the store. - errorpb.Error store_error = 2; + string message = 1; + + // We meet some internal errors of the store. + errorpb.Error store_error = 2; } message DownloadResponse { - // The actual key range (after rewrite) of the downloaded SST. The range is - // inclusive in both ends. - Range range = 1 [(gogoproto.nullable) = false]; + // The actual key range (after rewrite) of the downloaded SST. The range is + // inclusive in both ends. + Range range = 1 [(gogoproto.nullable) = false]; - // Whether the SST is empty. An empty SST is prohibited in TiKV, do not - // ingest if this field is true. - // (Deprecated, should be replaced by checking `length == 0` in the future) - bool is_empty = 2; + // Whether the SST is empty. An empty SST is prohibited in TiKV, do not + // ingest if this field is true. + // (Deprecated, should be replaced by checking `length == 0` in the future) + bool is_empty = 2; - Error error = 3; + Error error = 3; - // The CRC32 checksum of the rewritten SST file (implementation can return - // zero, indicating the CRC32 was not calculated). - uint32 crc32 = 4; - // The actual length of the rewritten SST file. - uint64 length = 5; + // The CRC32 checksum of the rewritten SST file (implementation can return + // zero, indicating the CRC32 was not calculated). + uint32 crc32 = 4; + // The actual length of the rewritten SST file. + uint64 length = 5; - // This field only return when file-copy backup enabled. - // Because it will merge many SST files in a download request. - repeated SSTMeta ssts = 6; + // This field only return when file-copy backup enabled. + // Because it will merge many SST files in a download request. + repeated SSTMeta ssts = 6; } message SetDownloadSpeedLimitRequest { - // The download speed limit (bytes/second). Set to 0 for unlimited speed. - uint64 speed_limit = 1; -} + // The download speed limit (bytes/second). Set to 0 for unlimited speed. + uint64 speed_limit = 1; + + // The download speed limit task id. + string task_id = 2; -message SetDownloadSpeedLimitResponse { + // The download speed limit ttl. Set to 0 means it will never expire. + uint64 ttl_seconds = 3; } +message SetDownloadSpeedLimitResponse {} + message Pair { - bytes key = 1; - bytes value = 2; - enum OP { - Put = 0; - Delete = 1; - } - OP op = 3; + bytes key = 1; + bytes value = 2; + enum OP { + Put = 0; + Delete = 1; + } + OP op = 3; } message WriteBatch { - uint64 commit_ts = 1; - repeated Pair pairs = 2; + uint64 commit_ts = 1; + repeated Pair pairs = 2; } message WriteRequest { - oneof chunk { - SSTMeta meta = 1; - WriteBatch batch = 2; - } - kvrpcpb.Context context = 3; + oneof chunk { + SSTMeta meta = 1; + WriteBatch batch = 2; + } + kvrpcpb.Context context = 3; } message WriteResponse { - Error error = 1; - repeated SSTMeta metas = 2; + Error error = 1; + repeated SSTMeta metas = 2; } message RawWriteBatch { - uint64 ttl = 1; - repeated Pair pairs = 2; + uint64 ttl = 1; + repeated Pair pairs = 2; - // To be compatible with the key encoding of API V2. - // This field should be generated from the client instead of the server, - // since the message will be send to all the replicas of a region. - // Otherwise, the underlying data generated by the server would be inconsistent which is hard to scale - // for other features like MVCC over RawKV. - uint64 ts = 3; + // To be compatible with the key encoding of API V2. + // This field should be generated from the client instead of the server, + // since the message will be send to all the replicas of a region. + // Otherwise, the underlying data generated by the server would be inconsistent which is hard to scale + // for other features like MVCC over RawKV. + uint64 ts = 3; } message RawWriteRequest { - oneof chunk { - SSTMeta meta = 1; - RawWriteBatch batch = 2; - } - kvrpcpb.Context context = 3; + oneof chunk { + SSTMeta meta = 1; + RawWriteBatch batch = 2; + } + kvrpcpb.Context context = 3; } message RawWriteResponse { - Error error = 1; - repeated SSTMeta metas = 2; + Error error = 1; + repeated SSTMeta metas = 2; } message DuplicateDetectRequest { - kvrpcpb.Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - // Return only the keys found by scanning, not their values. - bool key_only = 4; - // We only check the data whose timestamp is larger than `min_commit_ts`. `min_commit_ts` is exclueded. - uint64 min_commit_ts = 5; + kvrpcpb.Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + // Return only the keys found by scanning, not their values. + bool key_only = 4; + // We only check the data whose timestamp is larger than `min_commit_ts`. `min_commit_ts` is exclueded. + uint64 min_commit_ts = 5; } message KvPair { - bytes key = 1; - bytes value = 2; - uint64 commit_ts = 3; + bytes key = 1; + bytes value = 2; + uint64 commit_ts = 3; } message DuplicateDetectResponse { - errorpb.Error region_error = 1; - Error key_error = 2; - // The these keys will be in asc order (but commit time is in desc order), - // and the content is just like following: - // [ - // {key: "key1", value: "value11", commit_ts: 1005}, - // {key: "key1", value: "value12", commit_ts: 1004}, - // {key: "key1", value: "value13", commit_ts: 1001}, - // {key: "key2", value: "value21", commit_ts: 1004}, - // {key: "key2", value: "value22", commit_ts: 1002}, - // ... - // ] - repeated KvPair pairs = 3; + errorpb.Error region_error = 1; + Error key_error = 2; + // The these keys will be in asc order (but commit time is in desc order), + // and the content is just like following: + // [ + // {key: "key1", value: "value11", commit_ts: 1005}, + // {key: "key1", value: "value12", commit_ts: 1004}, + // {key: "key1", value: "value13", commit_ts: 1001}, + // {key: "key2", value: "value21", commit_ts: 1004}, + // {key: "key2", value: "value22", commit_ts: 1002}, + // ... + // ] + repeated KvPair pairs = 3; } message KVMeta { - // The file name of the KV file. - string name = 1; + // The file name of the KV file. + string name = 1; - // file offset, sometimes only need to get a part of data from the merged file - uint64 range_offset = 11; + // file offset, sometimes only need to get a part of data from the merged file + uint64 range_offset = 11; - // file length for check. - uint64 length = 2; + // file length for check. + uint64 length = 2; - // range length of the merged file, if it exists. - uint64 range_length = 12; + // range length of the merged file, if it exists. + uint64 range_length = 12; - // tell us which cf should apply. WRITE_CF or DEFAULT_CF e.g. - string cf = 3; + // tell us which cf should apply. WRITE_CF or DEFAULT_CF e.g. + string cf = 3; - // is_delete represents whether we should delete the kv in tikv. - // it may not be too much delete file. only rollBack operation will generate delete kv file. - bool is_delete = 4; + // is_delete represents whether we should delete the kv in tikv. + // it may not be too much delete file. only rollBack operation will generate delete kv file. + bool is_delete = 4; - // the key ts space being smaller than start_ts can be filter. - uint64 start_ts = 10; + // the key ts space being smaller than start_ts can be filter. + uint64 start_ts = 10; - // the key ts space large than restore_ts can be filter. - uint64 restore_ts = 5; + // the key ts space large than restore_ts can be filter. + uint64 restore_ts = 5; - bytes start_key = 6; + bytes start_key = 6; - bytes end_key = 7; + bytes end_key = 7; - // used for checksum when download kv file. - bytes sha256 = 8; + // used for checksum when download kv file. + bytes sha256 = 8; - // the key ts space less than start_snapshot_ts can be filter. - // Deprecated: this field 'start_snapshot_ts' is replaced by the field 'start_ts'. - uint64 start_snapshot_ts = 9; + // the key ts space less than start_snapshot_ts can be filter. + // Deprecated: this field 'start_snapshot_ts' is replaced by the field 'start_ts'. + uint64 start_snapshot_ts = 9; - // the compression type for the file. - backup.CompressionType compression_type = 13; -} + // the compression type for the file. + backup.CompressionType compression_type = 13; + // encryption information of the kv file, not set if encryption is not enabled. + encryptionpb.FileEncryptionInfo file_encryption_info = 14; +} message ApplyRequest { - // The meta of the KV file. - KVMeta meta = 1; - repeated KVMeta metas = 12; - - // Performs a key prefix rewrite after downloading the file. - // All keys in the files will be rewritten as: - // - // new_key = new_key_prefix + old_key[len(old_key_prefix)..] - // - // When used for TiDB, rewriting the prefix changes the table ID. Please - // note that key-rewrite is applied on the origin keys in encoded - // representation. - // - // You need to ensure that the keys before and after rewriting are in the - // same order, otherwise the RPC request will fail. - RewriteRule rewrite_rule = 2 [(gogoproto.nullable) = false]; - repeated RewriteRule rewrite_rules = 13; - - // The identity for the stroage backend. - // When this field presents, the storage would be cached. - // If there is a cached storage, TiKV would use it driectly. - string storage_cache_id = 5; - backup.StorageBackend storage_backend = 3; - - // context represents region info and it used to build raft commands. - kvrpcpb.Context context = 4; - - // cipher_info is used to decrypt kv file when download file. - backup.CipherInfo cipher_info = 11; + // The meta of the KV file. + KVMeta meta = 1; + repeated KVMeta metas = 12; + + // Performs a key prefix rewrite after downloading the file. + // All keys in the files will be rewritten as: + // + // new_key = new_key_prefix + old_key[len(old_key_prefix)..] + // + // When used for TiDB, rewriting the prefix changes the table ID. Please + // note that key-rewrite is applied on the origin keys in encoded + // representation. + // + // You need to ensure that the keys before and after rewriting are in the + // same order, otherwise the RPC request will fail. + RewriteRule rewrite_rule = 2 [(gogoproto.nullable) = false]; + repeated RewriteRule rewrite_rules = 13; + + // The identity for the stroage backend. + // When this field presents, the storage would be cached. + // If there is a cached storage, TiKV would use it driectly. + string storage_cache_id = 5; + backup.StorageBackend storage_backend = 3; + + // context represents region info and it used to build raft commands. + kvrpcpb.Context context = 4; + + // plaintext data key to decrypt kv file if configured during log backup. + backup.CipherInfo cipher_info = 11; + + // master keys config used to decrypt data keys in restore if configured during log backup. + repeated encryptionpb.MasterKey master_keys = 14; } message ApplyResponse { - // The actual key range (after rewrite) of the downloaded file. The range is - // inclusive in both ends. - Range range = 1 [(gogoproto.nullable) = false]; + // The actual key range (after rewrite) of the downloaded file. The range is + // inclusive in both ends. + Range range = 1 [(gogoproto.nullable) = false]; - Error error = 2; + Error error = 2; } message ClearRequest { - // clear files in import directory with given prefix. - string prefix = 1; + // clear files in import directory with given prefix. + string prefix = 1; } message ClearResponse { - Error error = 1; + Error error = 1; +} + +message AddPartitionRangeRequest { + Range range = 1; + // the number of seconds this range is valid. + //After this duration, if this range is still not removed by + // `RemoveForcePartitionRange`, tikv will automically remove it. So the client + // should set a big enough value to avoid auto cleanup. But in general, 1h + // should be a big enough value. If its value is 0, tikv will auto adjust it to 3600(1h). + uint64 ttl_seconds = 2; +} + +message AddPartitionRangeResponse {} + +message RemovePartitionRangeRequest { + Range range = 1; } +message RemovePartitionRangeResponse {} diff --git a/proto/include/eraftpb.proto b/proto/include/eraftpb.proto index d9ec4b12..b952d81f 100644 --- a/proto/include/eraftpb.proto +++ b/proto/include/eraftpb.proto @@ -2,9 +2,9 @@ syntax = "proto3"; package eraftpb; enum EntryType { - EntryNormal = 0; - EntryConfChange = 1; - EntryConfChangeV2 = 2; + EntryNormal = 0; + EntryConfChange = 1; + EntryConfChangeV2 = 2; } // The entry is a type of change that needs to be applied. It contains two data fields. @@ -18,134 +18,134 @@ enum EntryType { // context will provide anything needed to assist the configuration change. The context // if for the user to set and use in this case. message Entry { - EntryType entry_type = 1; - uint64 term = 2; - uint64 index = 3; - bytes data = 4; - bytes context = 6; - - // Deprecated! It is kept for backward compatibility. - // TODO: remove it in the next major release. - bool sync_log = 5; + EntryType entry_type = 1; + uint64 term = 2; + uint64 index = 3; + bytes data = 4; + bytes context = 6; + + // Deprecated! It is kept for backward compatibility. + // TODO: remove it in the next major release. + bool sync_log = 5; } message SnapshotMetadata { - // The current `ConfState`. - ConfState conf_state = 1; - // The applied index. - uint64 index = 2; - // The term of the applied index. - uint64 term = 3; + // The current `ConfState`. + ConfState conf_state = 1; + // The applied index. + uint64 index = 2; + // The term of the applied index. + uint64 term = 3; } message Snapshot { - bytes data = 1; - SnapshotMetadata metadata = 2; + bytes data = 1; + SnapshotMetadata metadata = 2; } enum MessageType { - MsgHup = 0; - MsgBeat = 1; - MsgPropose = 2; - MsgAppend = 3; - MsgAppendResponse = 4; - MsgRequestVote = 5; - MsgRequestVoteResponse = 6; - MsgSnapshot = 7; - MsgHeartbeat = 8; - MsgHeartbeatResponse = 9; - MsgUnreachable = 10; - MsgSnapStatus = 11; - MsgCheckQuorum = 12; - MsgTransferLeader = 13; - MsgTimeoutNow = 14; - MsgReadIndex = 15; - MsgReadIndexResp = 16; - MsgRequestPreVote = 17; - MsgRequestPreVoteResponse = 18; + MsgHup = 0; + MsgBeat = 1; + MsgPropose = 2; + MsgAppend = 3; + MsgAppendResponse = 4; + MsgRequestVote = 5; + MsgRequestVoteResponse = 6; + MsgSnapshot = 7; + MsgHeartbeat = 8; + MsgHeartbeatResponse = 9; + MsgUnreachable = 10; + MsgSnapStatus = 11; + MsgCheckQuorum = 12; + MsgTransferLeader = 13; + MsgTimeoutNow = 14; + MsgReadIndex = 15; + MsgReadIndexResp = 16; + MsgRequestPreVote = 17; + MsgRequestPreVoteResponse = 18; } message Message { - MessageType msg_type = 1; - uint64 to = 2; - uint64 from = 3; - uint64 term = 4; - uint64 log_term = 5; - uint64 index = 6; - repeated Entry entries = 7; - uint64 commit = 8; - Snapshot snapshot = 9; - uint64 request_snapshot = 13; - bool reject = 10; - uint64 reject_hint = 11; - bytes context = 12; - uint64 deprecated_priority = 14; - // If this new field is not set, then use the above old field; otherwise - // use the new field. When broadcasting request vote, both fields are - // set if the priority is larger than 0. This change is not a fully - // compatible change, but it makes minimal impact that only new priority - // is not recognized by the old nodes during rolling update. - int64 priority = 15; + MessageType msg_type = 1; + uint64 to = 2; + uint64 from = 3; + uint64 term = 4; + uint64 log_term = 5; + uint64 index = 6; + repeated Entry entries = 7; + uint64 commit = 8; + Snapshot snapshot = 9; + uint64 request_snapshot = 13; + bool reject = 10; + uint64 reject_hint = 11; + bytes context = 12; + uint64 deprecated_priority = 14; + // If this new field is not set, then use the above old field; otherwise + // use the new field. When broadcasting request vote, both fields are + // set if the priority is larger than 0. This change is not a fully + // compatible change, but it makes minimal impact that only new priority + // is not recognized by the old nodes during rolling update. + int64 priority = 15; } message HardState { - uint64 term = 1; - uint64 vote = 2; - uint64 commit = 3; + uint64 term = 1; + uint64 vote = 2; + uint64 commit = 3; } enum ConfChangeTransition { - // Automatically use the simple protocol if possible, otherwise fall back - // to ConfChangeType::Implicit. Most applications will want to use this. - Auto = 0; - // Use joint consensus unconditionally, and transition out of them - // automatically (by proposing a zero configuration change). - // - // This option is suitable for applications that want to minimize the time - // spent in the joint configuration and do not store the joint configuration - // in the state machine (outside of InitialState). - Implicit = 1; - // Use joint consensus and remain in the joint configuration until the - // application proposes a no-op configuration change. This is suitable for - // applications that want to explicitly control the transitions, for example - // to use a custom payload (via the Context field). - Explicit = 2; + // Automatically use the simple protocol if possible, otherwise fall back + // to ConfChangeType::Implicit. Most applications will want to use this. + Auto = 0; + // Use joint consensus unconditionally, and transition out of them + // automatically (by proposing a zero configuration change). + // + // This option is suitable for applications that want to minimize the time + // spent in the joint configuration and do not store the joint configuration + // in the state machine (outside of InitialState). + Implicit = 1; + // Use joint consensus and remain in the joint configuration until the + // application proposes a no-op configuration change. This is suitable for + // applications that want to explicitly control the transitions, for example + // to use a custom payload (via the Context field). + Explicit = 2; } message ConfState { - repeated uint64 voters = 1; - repeated uint64 learners = 2; - - // The voters in the outgoing config. If not empty the node is in joint consensus. - repeated uint64 voters_outgoing = 3; - // The nodes that will become learners when the outgoing config is removed. - // These nodes are necessarily currently in nodes_joint (or they would have - // been added to the incoming config right away). - repeated uint64 learners_next = 4; - // If set, the config is joint and Raft will automatically transition into - // the final config (i.e. remove the outgoing config) when this is safe. - bool auto_leave = 5; + repeated uint64 voters = 1; + repeated uint64 learners = 2; + + // The voters in the outgoing config. If not empty the node is in joint consensus. + repeated uint64 voters_outgoing = 3; + // The nodes that will become learners when the outgoing config is removed. + // These nodes are necessarily currently in nodes_joint (or they would have + // been added to the incoming config right away). + repeated uint64 learners_next = 4; + // If set, the config is joint and Raft will automatically transition into + // the final config (i.e. remove the outgoing config) when this is safe. + bool auto_leave = 5; } enum ConfChangeType { - AddNode = 0; - RemoveNode = 1; - AddLearnerNode = 2; + AddNode = 0; + RemoveNode = 1; + AddLearnerNode = 2; } message ConfChange { - ConfChangeType change_type = 2; - uint64 node_id = 3; - bytes context = 4; + ConfChangeType change_type = 2; + uint64 node_id = 3; + bytes context = 4; - uint64 id = 1; + uint64 id = 1; } // ConfChangeSingle is an individual configuration change operation. Multiple // such operations can be carried out atomically via a ConfChangeV2. message ConfChangeSingle { - ConfChangeType change_type = 1; - uint64 node_id = 2; + ConfChangeType change_type = 1; + uint64 node_id = 2; } // ConfChangeV2 messages initiate configuration changes. They support both the @@ -181,7 +181,7 @@ message ConfChangeSingle { // // [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf message ConfChangeV2 { - ConfChangeTransition transition = 1; - repeated ConfChangeSingle changes = 2; - bytes context = 3; + ConfChangeTransition transition = 1; + repeated ConfChangeSingle changes = 2; + bytes context = 3; } diff --git a/proto/include/gogoproto/gogo.proto b/proto/include/gogoproto/gogo.proto index bc8d889f..b80c8565 100644 --- a/proto/include/gogoproto/gogo.proto +++ b/proto/include/gogoproto/gogo.proto @@ -84,6 +84,9 @@ extend google.protobuf.FileOptions { optional bool goproto_registration = 63032; optional bool messagename_all = 63033; + + optional bool goproto_sizecache_all = 63034; + optional bool goproto_unkeyed_all = 63035; } extend google.protobuf.MessageOptions { @@ -118,6 +121,9 @@ extend google.protobuf.MessageOptions { optional bool typedecl = 64030; optional bool messagename = 64033; + + optional bool goproto_sizecache = 64034; + optional bool goproto_unkeyed = 64035; } extend google.protobuf.FieldOptions { @@ -133,4 +139,6 @@ extend google.protobuf.FieldOptions { optional bool stdtime = 65010; optional bool stdduration = 65011; + optional bool wktpointer = 65012; + } diff --git a/proto/include/rustproto.proto b/proto/include/rustproto.proto index 83e76fdf..1bc53345 100644 --- a/proto/include/rustproto.proto +++ b/proto/include/rustproto.proto @@ -1,47 +1,47 @@ syntax = "proto2"; -import "google/protobuf/descriptor.proto"; - // see https://github.com/gogo/protobuf/blob/master/gogoproto/gogo.proto // for the original idea package rustproto; +import "google/protobuf/descriptor.proto"; + extend google.protobuf.FileOptions { - // When true, oneof field is generated public - optional bool expose_oneof_all = 17001; - // When true all fields are public, and not accessors generated - optional bool expose_fields_all = 17003; - // When false, `get_`, `set_`, `mut_` etc. accessors are not generated - optional bool generate_accessors_all = 17004; - // Use `bytes::Bytes` for `bytes` fields - optional bool carllerche_bytes_for_bytes_all = 17011; - // Use `bytes::Bytes` for `string` fields - optional bool carllerche_bytes_for_string_all = 17012; - // When true, will only generate codes that works with lite runtime. - optional bool lite_runtime_all = 17035; + // When true, oneof field is generated public + optional bool expose_oneof_all = 17001; + // When true all fields are public, and not accessors generated + optional bool expose_fields_all = 17003; + // When false, `get_`, `set_`, `mut_` etc. accessors are not generated + optional bool generate_accessors_all = 17004; + // Use `bytes::Bytes` for `bytes` fields + optional bool carllerche_bytes_for_bytes_all = 17011; + // Use `bytes::Bytes` for `string` fields + optional bool carllerche_bytes_for_string_all = 17012; + // When true, will only generate codes that works with lite runtime. + optional bool lite_runtime_all = 17035; } extend google.protobuf.MessageOptions { - // When true, oneof field is generated public - optional bool expose_oneof = 17001; - // When true all fields are public, and not accessors generated - optional bool expose_fields = 17003; - // When false, `get_`, `set_`, `mut_` etc. accessors are not generated - optional bool generate_accessors = 17004; - // Use `bytes::Bytes` for `bytes` fields - optional bool carllerche_bytes_for_bytes = 17011; - // Use `bytes::Bytes` for `string` fields - optional bool carllerche_bytes_for_string = 17012; + // When true, oneof field is generated public + optional bool expose_oneof = 17001; + // When true all fields are public, and not accessors generated + optional bool expose_fields = 17003; + // When false, `get_`, `set_`, `mut_` etc. accessors are not generated + optional bool generate_accessors = 17004; + // Use `bytes::Bytes` for `bytes` fields + optional bool carllerche_bytes_for_bytes = 17011; + // Use `bytes::Bytes` for `string` fields + optional bool carllerche_bytes_for_string = 17012; } extend google.protobuf.FieldOptions { - // When true all fields are public, and not accessors generated - optional bool expose_fields_field = 17003; - // When false, `get_`, `set_`, `mut_` etc. accessors are not generated - optional bool generate_accessors_field = 17004; - // Use `bytes::Bytes` for `bytes` fields - optional bool carllerche_bytes_for_bytes_field = 17011; - // Use `bytes::Bytes` for `string` fields - optional bool carllerche_bytes_for_string_field = 17012; -} \ No newline at end of file + // When true all fields are public, and not accessors generated + optional bool expose_fields_field = 17003; + // When false, `get_`, `set_`, `mut_` etc. accessors are not generated + optional bool generate_accessors_field = 17004; + // Use `bytes::Bytes` for `bytes` fields + optional bool carllerche_bytes_for_bytes_field = 17011; + // Use `bytes::Bytes` for `string` fields + optional bool carllerche_bytes_for_string_field = 17012; +} diff --git a/proto/keyspacepb.proto b/proto/keyspacepb.proto index e817e980..033c01e5 100644 --- a/proto/keyspacepb.proto +++ b/proto/keyspacepb.proto @@ -1,24 +1,26 @@ syntax = "proto3"; package keyspacepb; -import "pdpb.proto"; - import "gogoproto/gogo.proto"; +import "pdpb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Keyspace provides services to manage keyspaces. service Keyspace { - rpc LoadKeyspace (LoadKeyspaceRequest) returns (LoadKeyspaceResponse) {} + rpc LoadKeyspace(LoadKeyspaceRequest) returns (LoadKeyspaceResponse) {} + rpc LoadKeyspaceByID(LoadKeyspaceByIDRequest) returns (LoadKeyspaceResponse) {} // WatchKeyspaces first return all current keyspaces' metadata as its first response. // Then, it returns responses containing keyspaces that had their metadata changed. - rpc WatchKeyspaces (WatchKeyspacesRequest) returns (stream WatchKeyspacesResponse) {} + rpc WatchKeyspaces(WatchKeyspacesRequest) returns (stream WatchKeyspacesResponse) {} rpc UpdateKeyspaceState(UpdateKeyspaceStateRequest) returns (UpdateKeyspaceStateResponse) {} rpc GetAllKeyspaces(GetAllKeyspacesRequest) returns (GetAllKeyspacesResponse) {} } @@ -44,6 +46,11 @@ message LoadKeyspaceRequest { string name = 2; } +message LoadKeyspaceByIDRequest { + pdpb.RequestHeader header = 1; + uint32 id = 2; +} + message LoadKeyspaceResponse { pdpb.ResponseHeader header = 1; KeyspaceMeta keyspace = 2; @@ -58,24 +65,24 @@ message WatchKeyspacesResponse { repeated KeyspaceMeta keyspaces = 2; } -message UpdateKeyspaceStateRequest{ +message UpdateKeyspaceStateRequest { pdpb.RequestHeader header = 1; uint32 id = 2; KeyspaceState state = 3; } -message UpdateKeyspaceStateResponse{ +message UpdateKeyspaceStateResponse { pdpb.ResponseHeader header = 1; KeyspaceMeta keyspace = 2; } -message GetAllKeyspacesRequest{ +message GetAllKeyspacesRequest { pdpb.RequestHeader header = 1; uint32 start_id = 2; uint32 limit = 3; } -message GetAllKeyspacesResponse{ +message GetAllKeyspacesResponse { pdpb.ResponseHeader header = 1; repeated KeyspaceMeta keyspaces = 2; } diff --git a/proto/kvrpcpb.proto b/proto/kvrpcpb.proto index 288ff39e..d2398e8d 100644 --- a/proto/kvrpcpb.proto +++ b/proto/kvrpcpb.proto @@ -1,21 +1,23 @@ syntax = "proto3"; package kvrpcpb; -import "metapb.proto"; +import "deadlock.proto"; import "errorpb.proto"; import "gogoproto/gogo.proto"; +import "metapb.proto"; +import "resource_manager.proto"; import "rustproto.proto"; -import "deadlock.proto"; import "tracepb.proto"; -import "resource_manager.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // This proto file defines requests, responses, and helper messages for KV and raw // APIs of TiKV (see tikvpb.proto). @@ -24,268 +26,280 @@ option java_package = "org.tikv.kvproto"; // A transactional get command. Lookup a value for `key` in the transaction with // starting timestamp = `version`. message GetRequest { - Context context = 1; - bytes key = 2; - uint64 version = 3; + Context context = 1; + bytes key = 2; + uint64 version = 3; + // If true, the response will include the commit ts of the key. + bool need_commit_ts = 4; } message GetResponse { - // A region error indicates that the request was sent to the wrong TiKV node - // (or other, similar errors). - errorpb.Error region_error = 1; - // A value could not be retrieved due to the state of the database for the requested key. - KeyError error = 2; - // A successful result. - bytes value = 3; - // True if the key does not exist in the database. - bool not_found = 4; - reserved 5; - // Time and scan details when processing the request. - ExecDetailsV2 exec_details_v2 = 6; + // A region error indicates that the request was sent to the wrong TiKV node + // (or other, similar errors). + errorpb.Error region_error = 1; + // A value could not be retrieved due to the state of the database for the requested key. + KeyError error = 2; + // A successful result. + bytes value = 3; + // True if the key does not exist in the database. + bool not_found = 4; + reserved 5; + // Time and scan details when processing the request. + ExecDetailsV2 exec_details_v2 = 6; + // The commit timestamp of the key. + // If it is zero, it means the commit timestamp is unknown. + uint64 commit_ts = 7; } // Scan fetches values for a range of keys; it is part of the transaction with // starting timestamp = `version`. message ScanRequest { - Context context = 1; - bytes start_key = 2; - // The maximum number of results to return. - uint32 limit = 3; - uint64 version = 4; - // Return only the keys found by scanning, not their values. - bool key_only = 5; - bool reverse = 6; - // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; - // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. - bytes end_key = 7; - // If sample_step > 0, skips 'sample_step - 1' number of keys after each returned key. - // locks are not checked. - uint32 sample_step = 8; + Context context = 1; + bytes start_key = 2; + // The maximum number of results to return. + uint32 limit = 3; + uint64 version = 4; + // Return only the keys found by scanning, not their values. + bool key_only = 5; + bool reverse = 6; + // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; + // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. + bytes end_key = 7; + // If sample_step > 0, skips 'sample_step - 1' number of keys after each returned key. + // locks are not checked. + uint32 sample_step = 8; } message ScanResponse { - errorpb.Error region_error = 1; - // Each KvPair may contain a key error. - repeated KvPair pairs = 2; - // This KeyError exists when some key is locked but we cannot check locks of all keys. - // In this case, `pairs` should be empty and the client should redo scanning all the keys - // after resolving the lock. - KeyError error = 3; + errorpb.Error region_error = 1; + // Each KvPair may contain a key error. + repeated KvPair pairs = 2; + // This KeyError exists when some key is locked but we cannot check locks of all keys. + // In this case, `pairs` should be empty and the client should redo scanning all the keys + // after resolving the lock. + KeyError error = 3; } // A prewrite is the first phase of writing to TiKV. It contains all data to be written in a transaction. // TiKV will write the data in a preliminary state. Data cannot be read until it has been committed. // The client should only commit a transaction once all prewrites succeed. message PrewriteRequest { - // What kind of checks need to be performed for keys in a pessimistic transaction. - enum PessimisticAction { - // The key needn't be locked and no extra write conflict checks are needed. - SKIP_PESSIMISTIC_CHECK = 0; - // The key should have been locked at the time of prewrite. - DO_PESSIMISTIC_CHECK = 1; - // The key doesn't need a pessimistic lock. But we need to do data constraint checks. - DO_CONSTRAINT_CHECK = 2; - } - - // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. - message ForUpdateTSConstraint { - // The index of key in the prewrite request that should be checked. - uint32 index = 1; - // The expected for_update_ts of the pessimistic lock of the key. - uint64 expected_for_update_ts = 2; - } - - Context context = 1; - // The data to be written to the database. - repeated Mutation mutations = 2; - // The client picks one key to be primary (unrelated to the primary key concept in SQL). This - // key's lock is the source of truth for the state of a transaction. All other locks due to a - // transaction will point to the primary lock. - bytes primary_lock = 3; - // Identifies the transaction being written. - uint64 start_version = 4; - uint64 lock_ttl = 5; - // TiKV can skip some checks, used for speeding up data migration. - bool skip_constraint_check = 6; - // For pessimistic transaction, some mutations don't need to be locked, for example, non-unique index key. - // Keys with deferred constraint checks are not locked. - repeated PessimisticAction pessimistic_actions = 7; - // How many keys this transaction involves in this region. - uint64 txn_size = 8; - // For pessimistic transactions only; used to check if a conflict lock is already committed. - uint64 for_update_ts = 9; - // If min_commit_ts > 0, this is a large transaction request, the final commit_ts - // will be inferred from `min_commit_ts`. - uint64 min_commit_ts = 10; - // When async commit is enabled, `secondaries` should be set as the key list of all secondary - // locks if the request prewrites the primary lock. - bool use_async_commit = 11; - repeated bytes secondaries = 12; - // When the transaction involves only one region, it's possible to commit the transaction - // directly with 1PC protocol. - bool try_one_pc = 13; - // The max commit ts is reserved for limiting the commit ts of 1PC or async commit, which can be used to avoid - // inconsistency with schema change. - uint64 max_commit_ts = 14; - // The level of assertion to use on this prewrte request. - AssertionLevel assertion_level = 15; - // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. - // See https://github.com/tikv/tikv/issues/14311 - repeated ForUpdateTSConstraint for_update_ts_constraints = 16; - - // Reserved for file based transaction. - repeated uint64 txn_file_chunks = 100; + // What kind of checks need to be performed for keys in a pessimistic transaction. + enum PessimisticAction { + // The key needn't be locked and no extra write conflict checks are needed. + // Deprecated in next-gen (cloud-storage-engine). + SKIP_PESSIMISTIC_CHECK = 0; + // The key should have been locked at the time of prewrite. If the lock is missing, + // the lock will be amended. This is the normal case for pessimistic transactions. + DO_PESSIMISTIC_CHECK = 1; + // The key does not acquire a pessimistic lock for performance optimization. + // Constraint checking (write conflicts + data constraints) is deferred to prewrite. + DO_CONSTRAINT_CHECK = 2; + } + + // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. + message ForUpdateTSConstraint { + // The index of key in the prewrite request that should be checked. + uint32 index = 1; + // The expected for_update_ts of the pessimistic lock of the key. + uint64 expected_for_update_ts = 2; + } + + Context context = 1; + // The data to be written to the database. + repeated Mutation mutations = 2; + // The client picks one key to be primary (unrelated to the primary key concept in SQL). This + // key's lock is the source of truth for the state of a transaction. All other locks due to a + // transaction will point to the primary lock. + bytes primary_lock = 3; + // Identifies the transaction being written. + uint64 start_version = 4; + uint64 lock_ttl = 5; + // TiKV can skip some checks, used for speeding up data migration. + bool skip_constraint_check = 6; + // For pessimistic transaction, some mutations don't need to be locked, for example, non-unique index key. + // Keys with deferred constraint checks are not locked. + repeated PessimisticAction pessimistic_actions = 7; + // How many keys this transaction involves in this region. + uint64 txn_size = 8; + // For pessimistic transactions only; used to check if a conflict lock is already committed. + uint64 for_update_ts = 9; + // If min_commit_ts > 0, this is a large transaction request, the final commit_ts + // will be inferred from `min_commit_ts`. + uint64 min_commit_ts = 10; + // When async commit is enabled, `secondaries` should be set as the key list of all secondary + // locks if the request prewrites the primary lock. + bool use_async_commit = 11; + repeated bytes secondaries = 12; + // When the transaction involves only one region, it's possible to commit the transaction + // directly with 1PC protocol. + bool try_one_pc = 13; + // The max commit ts is reserved for limiting the commit ts of 1PC or async commit, which can be used to avoid + // inconsistency with schema change. + uint64 max_commit_ts = 14; + // The level of assertion to use on this prewrte request. + AssertionLevel assertion_level = 15; + // for_update_ts constriants that should be checked when prewriting a pessimistic transaction. + // See https://github.com/tikv/tikv/issues/14311 + repeated ForUpdateTSConstraint for_update_ts_constraints = 16; + + // Reserved for file based transaction. + repeated uint64 txn_file_chunks = 100; } message PrewriteResponse { - errorpb.Error region_error = 1; - repeated KeyError errors = 2; - // 0 if the min_commit_ts is not ready or any other reason that async - // commit cannot proceed. The client can then fallback to normal way to - // continue committing the transaction if prewrite are all finished. - uint64 min_commit_ts = 3; - // When the transaction is successfully committed with 1PC protocol, this field will be set to - // the commit ts of the transaction. Otherwise, if TiKV failed to commit it with 1PC or the - // transaction is not 1PC, the value will be 0. - uint64 one_pc_commit_ts = 4; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 5; + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + // 0 if the min_commit_ts is not ready or any other reason that async + // commit cannot proceed. The client can then fallback to normal way to + // continue committing the transaction if prewrite are all finished. + uint64 min_commit_ts = 3; + // When the transaction is successfully committed with 1PC protocol, this field will be set to + // the commit ts of the transaction. Otherwise, if TiKV failed to commit it with 1PC or the + // transaction is not 1PC, the value will be 0. + uint64 one_pc_commit_ts = 4; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 5; } // Used to specify the behavior when a pessimistic lock request is woken up after waiting for another // lock. enum PessimisticLockWakeUpMode { - // When woken up, returns WriteConflict error to the client and the client should retry if necessary. - // In this mode, results of `return_values` or `check_existence` will be set to `values` and `not_founds` - // fields of the PessimisticLockResponse, which is compatible with old versions. - WakeUpModeNormal = 0; - // When woken up, continue trying to lock the key. This implicitly enables the `allow_lock_with_conflict` - // behavior, which means, allow acquiring the lock even if there is WriteConflict on the key. - // In this mode, `return_values` or `check_existence` fields of PessimisticLockResponse won't be used, and - // all results are carried in the `results` field. - WakeUpModeForceLock = 1; + // When woken up, returns WriteConflict error to the client and the client should retry if necessary. + // In this mode, results of `return_values` or `check_existence` will be set to `values` and `not_founds` + // fields of the PessimisticLockResponse, which is compatible with old versions. + WakeUpModeNormal = 0; + // When woken up, continue trying to lock the key. This implicitly enables the `allow_lock_with_conflict` + // behavior, which means, allow acquiring the lock even if there is WriteConflict on the key. + // In this mode, `return_values` or `check_existence` fields of PessimisticLockResponse won't be used, and + // all results are carried in the `results` field. + WakeUpModeForceLock = 1; } // Lock a set of keys to prepare to write to them. message PessimisticLockRequest { - Context context = 1; - // In this case every `Op` of the mutations must be `PessimisticLock`. - repeated Mutation mutations = 2; - bytes primary_lock = 3; - uint64 start_version = 4; - uint64 lock_ttl = 5; - // Each locking command in a pessimistic transaction has its own timestamp. If locking fails, then - // the corresponding SQL statement can be retried with a later timestamp, TiDB does not need to - // retry the whole transaction. The name comes from the `SELECT ... FOR UPDATE` SQL statement which - // is a locking read. Each `SELECT ... FOR UPDATE` in a transaction will be assigned its own - // timestamp. - uint64 for_update_ts = 6; - // If the request is the first lock request, we don't need to detect deadlock. - bool is_first_lock = 7; - // Time to wait for lock released in milliseconds when encountering locks. - // 0 means using default timeout in TiKV. Negative means no wait. - int64 wait_timeout = 8; - // If it is true, TiKV will acquire the pessimistic lock regardless of write conflict - // and return the latest value. It's only supported for single mutation. - bool force = 9 [deprecated = true]; - // If it is true, TiKV will return values of the keys if no error, so TiDB can cache the values for - // later read in the same transaction. - // When 'force' is set to true, this field is ignored. - bool return_values = 10; - // If min_commit_ts > 0, this is large transaction proto, the final commit_ts - // would be infered from min_commit_ts. - uint64 min_commit_ts = 11; - // If set to true, it means TiKV need to check if the key exists, and return the result in - // the `not_founds` feild in the response. This works no matter if `return_values` is set. If - // `return_values` is set, it simply makes no difference; otherwise, the `value` field of the - // repsonse will be empty while the `not_founds` field still indicates the keys' existence. - bool check_existence = 12; - // TiKV lock the record only when it exists - bool lock_only_if_exists = 13; - // Specifies the behavior when the request is woken up after wating for lock of another transaction. - PessimisticLockWakeUpMode wake_up_mode = 14; + Context context = 1; + // In this case every `Op` of the mutations must be `PessimisticLock`. + repeated Mutation mutations = 2; + bytes primary_lock = 3; + uint64 start_version = 4; + uint64 lock_ttl = 5; + // Each locking command in a pessimistic transaction has its own timestamp. If locking fails, then + // the corresponding SQL statement can be retried with a later timestamp, TiDB does not need to + // retry the whole transaction. The name comes from the `SELECT ... FOR UPDATE` SQL statement which + // is a locking read. Each `SELECT ... FOR UPDATE` in a transaction will be assigned its own + // timestamp. + uint64 for_update_ts = 6; + // If the request is the first lock request, we don't need to detect deadlock. + bool is_first_lock = 7; + // Time to wait for lock released in milliseconds when encountering locks. + // 0 means using default timeout in TiKV. Negative means no wait. + int64 wait_timeout = 8; + // If it is true, TiKV will acquire the pessimistic lock regardless of write conflict + // and return the latest value. It's only supported for single mutation. + bool force = 9 [deprecated = true]; + // If it is true, TiKV will return values of the keys if no error, so TiDB can cache the values for + // later read in the same transaction. + // When 'force' is set to true, this field is ignored. + bool return_values = 10; + // If min_commit_ts > 0, this is large transaction proto, the final commit_ts + // would be infered from min_commit_ts. + uint64 min_commit_ts = 11; + // If set to true, it means TiKV need to check if the key exists, and return the result in + // the `not_founds` feild in the response. This works no matter if `return_values` is set. If + // `return_values` is set, it simply makes no difference; otherwise, the `value` field of the + // repsonse will be empty while the `not_founds` field still indicates the keys' existence. + bool check_existence = 12; + // TiKV lock the record only when it exists + bool lock_only_if_exists = 13; + // Specifies the behavior when the request is woken up after wating for lock of another transaction. + PessimisticLockWakeUpMode wake_up_mode = 14; } enum PessimisticLockKeyResultType { - LockResultNormal = 0; - LockResultLockedWithConflict = 1; - LockResultFailed = 2; + LockResultNormal = 0; + LockResultLockedWithConflict = 1; + LockResultFailed = 2; } message PessimisticLockKeyResult { - PessimisticLockKeyResultType type = 1; - bytes value = 2; - bool existence = 3; - // We allow a key be locked when there is write conflict (latest commit_ts > for_update_ts). - // In this case, the key is semantically locked by a newer for_update_ts. - // For each requested key, the field is non-zero if the key is locked with write conflict, and it - // equals to the commit_ts of the latest version of the specified key. The for_update_ts field - // of the lock that's actually written to TiKV will also be this value. At the same time, - // `value` and `existence` will be returned regardless to how `return_values` and - // `check_existence` are set. - uint64 locked_with_conflict_ts = 4; - // Hint the client that resolving lock is not needed for this lock. For `PessimisticLock` - // requests only. - bool skip_resolving_lock = 11; + PessimisticLockKeyResultType type = 1; + bytes value = 2; + bool existence = 3; + // We allow a key be locked when there is write conflict (latest commit_ts > for_update_ts). + // In this case, the key is semantically locked by a newer for_update_ts. + // For each requested key, the field is non-zero if the key is locked with write conflict, and it + // equals to the commit_ts of the latest version of the specified key. The for_update_ts field + // of the lock that's actually written to TiKV will also be this value. At the same time, + // `value` and `existence` will be returned regardless to how `return_values` and + // `check_existence` are set. + uint64 locked_with_conflict_ts = 4; + // Hint the client that resolving lock is not needed for this lock. For `PessimisticLock` + // requests only. + bool skip_resolving_lock = 11; } message PessimisticLockResponse { - errorpb.Error region_error = 1; - repeated KeyError errors = 2; - // It carries the latest value and its commit ts if force in PessimisticLockRequest is true. - uint64 commit_ts = 3 [deprecated = true]; - bytes value = 4 [deprecated = true]; - // The values is set if 'return_values' is true in the request and no error. - // If 'force' is true, this field is not used. - // Only used when `wake_up_mode` is `WakeUpModeNormal`. - repeated bytes values = 5; - // Indicates whether the values at the same index is correspond to an existing key. - // In legacy TiKV, this field is not used even 'force' is false. In that case, an empty value indicates - // two possible situations: (1) the key does not exist. (2) the key exists but the value is empty. - // Only used when `wake_up_mode` is `WakeUpModeNormal`. - repeated bool not_founds = 6; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 7; - // Results of the request. Only used when `wake_up_mode` is `WakeUpModeForceLock`. - repeated PessimisticLockKeyResult results = 8; + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + // It carries the latest value and its commit ts if force in PessimisticLockRequest is true. + uint64 commit_ts = 3 [deprecated = true]; + bytes value = 4 [deprecated = true]; + // The values is set if 'return_values' is true in the request and no error. + // If 'force' is true, this field is not used. + // Only used when `wake_up_mode` is `WakeUpModeNormal`. + repeated bytes values = 5; + // Indicates whether the values at the same index is correspond to an existing key. + // In legacy TiKV, this field is not used even 'force' is false. In that case, an empty value indicates + // two possible situations: (1) the key does not exist. (2) the key exists but the value is empty. + // Only used when `wake_up_mode` is `WakeUpModeNormal`. + repeated bool not_founds = 6; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 7; + // Results of the request. Only used when `wake_up_mode` is `WakeUpModeForceLock`. + repeated PessimisticLockKeyResult results = 8; } // Unlock keys locked using `PessimisticLockRequest`. message PessimisticRollbackRequest { - Context context = 1; - uint64 start_version = 2; - uint64 for_update_ts = 3; - repeated bytes keys = 4; + Context context = 1; + uint64 start_version = 2; + uint64 for_update_ts = 3; + repeated bytes keys = 4; } message PessimisticRollbackResponse { - errorpb.Error region_error = 1; - repeated KeyError errors = 2; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 3; + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 3; } // Used to update the lock_ttl of a psessimistic and/or large transaction to prevent it from been killed. message TxnHeartBeatRequest { - Context context = 1; - // The key of the lock to update. - bytes primary_lock = 2; - // Start timestamp of the large transaction. - uint64 start_version = 3; - // The new TTL the sender would like. - uint64 advise_lock_ttl = 4; - - // Reserved for file based transaction. - bool is_txn_file = 100; + Context context = 1; + // The key of the lock to update. + bytes primary_lock = 2; + // Start timestamp of the large transaction. + uint64 start_version = 3; + // The new TTL the sender would like. + uint64 advise_lock_ttl = 4; + // Optionally update PK's min_commit_ts. + // Only for non-async-commmit and non-1PC transactions. + // If it is 0, ignore this field. + uint64 min_commit_ts = 5; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message TxnHeartBeatResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // The TTL actually set on the requested lock. - uint64 lock_ttl = 3; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 4; + errorpb.Error region_error = 1; + KeyError error = 2; + // The TTL actually set on the requested lock. + uint64 lock_ttl = 3; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 4; } // CheckTxnStatusRequest checks the status of a transaction. @@ -294,1211 +308,1366 @@ message TxnHeartBeatResponse { // Otherwise, returns the TTL information for the transaction. // CheckTxnStatusRequest may also push forward the minCommitTS of a large transaction. message CheckTxnStatusRequest { - Context context = 1; - // Primary key and lock ts together to locate the primary lock of a transaction. - bytes primary_key = 2; - // Starting timestamp of the transaction being checked. - uint64 lock_ts = 3; - // The start timestamp of the transaction which this request is part of. - uint64 caller_start_ts = 4; - // The client must specify the current time to TiKV using this timestamp. It is used to check TTL - // timeouts. It may be inaccurate. - uint64 current_ts = 5; - // If true, then TiKV will leave a rollback tombstone in the write CF for `primary_key`, even if - // that key is not locked. - bool rollback_if_not_exist = 6; - // This field is set to true only if the transaction is known to fall back from async commit. - // Then, CheckTxnStatus treats the transaction as non-async-commit even if the use_async_commit - // field in the primary lock is true. - bool force_sync_commit = 7; - // If the check request is used to resolve or decide the transaction status for a input pessimistic - // lock, the transaction status could not be decided if the primary lock is pessimistic too and - // it's still uncertain. - bool resolving_pessimistic_lock = 8; - // Whether it's needed to check if the lock on the key (if any) is the primary lock. - // This is for handling some corner cases when a pessimistic transaction changes its primary - // (see https://github.com/pingcap/tidb/issues/42937 for details). This field is necessary - // because the old versions of clients cannot handle some results returned from TiKV correctly. - // For new versions, this field should always be set to true. - bool verify_is_primary = 9; - - // Reserved for file based transaction. - bool is_txn_file = 100; + Context context = 1; + // Primary key and lock ts together to locate the primary lock of a transaction. + bytes primary_key = 2; + // Starting timestamp of the transaction being checked. + uint64 lock_ts = 3; + // The start timestamp of the transaction which this request is part of. + uint64 caller_start_ts = 4; + // The client must specify the current time to TiKV using this timestamp. It is used to check TTL + // timeouts. It may be inaccurate. + uint64 current_ts = 5; + // If true, then TiKV will leave a rollback tombstone in the write CF for `primary_key`, even if + // that key is not locked. + bool rollback_if_not_exist = 6; + // This field is set to true only if the transaction is known to fall back from async commit. + // Then, CheckTxnStatus treats the transaction as non-async-commit even if the use_async_commit + // field in the primary lock is true. + bool force_sync_commit = 7; + // If the check request is used to resolve or decide the transaction status for a input pessimistic + // lock, the transaction status could not be decided if the primary lock is pessimistic too and + // it's still uncertain. + bool resolving_pessimistic_lock = 8; + // Whether it's needed to check if the lock on the key (if any) is the primary lock. + // This is for handling some corner cases when a pessimistic transaction changes its primary + // (see https://github.com/pingcap/tidb/issues/42937 for details). This field is necessary + // because the old versions of clients cannot handle some results returned from TiKV correctly. + // For new versions, this field should always be set to true. + bool verify_is_primary = 9; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message CheckTxnStatusResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Three kinds of transaction status: - // locked: lock_ttl > 0 - // committed: commit_version > 0 - // rollbacked: lock_ttl = 0 && commit_version = 0 - uint64 lock_ttl = 3; - uint64 commit_version = 4; - // The action performed by TiKV (and why if the action is to rollback). - Action action = 5; - LockInfo lock_info = 6; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 7; + errorpb.Error region_error = 1; + KeyError error = 2; + // Three kinds of transaction status: + // locked: lock_ttl > 0 + // committed: commit_version > 0 + // rollbacked: lock_ttl = 0 && commit_version = 0 + uint64 lock_ttl = 3; + uint64 commit_version = 4; + // The action performed by TiKV (and why if the action is to rollback). + Action action = 5; + LockInfo lock_info = 6; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 7; } // Part of the async commit protocol, checks for locks on all supplied keys. If a lock is missing, // does not have a successful status, or belongs to another transaction, TiKV will leave a rollback // tombstone for that key. message CheckSecondaryLocksRequest { - Context context = 1; - repeated bytes keys = 2; - // Identifies the transaction we are investigating. - uint64 start_version = 3; + Context context = 1; + repeated bytes keys = 2; + // Identifies the transaction we are investigating. + uint64 start_version = 3; } message CheckSecondaryLocksResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // For each key in `keys` in `CheckSecondaryLocks`, there will be a lock in - // this list if there is a lock present and belonging to the correct transaction, - // nil otherwise. - repeated LockInfo locks = 3; - // If any of the locks have been committed, this is the commit ts used. If no - // locks have been committed, it will be zero. - uint64 commit_ts = 4; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 5; + errorpb.Error region_error = 1; + KeyError error = 2; + // For each key in `keys` in `CheckSecondaryLocks`, there will be a lock in + // this list if there is a lock present and belonging to the correct transaction, + // nil otherwise. + repeated LockInfo locks = 3; + // If any of the locks have been committed, this is the commit ts used. If no + // locks have been committed, it will be zero. + uint64 commit_ts = 4; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 5; +} + +// CommitRole indicates the current commit request is a primary commit or a secondary commit. +enum CommitRole { + // Unknown about the current request is a primary commit or a secondary commit; + Unknown = 0; + // Current request is a primary commit; + Primary = 1; + // Current request is a secondary commit; + Secondary = 2; } // The second phase of writing to TiKV. If there are no errors or conflicts, then this request // commits a transaction so that its data can be read by other transactions. message CommitRequest { - reserved 5; - reserved "binlog"; - Context context = 1; - // Identifies the transaction. - uint64 start_version = 2; - // All keys in the transaction (to be committed). - repeated bytes keys = 3; - // Timestamp for the end of the transaction. Must be greater than `start_version`. - uint64 commit_version = 4; - - // Reserved for file based transaction. - bool is_txn_file = 100; + reserved 5; + reserved "binlog"; + Context context = 1; + // Identifies the transaction. + uint64 start_version = 2; + // All keys in the transaction (to be committed). + repeated bytes keys = 3; + // Timestamp for the end of the transaction. Must be greater than `start_version`. + uint64 commit_version = 4; + // commit_role indicates the current commit request is a primary commit or a secondary commit. + // It's value maybe `Unknown` when using a client with an old version. + CommitRole commit_role = 6; + // primary_key indicates the primary key of the transaction. + // Its value may be empty when using an old version client. + bytes primary_key = 7; + // Whether committing an `use_async_commit` (i.e. should be treated as committed) prewrite. + bool use_async_commit = 8; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message CommitResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // If the commit ts is derived from min_commit_ts, this field should be set. - uint64 commit_version = 3; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 4; + errorpb.Error region_error = 1; + KeyError error = 2; + // If the commit ts is derived from min_commit_ts, this field should be set. + uint64 commit_version = 3; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 4; } // Not yet implemented. message ImportRequest { - repeated Mutation mutations = 1; - uint64 commit_version = 2; + repeated Mutation mutations = 1; + uint64 commit_version = 2; } message ImportResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Cleanup a key by possibly unlocking it. // From 4.0 onwards, this message is no longer used. message CleanupRequest { - Context context = 1; - bytes key = 2; - uint64 start_version = 3; - // The current timestamp, used in combination with a lock's TTL to determine - // if the lock has expired. If `current_ts == 0`, then the key will be unlocked - // irrespective of its TTL. - uint64 current_ts = 4; + Context context = 1; + bytes key = 2; + uint64 start_version = 3; + // The current timestamp, used in combination with a lock's TTL to determine + // if the lock has expired. If `current_ts == 0`, then the key will be unlocked + // irrespective of its TTL. + uint64 current_ts = 4; } message CleanupResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Set if the key is already committed. - uint64 commit_version = 3; + errorpb.Error region_error = 1; + KeyError error = 2; + // Set if the key is already committed. + uint64 commit_version = 3; } // Similar to a `Get` request. message BatchGetRequest { - Context context = 1; - repeated bytes keys = 2; - uint64 version = 3; + Context context = 1; + repeated bytes keys = 2; + uint64 version = 3; + // If true, the response will include the commit ts of the key. + bool need_commit_ts = 4; } message BatchGetResponse { - errorpb.Error region_error = 1; - repeated KvPair pairs = 2; - reserved 3; - // Time and scan details when processing the request. - ExecDetailsV2 exec_details_v2 = 4; - // This KeyError exists when some key is locked but we cannot check locks of all keys. - // In this case, `pairs` should be empty and the client should redo batch get all the keys - // after resolving the lock. - KeyError error = 5; + errorpb.Error region_error = 1; + repeated KvPair pairs = 2; + reserved 3; + // Time and scan details when processing the request. + ExecDetailsV2 exec_details_v2 = 4; + // This KeyError exists when some key is locked but we cannot check locks of all keys. + // In this case, `pairs` should be empty and the client should redo batch get all the keys + // after resolving the lock. + KeyError error = 5; } // Rollback a prewritten transaction. This will remove the preliminary data from the database, // unlock locks, and leave a rollback tombstone. message BatchRollbackRequest { - Context context = 1; - // Identify the transaction to be rolled back. - uint64 start_version = 2; - // The keys to rollback. - repeated bytes keys = 3; + Context context = 1; + // Identify the transaction to be rolled back. + uint64 start_version = 2; + // The keys to rollback. + repeated bytes keys = 3; - // Reserved for file based transaction. - bool is_txn_file = 100; + // Reserved for file based transaction. + bool is_txn_file = 100; } message BatchRollbackResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 3; + errorpb.Error region_error = 1; + KeyError error = 2; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 3; } // Scan the database for locks. Used at the start of the GC process to find all // old locks. message ScanLockRequest { - Context context = 1; - // Returns all locks with a start timestamp before `max_version`. - uint64 max_version = 2; - // Start scanning from this key. - bytes start_key = 3; - // The maximum number of locks to return. - uint32 limit = 4; - // The exclusive upperbound for scanning. - bytes end_key = 5; + Context context = 1; + // Returns all locks with a start timestamp before `max_version`. + uint64 max_version = 2; + // Start scanning from this key. + bytes start_key = 3; + // The maximum number of locks to return. + uint32 limit = 4; + // The exclusive upperbound for scanning. + bytes end_key = 5; } message ScanLockResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Info on all locks found by the scan. - repeated LockInfo locks = 3; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 4; + errorpb.Error region_error = 1; + KeyError error = 2; + // Info on all locks found by the scan. + repeated LockInfo locks = 3; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 4; } // For all keys locked by the transaction identified by `start_version`, either // commit or rollback the transaction and unlock the key. message ResolveLockRequest { - Context context = 1; - uint64 start_version = 2; - // `commit_version == 0` means the transaction was rolled back. - // `commit_version > 0` means the transaction was committed at the given timestamp. - uint64 commit_version = 3; - repeated TxnInfo txn_infos = 4; - // Only resolve specified keys. - repeated bytes keys = 5; + Context context = 1; + uint64 start_version = 2; + // `commit_version == 0` means the transaction was rolled back. + // `commit_version > 0` means the transaction was committed at the given timestamp. + uint64 commit_version = 3; + repeated TxnInfo txn_infos = 4; + // Only resolve specified keys. + repeated bytes keys = 5; + + // If true, the resolve lock action is performed asynchronously and the response may be returned before it's finished. + bool is_async = 6; - // Reserved for file based transaction. - bool is_txn_file = 100; + // Reserved for file based transaction. + bool is_txn_file = 100; } message ResolveLockResponse { - errorpb.Error region_error = 1; - KeyError error = 2; - // Execution details about the request processing. - ExecDetailsV2 exec_details_v2 = 3; + errorpb.Error region_error = 1; + KeyError error = 2; + // Execution details about the request processing. + ExecDetailsV2 exec_details_v2 = 3; } // Request TiKV to garbage collect all non-current data older than `safe_point`. message GCRequest { - Context context = 1; - uint64 safe_point = 2; + Context context = 1; + uint64 safe_point = 2; } message GCResponse { - errorpb.Error region_error = 1; - KeyError error = 2; + errorpb.Error region_error = 1; + KeyError error = 2; } // Delete a range of data from TiKV. message DeleteRangeRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - // If true, the data will not be immediately deleted, but the operation will - // still be replicated via Raft. This is used to notify TiKV that the data - // will be deleted using `unsafe_destroy_range` soon. - bool notify_only = 4; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + // If true, the data will not be immediately deleted, but the operation will + // still be replicated via Raft. This is used to notify TiKV that the data + // will be deleted using `unsafe_destroy_range` soon. + bool notify_only = 4; } message DeleteRangeResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Preparing the flashback for a region/key range will "lock" the region // so that there is no any read, write or schedule operation could be proposed before // the actual flashback operation. message PrepareFlashbackToVersionRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - // The `start_ts` which we will use to write a lock to prevent - // the `resolved_ts` from advancing during the whole process. - uint64 start_ts = 4; - // The TS version which the data will flashback to later. - uint64 version = 5; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + // The `start_ts` which we will use to write a lock to prevent + // the `resolved_ts` from advancing during the whole process. + uint64 start_ts = 4; + // The TS version which the data will flashback to later. + uint64 version = 5; } message PrepareFlashbackToVersionResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Flashback the region to a specific point with the given `version`, please // make sure the region is "locked" by `PrepareFlashbackToVersionRequest` first, // otherwise this request will fail. message FlashbackToVersionRequest { - Context context = 1; - // The TS version which the data should flashback to. - uint64 version = 2; - bytes start_key = 3; - bytes end_key = 4; - // The `start_ts` and `commit_ts` which the newly written MVCC version will use. - // Please make sure the `start_ts` is the same one in `PrepareFlashbackToVersionRequest`. - uint64 start_ts = 5; - uint64 commit_ts = 6; + Context context = 1; + // The TS version which the data should flashback to. + uint64 version = 2; + bytes start_key = 3; + bytes end_key = 4; + // The `start_ts` and `commit_ts` which the newly written MVCC version will use. + // Please make sure the `start_ts` is the same one in `PrepareFlashbackToVersionRequest`. + uint64 start_ts = 5; + uint64 commit_ts = 6; } message FlashbackToVersionResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } // Raw commands. message RawGetRequest { - Context context = 1; - bytes key = 2; - string cf = 3; + Context context = 1; + bytes key = 2; + string cf = 3; } message RawGetResponse { - errorpb.Error region_error = 1; - string error = 2; - bytes value = 3; - bool not_found = 4; + errorpb.Error region_error = 1; + string error = 2; + bytes value = 3; + bool not_found = 4; } message RawBatchGetRequest { - Context context = 1; - repeated bytes keys = 2; - string cf = 3; + Context context = 1; + repeated bytes keys = 2; + string cf = 3; } message RawBatchGetResponse { - errorpb.Error region_error = 1; - repeated KvPair pairs = 2; + errorpb.Error region_error = 1; + repeated KvPair pairs = 2; } message RawPutRequest { - Context context = 1; - bytes key = 2; - bytes value = 3; - string cf = 4; - uint64 ttl = 5; - bool for_cas = 6; + Context context = 1; + bytes key = 2; + bytes value = 3; + string cf = 4; + uint64 ttl = 5; + bool for_cas = 6; } message RawPutResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawBatchPutRequest { - Context context = 1; - repeated KvPair pairs = 2; - string cf = 3; - uint64 ttl = 4 [deprecated=true]; - bool for_cas = 5; + Context context = 1; + repeated KvPair pairs = 2; + string cf = 3; + uint64 ttl = 4 [deprecated = true]; + bool for_cas = 5; - // The time-to-live for each keys in seconds, and if the length of `ttls` - // is exactly one, the ttl will be applied to all keys. Otherwise, the length - // mismatch between `ttls` and `pairs` will return an error. - repeated uint64 ttls = 6; + // The time-to-live for each keys in seconds, and if the length of `ttls` + // is exactly one, the ttl will be applied to all keys. Otherwise, the length + // mismatch between `ttls` and `pairs` will return an error. + repeated uint64 ttls = 6; } message RawBatchPutResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawDeleteRequest { - Context context = 1; - bytes key = 2; - string cf = 3; - bool for_cas = 4; + Context context = 1; + bytes key = 2; + string cf = 3; + bool for_cas = 4; } message RawDeleteResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawBatchDeleteRequest { - Context context = 1; - repeated bytes keys = 2; - string cf = 3; - bool for_cas = 4; + Context context = 1; + repeated bytes keys = 2; + string cf = 3; + bool for_cas = 4; } message RawBatchDeleteResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawScanRequest { - Context context = 1; - bytes start_key = 2; - uint32 limit = 3; - bool key_only = 4; - string cf = 5; - bool reverse = 6; - // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; - // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. - bytes end_key = 7; + Context context = 1; + bytes start_key = 2; + uint32 limit = 3; + bool key_only = 4; + string cf = 5; + bool reverse = 6; + // For compatibility, when scanning forward, the range to scan is [start_key, end_key), where start_key < end_key; + // and when scanning backward, it scans [end_key, start_key) in descending order, where end_key < start_key. + bytes end_key = 7; } message RawScanResponse { - errorpb.Error region_error = 1; - repeated KvPair kvs = 2; + errorpb.Error region_error = 1; + repeated KvPair kvs = 2; } message RawDeleteRangeRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; - string cf = 4; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; + string cf = 4; } message RawDeleteRangeResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RawBatchScanRequest { - Context context = 1; - repeated KeyRange ranges = 2; // scanning range - uint32 each_limit = 3; // max number of returning kv pairs for each scanning range - bool key_only = 4; - string cf = 5; - bool reverse = 6; + Context context = 1; + repeated KeyRange ranges = 2; // scanning range + uint32 each_limit = 3; // max number of returning kv pairs for each scanning range + bool key_only = 4; + string cf = 5; + bool reverse = 6; } message RawBatchScanResponse { - errorpb.Error region_error = 1; - repeated KvPair kvs = 2; + errorpb.Error region_error = 1; + repeated KvPair kvs = 2; } // Store commands (sent to a whole TiKV cluster, rather than a certain region). message UnsafeDestroyRangeRequest { - Context context = 1; - bytes start_key = 2; - bytes end_key = 3; + Context context = 1; + bytes start_key = 2; + bytes end_key = 3; } message UnsafeDestroyRangeResponse { - errorpb.Error region_error = 1; - string error = 2; + errorpb.Error region_error = 1; + string error = 2; } message RegisterLockObserverRequest { - Context context = 1; - uint64 max_ts = 2; + Context context = 1; + uint64 max_ts = 2; } message RegisterLockObserverResponse { - string error = 1; + string error = 1; } message CheckLockObserverRequest { - Context context = 1; - uint64 max_ts = 2; + Context context = 1; + uint64 max_ts = 2; } message CheckLockObserverResponse { - string error = 1; - bool is_clean = 2; - repeated LockInfo locks = 3; + string error = 1; + bool is_clean = 2; + repeated LockInfo locks = 3; } message RemoveLockObserverRequest { - Context context = 1; - uint64 max_ts = 2; + Context context = 1; + uint64 max_ts = 2; } message RemoveLockObserverResponse { - string error = 1; + string error = 1; } message PhysicalScanLockRequest { - Context context = 1; - uint64 max_ts = 2; - bytes start_key = 3; - uint32 limit = 4; + Context context = 1; + uint64 max_ts = 2; + bytes start_key = 3; + uint32 limit = 4; } message PhysicalScanLockResponse { - string error = 1; - repeated LockInfo locks = 2; + string error = 1; + repeated LockInfo locks = 2; } // Sent from PD to a TiKV node. message SplitRegionRequest { - Context context = 1; - bytes split_key = 2 [deprecated=true]; - repeated bytes split_keys = 3; // when use it to do batch split, `split_key` should be empty. - // Once enabled, the split_key will not be encoded. - bool is_raw_kv = 4; + Context context = 1; + bytes split_key = 2 [deprecated = true]; + repeated bytes split_keys = 3; // when use it to do batch split, `split_key` should be empty. + // Once enabled, the split_key will not be encoded. + bool is_raw_kv = 4; } message SplitRegionResponse { - errorpb.Error region_error = 1; - metapb.Region left = 2 [deprecated=true]; // set when there are only 2 result regions. - metapb.Region right = 3 [deprecated=true]; // set when there are only 2 result regions. - repeated metapb.Region regions = 4; // include all result regions. + errorpb.Error region_error = 1; + metapb.Region left = 2 [deprecated = true]; // set when there are only 2 result regions. + metapb.Region right = 3 [deprecated = true]; // set when there are only 2 result regions. + repeated metapb.Region regions = 4; // include all result regions. + + // Reserved for file based transaction. + repeated KeyError errors = 100; } // Sent from TiFlash to a TiKV node. -message ReadIndexRequest{ - Context context = 1; +message ReadIndexRequest { + Context context = 1; - // TiKV checks the given range if there is any unapplied lock - // blocking the read request. - uint64 start_ts = 2; - repeated KeyRange ranges = 3; + // TiKV checks the given range if there is any unapplied lock + // blocking the read request. + uint64 start_ts = 2; + repeated KeyRange ranges = 3; } -message ReadIndexResponse{ - errorpb.Error region_error = 1; - uint64 read_index = 2; - // If `locked` is set, this read request is blocked by a lock. - // The lock should be returned to the client. - kvrpcpb.LockInfo locked = 3; +message ReadIndexResponse { + errorpb.Error region_error = 1; + uint64 read_index = 2; + // If `locked` is set, this read request is blocked by a lock. + // The lock should be returned to the client. + kvrpcpb.LockInfo locked = 3; } // Commands for debugging transactions. message MvccGetByKeyRequest { - Context context = 1; - bytes key = 2; + Context context = 1; + bytes key = 2; } message MvccGetByKeyResponse { - errorpb.Error region_error = 1; - string error = 2; - MvccInfo info = 3; + errorpb.Error region_error = 1; + string error = 2; + MvccInfo info = 3; } message MvccGetByStartTsRequest { - Context context = 1; - uint64 start_ts = 2; + Context context = 1; + uint64 start_ts = 2; } message MvccGetByStartTsResponse { - errorpb.Error region_error = 1; - string error = 2; - bytes key = 3; - MvccInfo info = 4; + errorpb.Error region_error = 1; + string error = 2; + bytes key = 3; + MvccInfo info = 4; } // Helper messages. +enum RequestOrigin { + RequestOriginUnknown = 0; + RequestOriginTiDB = 1; +} + // Miscellaneous metadata attached to most requests. message Context { - reserved 4; - reserved "read_quorum"; - uint64 region_id = 1; - metapb.RegionEpoch region_epoch = 2; - metapb.Peer peer = 3; - uint64 term = 5; - CommandPri priority = 6; - IsolationLevel isolation_level = 7; - bool not_fill_cache = 8; - bool sync_log = 9; - - // True means execution time statistics should be recorded and returned. - bool record_time_stat = 10; - // True means RocksDB scan statistics should be recorded and returned. - bool record_scan_stat = 11; - - bool replica_read = 12; - // Read requests can ignore locks belonging to these transactions because either - // these transactions are rolled back or theirs commit_ts > read request's start_ts. - repeated uint64 resolved_locks = 13; - uint64 max_execution_duration_ms = 14; - - // After a region applies to `applied_index`, we can get a - // snapshot for the region even if the peer is a follower. - uint64 applied_index = 15; - // A hint for TiKV to schedule tasks more fairly. Query with same task ID - // may share same priority and resource quota. - uint64 task_id = 16; - - // Not required to read the most up-to-date data, replicas with `safe_ts` >= `start_ts` - // can handle read request directly - bool stale_read = 17; - - // Any additional serialized information about the request. - bytes resource_group_tag = 18; - - // Used to tell TiKV whether operations are allowed or not on different disk usages. - DiskFullOpt disk_full_opt = 19; - - // Indicates the request is a retry request and the same request may have been sent before. - bool is_retry_request = 20; - - // API version implies the encode of the key and value. - APIVersion api_version = 21; - - // Read request should read through locks belonging to these transactions because these - // transactions are committed and theirs commit_ts <= read request's start_ts. - repeated uint64 committed_locks = 22; - - // The informantion to trace a request sent to TiKV. - tracepb.TraceContext trace_context = 23; - - // The source of the request, will be used as the tag of the metrics reporting. - // This field can be set for any requests that require to report metrics with any extra labels. - string request_source = 24; - - // The source of the current transaction. - uint64 txn_source = 25; - - reserved 26; // Used for resource_group_name, now it's moved into ResourceControlContext. - - // If `busy_threshold_ms` is given, TiKV can reject the request and return a `ServerIsBusy` - // error before processing if the estimated waiting duration exceeds the threshold. - uint32 busy_threshold_ms = 27; - - // Some information used for resource control. - ResourceControlContext resource_control_context = 28; - - // The keyspace that the request is sent to. - // NOTE: This field is only meaningful while the api_version is V2. - uint32 keyspace_id = 32; - - // The buckets version that the request is sent to. - // NOTE: This field is only meaningful while enable buckets. - uint64 buckets_version = 33; - - // It tells us where the request comes from in TiDB. If it isn't from TiDB, leave it blank. - // This is for tests only and thus can be safely changed/removed without affecting compatibility. - SourceStmt source_stmt = 34; + reserved 4; + reserved "read_quorum"; + uint64 region_id = 1; + metapb.RegionEpoch region_epoch = 2; + metapb.Peer peer = 3; + uint64 term = 5; + CommandPri priority = 6; + IsolationLevel isolation_level = 7; + bool not_fill_cache = 8; + bool sync_log = 9; + + // True means execution time statistics should be recorded and returned. + bool record_time_stat = 10; + // True means RocksDB scan statistics should be recorded and returned. + bool record_scan_stat = 11; + + bool replica_read = 12; + // Read requests can ignore locks belonging to these transactions because either + // these transactions are rolled back or theirs commit_ts > read request's start_ts. + repeated uint64 resolved_locks = 13; + uint64 max_execution_duration_ms = 14; + + // After a region applies to `applied_index`, we can get a + // snapshot for the region even if the peer is a follower. + uint64 applied_index = 15; + // A hint for TiKV to schedule tasks more fairly. Query with same task ID + // may share same priority and resource quota. + uint64 task_id = 16; + + // Not required to read the most up-to-date data, replicas with `safe_ts` >= `start_ts` + // can handle read request directly + bool stale_read = 17; + + // Any additional serialized information about the request. + bytes resource_group_tag = 18; + + // Used to tell TiKV whether operations are allowed or not on different disk usages. + DiskFullOpt disk_full_opt = 19; + + // Indicates the request is a retry request and the same request may have been sent before. + bool is_retry_request = 20; + + // API version implies the encode of the key and value. + APIVersion api_version = 21; + + // Read request should read through locks belonging to these transactions because these + // transactions are committed and theirs commit_ts <= read request's start_ts. + repeated uint64 committed_locks = 22; + + // The informantion to trace a request sent to TiKV. + tracepb.TraceContext trace_context = 23; + + // The source of the request, will be used as the tag of the metrics reporting. + // This field can be set for any requests that require to report metrics with any extra labels. + string request_source = 24; + + // The source of the current transaction. + uint64 txn_source = 25; + + reserved 26; // Used for resource_group_name, now it's moved into ResourceControlContext. + + // If `busy_threshold_ms` is given, TiKV can reject the request and return a `ServerIsBusy` + // error before processing if the estimated waiting duration exceeds the threshold. + uint32 busy_threshold_ms = 27; + + // Some information used for resource control. + ResourceControlContext resource_control_context = 28; + + // The declared origin of the request. TiDB server requests set this to TIDB. + // This field is client-provided metadata; consumers should rely on it only when + // the caller is authenticated/authorized by the transport or deployment boundary. + // UNKNOWN means the origin is unset or not recognized by this protocol version. + // Other components should add dedicated enum variants when they need origin-specific behavior. + RequestOrigin request_origin = 29; + + // The keyspace that the request is sent to. + // NOTE: This field is only meaningful while the api_version is V2. + string keyspace_name = 31; + + // The keyspace that the request is sent to. + // NOTE: This field is only meaningful while the api_version is V2. + uint32 keyspace_id = 32; + + // The buckets version that the request is sent to. + // NOTE: This field is only meaningful while enable buckets. + uint64 buckets_version = 33; + + // It tells us where the request comes from in TiDB. If it isn't from TiDB, leave it blank. + // This is for tests only and thus can be safely changed/removed without affecting compatibility. + SourceStmt source_stmt = 34; + + // The cluster id of the request + uint64 cluster_id = 35; + + // The trace id of the request, will be used for tracing the request's execution's inner steps. + bytes trace_id = 36; + + // Control flags for trace logging behavior. + // Bit 0: immediate_log - Force immediate logging without buffering + // Bit 1: category_req_resp - Enable request/response tracing + // Bit 2: category_write_details - Enable detailed write tracing + // Bit 3: category_read_details - Enable detailed read tracing + // Bits 4-63: Reserved for future use + // This field is set by client-go based on an extractor function provided by TiDB. + uint64 trace_control_flags = 37; + + // TODO(slock): tidb should send requests with a feature flag to indicate whether it + // supports shared locks, so that tikv can fail the requests from old tidb versions + // when needed. } message ResourceControlContext { - // It's used to identify which resource group the request belongs to. - string resource_group_name = 1; + // It's used to identify which resource group the request belongs to. + string resource_group_name = 1; - // The resource consumption of the resource group that have completed at all TiKVs between the previous request to this TiKV and current request. - // It's used as penalty to make the local resource scheduling on one TiKV takes the gloabl resource consumption into consideration. - resource_manager.Consumption penalty = 2; + // The resource consumption of the resource group that have completed at all TiKVs between the previous request to this TiKV and current request. + // It's used as penalty to make the local resource scheduling on one TiKV takes the gloabl resource consumption into consideration. + resource_manager.Consumption penalty = 2; - // This priority would override the original priority of the resource group for the request. - // Used to deprioritize the runaway queries. - uint64 override_priority = 3; + // This priority would override the original priority of the resource group for the request. + // Used to deprioritize the runaway queries. + uint64 override_priority = 3; } // The API version the server and the client is using. // See more details in https://github.com/tikv/rfcs/blob/master/text/0069-api-v2.md. enum APIVersion { - // `V1` is mainly for TiDB & TxnKV, and is not safe to use RawKV along with the others. - // V1 server only accepts V1 requests. V1 raw requests with TTL will be rejected. - V1 = 0; - // - // `V1TTL` is only available to RawKV, and 8 bytes representing the unix timestamp in - // seconds for expiring time will be append to the value of all RawKV entries. For example: - // ------------------------------------------------------------ - // | User value | Expire Ts | - // ------------------------------------------------------------ - // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | - // ------------------------------------------------------------ - // V1TTL server only accepts V1 raw requests. - // V1 client should not use `V1TTL` in request. V1 client should always send `V1`. - V1TTL = 1; - // - // `V2` use new encoding for RawKV & TxnKV to support more features. - // - // Key Encoding: - // TiDB: start with `m` or `t`, the same as `V1`. - // TxnKV: prefix with `x`, encoded as `MCE( x{keyspace id} + {user key} ) + timestamp`. - // RawKV: prefix with `r`, encoded as `MCE( r{keyspace id} + {user key} ) + timestamp`. - // Where the `{keyspace id}` is fixed-length of 3 bytes in network byte order. - // Besides, RawKV entires must be in `default` CF. - // - // Value Encoding: - // TiDB & TxnKV: the same as `V1`. - // RawKV: `{user value} + {optional fields} + {meta flag}`. The last byte in the - // raw value must be meta flags. For example: - // -------------------------------------- - // | User value | Meta flags | - // -------------------------------------- - // | 0x12 0x34 0x56 | 0x00 (0b00000000) | - // -------------------------------------- - // Bit 0 of meta flags is for TTL. If set, the value contains 8 bytes expiring time as - // unix timestamp in seconds at the very left to the meta flags. - // -------------------------------------------------------------------------------- - // | User value | Expiring time | Meta flags | - // -------------------------------------------------------------------------------- - // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | 0x01 (0b00000001) | - // -------------------------------------------------------------------------------- - // Bit 1 is for deletion. If set, the entry is logical deleted. - // --------------------- - // | Meta flags | - // --------------------- - // | 0x02 (0b00000010) | - // --------------------- - // - // V2 server accpets V2 requests and V1 transactional requests that statrts with TiDB key - // prefix (`m` and `t`). - V2 = 2; + // `V1` is mainly for TiDB & TxnKV, and is not safe to use RawKV along with the others. + // V1 server only accepts V1 requests. V1 raw requests with TTL will be rejected. + V1 = 0; + // + // `V1TTL` is only available to RawKV, and 8 bytes representing the unix timestamp in + // seconds for expiring time will be append to the value of all RawKV entries. For example: + // ------------------------------------------------------------ + // | User value | Expire Ts | + // ------------------------------------------------------------ + // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | + // ------------------------------------------------------------ + // V1TTL server only accepts V1 raw requests. + // V1 client should not use `V1TTL` in request. V1 client should always send `V1`. + V1TTL = 1; + // + // `V2` use new encoding for RawKV & TxnKV to support more features. + // + // Key Encoding: + // TiDB: start with `m` or `t`, the same as `V1`. + // TxnKV: prefix with `x`, encoded as `MCE( x{keyspace id} + {user key} ) + timestamp`. + // RawKV: prefix with `r`, encoded as `MCE( r{keyspace id} + {user key} ) + timestamp`. + // Where the `{keyspace id}` is fixed-length of 3 bytes in network byte order. + // Besides, RawKV entires must be in `default` CF. + // + // Value Encoding: + // TiDB & TxnKV: the same as `V1`. + // RawKV: `{user value} + {optional fields} + {meta flag}`. The last byte in the + // raw value must be meta flags. For example: + // -------------------------------------- + // | User value | Meta flags | + // -------------------------------------- + // | 0x12 0x34 0x56 | 0x00 (0b00000000) | + // -------------------------------------- + // Bit 0 of meta flags is for TTL. If set, the value contains 8 bytes expiring time as + // unix timestamp in seconds at the very left to the meta flags. + // -------------------------------------------------------------------------------- + // | User value | Expiring time | Meta flags | + // -------------------------------------------------------------------------------- + // | 0x12 0x34 0x56 | 0x00 0x00 0x00 0x00 0x00 0x00 0xff 0xff | 0x01 (0b00000001) | + // -------------------------------------------------------------------------------- + // Bit 1 is for deletion. If set, the entry is logical deleted. + // --------------------- + // | Meta flags | + // --------------------- + // | 0x02 (0b00000010) | + // --------------------- + // + // V2 server accpets V2 requests and V1 transactional requests that statrts with TiDB key + // prefix (`m` and `t`). + V2 = 2; } message SourceStmt { - uint64 start_ts = 1; - uint64 connection_id = 2; - uint64 stmt_id = 3; - // session alias set by user - string session_alias = 4; + uint64 start_ts = 1; + uint64 connection_id = 2; + uint64 stmt_id = 3; + // session alias set by user + string session_alias = 4; } message LockInfo { - bytes primary_lock = 1; - uint64 lock_version = 2; - bytes key = 3; - uint64 lock_ttl = 4; - // How many keys this transaction involves in this region. - uint64 txn_size = 5; - Op lock_type = 6; - uint64 lock_for_update_ts = 7; - // Fields for transactions that are using Async Commit. - bool use_async_commit = 8; - uint64 min_commit_ts = 9; - repeated bytes secondaries = 10; - // The time elapsed since last update of lock wait info when waiting. - // It's used in timeout errors. 0 means unknown or not applicable. - // It can be used to help the client decide whether to try resolving the lock. - uint64 duration_to_last_update_ms = 11; - - // Reserved for file based transaction. - bool is_txn_file = 100; + bytes primary_lock = 1; + uint64 lock_version = 2; + bytes key = 3; + uint64 lock_ttl = 4; + // How many keys this transaction involves in this region. + uint64 txn_size = 5; + Op lock_type = 6; + uint64 lock_for_update_ts = 7; + // Fields for transactions that are using Async Commit. + bool use_async_commit = 8; + uint64 min_commit_ts = 9; + repeated bytes secondaries = 10; + // The time elapsed since last update of lock wait info when waiting. + // It's used in timeout errors. 0 means unknown or not applicable. + // It can be used to help the client decide whether to try resolving the lock. + uint64 duration_to_last_update_ms = 11; + // When lock_type is SharedLock, this describes transactions holding the shared lock. + // Important: when lock_type is SharedLock, all shared locks must use shared_lock_infos; + // DO NOT read from the wrapper LockInfo. + // TODO(slock): tidb should send requests with a feature flag to indicate whether it + // supports shared locks, so that tikv can fail the requests from old tidb versions + // when needed. + repeated LockInfo shared_lock_infos = 12; + + // Reserved for file based transaction. + bool is_txn_file = 100; } message KeyError { - LockInfo locked = 1; // Client should backoff or cleanup the lock then retry. - string retryable = 2; // Client may restart the txn. e.g write conflict. - string abort = 3; // Client should abort the txn. - WriteConflict conflict = 4; // Write conflict is moved from retryable to here. - AlreadyExist already_exist = 5; // Key already exists - Deadlock deadlock = 6; // Deadlock is used in pessimistic transaction for single statement rollback. - CommitTsExpired commit_ts_expired = 7; // Commit ts is earlier than min commit ts of a transaction. - TxnNotFound txn_not_found = 8; // Txn not found when checking txn status. - CommitTsTooLarge commit_ts_too_large = 9; // Calculated commit TS exceeds the limit given by the user. - AssertionFailed assertion_failed = 10; // Assertion of a `Mutation` is evaluated as a failure. - PrimaryMismatch primary_mismatch = 11; // CheckTxnStatus is sent to a lock that's not the primary. + LockInfo locked = 1; // Client should backoff or cleanup the lock then retry. + string retryable = 2; // Client may restart the txn. e.g write conflict. + string abort = 3; // Client should abort the txn. + WriteConflict conflict = 4; // Write conflict is moved from retryable to here. + AlreadyExist already_exist = 5; // Key already exists + Deadlock deadlock = 6; // Deadlock is used in pessimistic transaction for single statement rollback. + CommitTsExpired commit_ts_expired = 7; // Commit ts is earlier than min commit ts of a transaction. + TxnNotFound txn_not_found = 8; // Txn not found when checking txn status. + CommitTsTooLarge commit_ts_too_large = 9; // Calculated commit TS exceeds the limit given by the user. + AssertionFailed assertion_failed = 10; // Assertion of a `Mutation` is evaluated as a failure. + PrimaryMismatch primary_mismatch = 11; // CheckTxnStatus is sent to a lock that's not the primary. + TxnLockNotFound txn_lock_not_found = 12; // TxnLockNotFound indicates the txn lock is not found. + + // Extra information for error debugging + DebugInfo debug_info = 100; } message WriteConflict { - enum Reason { - Unknown = 0; - Optimistic = 1; // in optimistic transactions. - PessimisticRetry = 2; // a lock acquisition request waits for a lock and awakes, or meets a newer version of data, let TiDB retry. - SelfRolledBack = 3; // the transaction itself has been rolled back when it tries to prewrite. - RcCheckTs = 4; // RcCheckTs failure by meeting a newer version, let TiDB retry. - LazyUniquenessCheck = 5; // write conflict found in lazy uniqueness check in pessimistic transactions. - } - - uint64 start_ts = 1; - uint64 conflict_ts = 2; - bytes key = 3; - bytes primary = 4; - uint64 conflict_commit_ts = 5; - Reason reason = 6; + enum Reason { + Unknown = 0; + Optimistic = 1; // in optimistic transactions. + PessimisticRetry = 2; // a lock acquisition request waits for a lock and awakes, or meets a newer version of data, let TiDB retry. + SelfRolledBack = 3; // the transaction itself has been rolled back when it tries to prewrite. + RcCheckTs = 4; // RcCheckTs failure by meeting a newer version, let TiDB retry. + LazyUniquenessCheck = 5; // write conflict found when deferring constraint checks in pessimistic transactions. Deprecated in next-gen (cloud-storage-engine). + NotLockedKeyConflict = 6; // write conflict found on keys that do not acquire pessimistic locks in pessimistic transactions. + } + + uint64 start_ts = 1; + uint64 conflict_ts = 2; + bytes key = 3; + bytes primary = 4; + uint64 conflict_commit_ts = 5; + Reason reason = 6; } message AlreadyExist { - bytes key = 1; + bytes key = 1; } message Deadlock { - uint64 lock_ts = 1; - bytes lock_key = 2; - uint64 deadlock_key_hash = 3; - repeated deadlock.WaitForEntry wait_chain = 4; + uint64 lock_ts = 1; + bytes lock_key = 2; + + // The hash of `deadlock_key` field. + uint64 deadlock_key_hash = 3; + repeated deadlock.WaitForEntry wait_chain = 4; + + // The key that the current transaction has already acquired and blocks another transaction to form the deadlock. + bytes deadlock_key = 5; } message CommitTsExpired { - uint64 start_ts = 1; - uint64 attempted_commit_ts = 2; - bytes key = 3; - uint64 min_commit_ts = 4; + uint64 start_ts = 1; + uint64 attempted_commit_ts = 2; + bytes key = 3; + uint64 min_commit_ts = 4; } message TxnNotFound { - uint64 start_ts = 1; - bytes primary_key = 2; + uint64 start_ts = 1; + bytes primary_key = 2; } message CommitTsTooLarge { - uint64 commit_ts = 1; // The calculated commit TS. + uint64 commit_ts = 1; // The calculated commit TS. } message AssertionFailed { - uint64 start_ts = 1; - bytes key = 2; - Assertion assertion = 3; - uint64 existing_start_ts = 4; - uint64 existing_commit_ts = 5; + uint64 start_ts = 1; + bytes key = 2; + Assertion assertion = 3; + uint64 existing_start_ts = 4; + uint64 existing_commit_ts = 5; } message PrimaryMismatch { - LockInfo lock_info = 1; + LockInfo lock_info = 1; +} + +message TxnLockNotFound { + bytes key = 1; +} + +message MvccDebugInfo { + bytes key = 1; + MvccInfo mvcc = 2; +} + +message DebugInfo { + repeated MvccDebugInfo mvcc_info = 1; } enum CommandPri { - Normal = 0; // Normal is the default value. - Low = 1; - High = 2; + Normal = 0; // Normal is the default value. + Low = 1; + High = 2; } enum IsolationLevel { - SI = 0; // SI = snapshot isolation - RC = 1; // RC = read committed - RCCheckTS = 2; // RC read and it's needed to check if there exists more recent versions. + SI = 0; // SI = snapshot isolation + RC = 1; // RC = read committed + RCCheckTS = 2; // RC read and it's needed to check if there exists more recent versions. } // Operation allowed info during each TiKV storage threshold. enum DiskFullOpt { - NotAllowedOnFull = 0; // The default value, means operations are not allowed either under almost full or already full. - AllowedOnAlmostFull = 1; // Means operations will be allowed when disk is almost full. - AllowedOnAlreadyFull = 2; // Means operations will be allowed when disk is already full. + NotAllowedOnFull = 0; // The default value, means operations are not allowed either under almost full or already full. + AllowedOnAlmostFull = 1; // Means operations will be allowed when disk is almost full. + AllowedOnAlreadyFull = 2; // Means operations will be allowed when disk is already full. } message TimeDetail { - // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and - // other kind of waitings in series. (Wait time in the raftstore is not included.) - uint64 wait_wall_time_ms = 1; - // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not - // include `wait_wall_time`. - // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB - // cannot be excluded for now, like Mutex wait time, which is included in this field, so that - // this field is called wall time instead of CPU time. - uint64 process_wall_time_ms = 2; - // KV read wall Time means the time used in key/value scan and get. - uint64 kv_read_wall_time_ms = 3; - // Total wall clock time spent on this RPC in TiKV . - uint64 total_rpc_wall_time_ns = 4; + // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and + // other kind of waitings in series. (Wait time in the raftstore is not included.) + uint64 wait_wall_time_ms = 1; + // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not + // include `wait_wall_time`. + // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB + // cannot be excluded for now, like Mutex wait time, which is included in this field, so that + // this field is called wall time instead of CPU time. + uint64 process_wall_time_ms = 2; + // KV read wall Time means the time used in key/value scan and get. + uint64 kv_read_wall_time_ms = 3; + // Total wall clock time spent on this RPC in TiKV . + uint64 total_rpc_wall_time_ns = 4; } message TimeDetailV2 { - // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and - // other kind of waitings in series. (Wait time in the raftstore is not included.) - uint64 wait_wall_time_ns = 1; - // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not - // include `wait_wall_time` and `suspend_wall_time`. - // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB - // cannot be excluded for now, like Mutex wait time, which is included in this field, so that - // this field is called wall time instead of CPU time. - uint64 process_wall_time_ns = 2; - // Cpu wall time elapsed that task is waiting in queue. - uint64 process_suspend_wall_time_ns = 3; - // KV read wall Time means the time used in key/value scan and get. - uint64 kv_read_wall_time_ns = 4; - // Total wall clock time spent on this RPC in TiKV . - uint64 total_rpc_wall_time_ns = 5; + // Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and + // other kind of waitings in series. (Wait time in the raftstore is not included.) + uint64 wait_wall_time_ns = 1; + // Off-cpu and on-cpu wall time elapsed to actually process the request payload. It does not + // include `wait_wall_time` and `suspend_wall_time`. + // This field is very close to the CPU time in most cases. Some wait time spend in RocksDB + // cannot be excluded for now, like Mutex wait time, which is included in this field, so that + // this field is called wall time instead of CPU time. + uint64 process_wall_time_ns = 2; + // Cpu wall time elapsed that task is waiting in queue. + uint64 process_suspend_wall_time_ns = 3; + // KV read wall Time means the time used in key/value scan and get. + uint64 kv_read_wall_time_ns = 4; + // Total wall clock time spent on this RPC in TiKV . + uint64 total_rpc_wall_time_ns = 5; + // Time spent on the gRPC layer. + uint64 kv_grpc_process_time_ns = 6; + // Time spent on waiting for run again in grpc pool from other executor pool. + uint64 kv_grpc_wait_time_ns = 7; } message ScanInfo { - int64 total = 1; - int64 processed = 2; - int64 read_bytes = 3; + int64 total = 1; + int64 processed = 2; + int64 read_bytes = 3; } // Only reserved for compatibility. message ScanDetail { - ScanInfo write = 1; - ScanInfo lock = 2; - ScanInfo data = 3; + ScanInfo write = 1; + ScanInfo lock = 2; + ScanInfo data = 3; } message ScanDetailV2 { - // Number of user keys scanned from the storage. - // It does not include deleted version or RocksDB tombstone keys. - // For Coprocessor requests, it includes keys that has been filtered out by - // Selection. - uint64 processed_versions = 1; + // Number of user keys scanned from the storage. + // It does not include deleted version or RocksDB tombstone keys. + // For Coprocessor requests, it includes keys that has been filtered out by + // Selection. + uint64 processed_versions = 1; - // Number of bytes of user key-value pairs scanned from the storage, i.e. - // total size of data returned from MVCC layer. - uint64 processed_versions_size = 8; + // Number of bytes of user key-value pairs scanned from the storage, i.e. + // total size of data returned from MVCC layer. + uint64 processed_versions_size = 8; - // Approximate number of MVCC keys meet during scanning. It includes - // deleted versions, but does not include RocksDB tombstone keys. - // - // When this field is notably larger than `processed_versions`, it means - // there are a lot of deleted MVCC keys. - uint64 total_versions = 2; + // Approximate number of MVCC keys meet during scanning. It includes + // deleted versions, but does not include RocksDB tombstone keys. + // + // When this field is notably larger than `processed_versions`, it means + // there are a lot of deleted MVCC keys. + uint64 total_versions = 2; - // Total number of deletes and single deletes skipped over during - // iteration, i.e. how many RocksDB tombstones are skipped. - uint64 rocksdb_delete_skipped_count = 3; + // Total number of deletes and single deletes skipped over during + // iteration, i.e. how many RocksDB tombstones are skipped. + uint64 rocksdb_delete_skipped_count = 3; - // Total number of internal keys skipped over during iteration. - // See https://github.com/facebook/rocksdb/blob/9f1c84ca471d8b1ad7be9f3eebfc2c7e07dfd7a7/include/rocksdb/perf_context.h#L84 for details. - uint64 rocksdb_key_skipped_count = 4; + // Total number of internal keys skipped over during iteration. + // See https://github.com/facebook/rocksdb/blob/9f1c84ca471d8b1ad7be9f3eebfc2c7e07dfd7a7/include/rocksdb/perf_context.h#L84 for details. + uint64 rocksdb_key_skipped_count = 4; - // Total number of RocksDB block cache hits. - uint64 rocksdb_block_cache_hit_count = 5; + // Total number of RocksDB block cache hits. + uint64 rocksdb_block_cache_hit_count = 5; - // Total number of block reads (with IO). - uint64 rocksdb_block_read_count = 6; + // Total number of block reads (with IO). + uint64 rocksdb_block_read_count = 6; - // Total number of bytes from block reads. - uint64 rocksdb_block_read_byte = 7; + // Total number of bytes from block reads. + uint64 rocksdb_block_read_byte = 7; - // Total time used for block reads. - uint64 rocksdb_block_read_nanos = 9; + // Total time used for block reads. + uint64 rocksdb_block_read_nanos = 9; - // Time used for getting a raftstore snapshot (including proposing read index, leader confirmation and getting the RocksDB snapshot). - uint64 get_snapshot_nanos = 10; + // Time used for getting a raftstore snapshot (including proposing read index, leader confirmation and getting the RocksDB snapshot). + uint64 get_snapshot_nanos = 10; - // Time used for proposing read index from read pool to store pool, equals 0 when performing lease read. - uint64 read_index_propose_wait_nanos = 11; + // Time used for proposing read index from read pool to store pool, equals 0 when performing lease read. + uint64 read_index_propose_wait_nanos = 11; - // Time used for leader confirmation, equals 0 when performing lease read. - uint64 read_index_confirm_wait_nanos = 12; + // Time used for leader confirmation, equals 0 when performing lease read. + uint64 read_index_confirm_wait_nanos = 12; - // Time used for read pool scheduling. - uint64 read_pool_schedule_wait_nanos = 13; + // Time used for read pool scheduling. + uint64 read_pool_schedule_wait_nanos = 13; + + // The total MVCC key-value size meet during scanning. It includes + // deleted versions, but does not include RocksDB tombstone keys. + // + // When this field is notably larger than `processed_versions_size`, it means + // there are a lot of deleted MVCC keys. + uint64 total_versions_size = 14; + + // Total number of IA segment cache hits for this request. + uint64 ia_cache_hit_count = 15; + + // Total number of IA remote segment reads for this request. + // + // IA segments are only read on cache misses, so this is also the IA segment + // cache miss count for the request. + uint64 ia_remote_read_segment_count = 16; + + // Total number of logical bytes returned from IA remote segment reads for this + // request. + uint64 ia_remote_read_segment_bytes = 17; + + // Total time spent serving IA remote segment reads for this request. + uint64 ia_remote_read_segment_nanos = 18; } message ExecDetails { - // Available when ctx.record_time_stat = true or meet slow query. - TimeDetail time_detail = 1; + // Available when ctx.record_time_stat = true or meet slow query. + TimeDetail time_detail = 1; - // Available when ctx.record_scan_stat = true or meet slow query. - ScanDetail scan_detail = 2; + // Available when ctx.record_scan_stat = true or meet slow query. + ScanDetail scan_detail = 2; - // See https://github.com/pingcap/kvproto/pull/689 - reserved 3; - reserved 4; + // See https://github.com/pingcap/kvproto/pull/689 + reserved 3; + reserved 4; } message ExecDetailsV2 { - // Available when ctx.record_time_stat = true or meet slow query. - // deprecated. Should use `time_detail_v2` instead. - TimeDetail time_detail = 1; - - // Available when ctx.record_scan_stat = true or meet slow query. - ScanDetailV2 scan_detail_v2 = 2; - - // Raftstore writing durations of the request. Only available for some write requests. - WriteDetail write_detail = 3; - // Available when ctx.record_time_stat = true or meet slow query. - TimeDetailV2 time_detail_v2 = 4; + // Available when ctx.record_time_stat = true or meet slow query. + // deprecated. Should use `time_detail_v2` instead. + TimeDetail time_detail = 1; + + // Available when ctx.record_scan_stat = true or meet slow query. + ScanDetailV2 scan_detail_v2 = 2; + + // Raftstore writing durations of the request. Only available for some write requests. + WriteDetail write_detail = 3; + // Available when ctx.record_time_stat = true or meet slow query. + TimeDetailV2 time_detail_v2 = 4; + + // RU (Request Unit) consumption details. + RUV2 ru_v2 = 5; +} + +message RUV2 { + uint64 kv_engine_cache_miss = 1; + ExecutorInputs executor_inputs = 2; + uint64 coprocessor_executor_iterations = 3; + uint64 coprocessor_response_bytes = 4; + uint64 raftstore_store_write_trigger_wb_bytes = 5; + uint64 storage_processed_keys_batch_get = 6; + uint64 storage_processed_keys_get = 7; + // Filled by client-go. + uint64 read_rpc_count = 8; + // Filled by client-go. + uint64 write_rpc_count = 9; +} + +message ExecutorInputs { + uint64 tikv_coprocessor_executor_work_total_batch_index_scan = 1; + uint64 tikv_coprocessor_executor_work_total_batch_table_scan = 2; + uint64 tikv_coprocessor_executor_work_total_batch_selection = 3; + uint64 tikv_coprocessor_executor_work_total_batch_top_n = 4; + uint64 tikv_coprocessor_executor_work_total_batch_limit = 5; + uint64 tikv_coprocessor_executor_work_total_batch_simple_aggr = 6; + uint64 tikv_coprocessor_executor_work_total_batch_fast_hash_aggr = 7; } message WriteDetail { - // Wait duration in the store loop. - uint64 store_batch_wait_nanos = 1; + // Wait duration in the store loop. + uint64 store_batch_wait_nanos = 1; - // Wait duration before sending proposal to peers. - uint64 propose_send_wait_nanos = 2; + // Wait duration before sending proposal to peers. + uint64 propose_send_wait_nanos = 2; - // Total time spent on persisting the log. - uint64 persist_log_nanos = 3; + // Total time spent on persisting the log. + uint64 persist_log_nanos = 3; - // Wait time until the Raft log write leader begins to write. - uint64 raft_db_write_leader_wait_nanos = 4; + // Wait time until the Raft log write leader begins to write. + uint64 raft_db_write_leader_wait_nanos = 4; - // Time spent on synchronizing the Raft log to the disk. - uint64 raft_db_sync_log_nanos = 5; + // Time spent on synchronizing the Raft log to the disk. + uint64 raft_db_sync_log_nanos = 5; - // Time spent on writing the Raft log to the Raft memtable. - uint64 raft_db_write_memtable_nanos = 6; + // Time spent on writing the Raft log to the Raft memtable. + uint64 raft_db_write_memtable_nanos = 6; - // Time waiting for peers to confirm the proposal (counting from the instant when the leader sends the proposal message). - uint64 commit_log_nanos = 7; + // Time waiting for peers to confirm the proposal (counting from the instant when the leader sends the proposal message). + uint64 commit_log_nanos = 7; - // Wait duration in the apply loop. - uint64 apply_batch_wait_nanos = 8; + // Wait duration in the apply loop. + uint64 apply_batch_wait_nanos = 8; - // Total time spend to applying the log. - uint64 apply_log_nanos = 9; + // Total time spend to applying the log. + uint64 apply_log_nanos = 9; - // Wait time until the KV RocksDB lock is acquired. - uint64 apply_mutex_lock_nanos = 10; + // Wait time until the KV RocksDB lock is acquired. + uint64 apply_mutex_lock_nanos = 10; - // Wait time until becoming the KV RocksDB write leader. - uint64 apply_write_leader_wait_nanos = 11; + // Wait time until becoming the KV RocksDB write leader. + uint64 apply_write_leader_wait_nanos = 11; - // Time spent on writing the KV DB WAL to the disk. - uint64 apply_write_wal_nanos = 12; + // Time spent on writing the KV DB WAL to the disk. + uint64 apply_write_wal_nanos = 12; - // Time spent on writing to the memtable of the KV RocksDB. - uint64 apply_write_memtable_nanos = 13; + // Time spent on writing to the memtable of the KV RocksDB. + uint64 apply_write_memtable_nanos = 13; - // Time spent on waiting in the latch. - uint64 latch_wait_nanos = 14; + // Time spent on waiting in the latch. + uint64 latch_wait_nanos = 14; - // Processing time in the transaction layer. - uint64 process_nanos = 15; + // Processing time in the transaction layer. + uint64 process_nanos = 15; - // Wait time because of the scheduler flow control or quota limiter throttling. - uint64 throttle_nanos = 16; + // Wait time because of the scheduler flow control or quota limiter throttling. + uint64 throttle_nanos = 16; - // Wait time in the waiter manager for pessimistic locking. - uint64 pessimistic_lock_wait_nanos = 17; + // Wait time in the waiter manager for pessimistic locking. + uint64 pessimistic_lock_wait_nanos = 17; } message KvPair { - KeyError error = 1; - bytes key = 2; - bytes value = 3; + KeyError error = 1; + bytes key = 2; + bytes value = 3; + // The commit timestamp of the key. + // If it is zero, it means the commit timestamp is unknown. + uint64 commit_ts = 4; } enum Op { - Put = 0; - Del = 1; - Lock = 2; - Rollback = 3; - // insert operation has a constraint that key should not exist before. - Insert = 4; - PessimisticLock = 5; - CheckNotExists = 6; + Put = 0; + Del = 1; + Lock = 2; + Rollback = 3; + // insert operation has a constraint that key should not exist before. + Insert = 4; + // PessimisticLock is exclusive lock acquired in pessimistic transaction. + PessimisticLock = 5; + CheckNotExists = 6; + // SharedLock likes Lock but in shared mode. + SharedLock = 7; + // SharedPessimisticLock is shared lock acquired in pessimistic transaction. + SharedPessimisticLock = 8; } enum Assertion { - None = 0; - Exist = 1; - NotExist = 2; + None = 0; + Exist = 1; + NotExist = 2; } enum AssertionLevel { - // No assertion. - Off = 0; - // Assertion is enabled, but not enforced when it might affect performance. - Fast = 1; - // Assertion is enabled and enforced. - Strict = 2; + // No assertion. + Off = 0; + // Assertion is enabled, but not enforced when it might affect performance. + Fast = 1; + // Assertion is enabled and enforced. + Strict = 2; } message Mutation { - Op op = 1; - bytes key = 2; - bytes value = 3; - Assertion assertion = 4; + Op op = 1; + bytes key = 2; + bytes value = 3; + Assertion assertion = 4; } message MvccWrite { - Op type = 1; - uint64 start_ts = 2; - uint64 commit_ts = 3; - bytes short_value = 4; - bool has_overlapped_rollback = 5; - bool has_gc_fence = 6; - uint64 gc_fence = 7; - uint64 last_change_ts = 8; - uint64 versions_to_last_change = 9; + Op type = 1; + uint64 start_ts = 2; + uint64 commit_ts = 3; + bytes short_value = 4; + bool has_overlapped_rollback = 5; + bool has_gc_fence = 6; + uint64 gc_fence = 7; + uint64 last_change_ts = 8; + uint64 versions_to_last_change = 9; } message MvccValue { - uint64 start_ts = 1; - bytes value = 2; + uint64 start_ts = 1; + bytes value = 2; } message MvccLock { - Op type = 1; - uint64 start_ts = 2; - bytes primary = 3; - bytes short_value = 4; - uint64 ttl = 5; - uint64 for_update_ts = 6; - uint64 txn_size = 7; - bool use_async_commit = 8; - repeated bytes secondaries = 9; - repeated uint64 rollback_ts = 10; - uint64 last_change_ts = 11; - uint64 versions_to_last_change = 12; + Op type = 1; + uint64 start_ts = 2; + bytes primary = 3; + bytes short_value = 4; + uint64 ttl = 5; + uint64 for_update_ts = 6; + uint64 txn_size = 7; + bool use_async_commit = 8; + repeated bytes secondaries = 9; + repeated uint64 rollback_ts = 10; + uint64 last_change_ts = 11; + uint64 versions_to_last_change = 12; } message MvccInfo { - MvccLock lock = 1; - repeated MvccWrite writes = 2; - repeated MvccValue values = 3; + MvccLock lock = 1; + repeated MvccWrite writes = 2; + repeated MvccValue values = 3; } message TxnInfo { - uint64 txn = 1; - uint64 status = 2; + uint64 txn = 1; + uint64 status = 2; - // Reserved for file based transaction. - bool is_txn_file = 100; + // Reserved for file based transaction. + bool is_txn_file = 100; } enum Action { - NoAction = 0; - TTLExpireRollback = 1; - LockNotExistRollback = 2; - MinCommitTSPushed = 3; - TTLExpirePessimisticRollback = 4; - LockNotExistDoNothing = 5; + NoAction = 0; + TTLExpireRollback = 1; + LockNotExistRollback = 2; + MinCommitTSPushed = 3; + TTLExpirePessimisticRollback = 4; + LockNotExistDoNothing = 5; } message KeyRange { - bytes start_key = 1; - bytes end_key = 2; + bytes start_key = 1; + bytes end_key = 2; } enum ExtraOp { - Noop = 0; - // ReadOldValue represents to output the previous value for delete/update operations. - ReadOldValue = 1; + Noop = 0; + // ReadOldValue represents to output the previous value for delete/update operations. + ReadOldValue = 1; } message LeaderInfo { - uint64 region_id = 1; - uint64 peer_id = 2; - uint64 term = 3; - metapb.RegionEpoch region_epoch = 4; - ReadState read_state = 5; + uint64 region_id = 1; + uint64 peer_id = 2; + uint64 term = 3; + metapb.RegionEpoch region_epoch = 4; + ReadState read_state = 5; } message ReadState { - uint64 applied_index = 1; - uint64 safe_ts = 2; + uint64 applied_index = 1; + uint64 safe_ts = 2; } message CheckLeaderRequest { - repeated LeaderInfo regions = 1; - uint64 ts = 2; + repeated LeaderInfo regions = 1; + uint64 ts = 2; } message CheckLeaderResponse { - repeated uint64 regions = 1; - uint64 ts = 2; + repeated uint64 regions = 1; + uint64 ts = 2; } message StoreSafeTSRequest { - // Get the minimal `safe_ts` from regions that overlap with the key range [`start_key`, `end_key`) - // An empty key range means all regions in the store - KeyRange key_range = 1; + // Get the minimal `safe_ts` from regions that overlap with the key range [`start_key`, `end_key`) + // An empty key range means all regions in the store + KeyRange key_range = 1; } message StoreSafeTSResponse { - uint64 safe_ts = 1; + uint64 safe_ts = 1; } message RawGetKeyTTLRequest { - Context context = 1; - bytes key = 2; - string cf = 3; + Context context = 1; + bytes key = 2; + string cf = 3; } message RawGetKeyTTLResponse { - errorpb.Error region_error = 1; - string error = 2; - uint64 ttl = 3; - bool not_found = 4; + errorpb.Error region_error = 1; + string error = 2; + uint64 ttl = 3; + bool not_found = 4; } message RawCASRequest { - Context context = 1; - bytes key = 2; - bytes value = 3; - bool previous_not_exist = 4; - bytes previous_value = 5; - string cf = 6; - uint64 ttl = 7; + Context context = 1; + bytes key = 2; + bytes value = 3; + bool previous_not_exist = 4; + bytes previous_value = 5; + string cf = 6; + uint64 ttl = 7; + // If `delete` is true, the request is to delete the key when the comparison succeeds + bool delete = 8; } message RawCASResponse { - errorpb.Error region_error = 1; - string error = 2; - bool succeed = 3; - // The previous value regardless of whether the comparison is succeed. - bool previous_not_exist = 4; - bytes previous_value = 5; + errorpb.Error region_error = 1; + string error = 2; + bool succeed = 3; + // The previous value regardless of whether the comparison is succeed. + bool previous_not_exist = 4; + bytes previous_value = 5; } message GetLockWaitInfoRequest { - Context context = 1; - // TODO: There may need some filter options to be used on conditional querying, e.g., finding - // the lock waiting status for some specified transaction. + Context context = 1; + // TODO: There may need some filter options to be used on conditional querying, e.g., finding + // the lock waiting status for some specified transaction. } message GetLockWaitInfoResponse { - errorpb.Error region_error = 1; - string error = 2; - repeated deadlock.WaitForEntry entries = 3; + errorpb.Error region_error = 1; + string error = 2; + repeated deadlock.WaitForEntry entries = 3; } message GetLockWaitHistoryRequest { - Context context = 1; - // TODO: There may need some filter options to be used on conditional querying, e.g., finding - // the lock waiting status for some specified transaction. + Context context = 1; + // TODO: There may need some filter options to be used on conditional querying, e.g., finding + // the lock waiting status for some specified transaction. } message GetLockWaitHistoryResponse { - errorpb.Error region_error = 1; - string error = 2; - repeated deadlock.WaitForEntry entries = 3; + errorpb.Error region_error = 1; + string error = 2; + repeated deadlock.WaitForEntry entries = 3; } message RawCoprocessorRequest { - kvrpcpb.Context context = 1; + kvrpcpb.Context context = 1; - string copr_name = 2; - // Coprorcessor version constraint following SEMVER definition. - string copr_version_req = 3; + string copr_name = 2; + // Coprorcessor version constraint following SEMVER definition. + string copr_version_req = 3; - repeated KeyRange ranges = 4; - bytes data = 5; + repeated KeyRange ranges = 4; + bytes data = 5; } message RawCoprocessorResponse { - errorpb.Error region_error = 1; - // Error message for cases like if no coprocessor with a matching name is found - // or on a version mismatch between plugin_api and the coprocessor. - string error = 2; - bytes data = 3; + errorpb.Error region_error = 1; + // Error message for cases like if no coprocessor with a matching name is found + // or on a version mismatch between plugin_api and the coprocessor. + string error = 2; + bytes data = 3; } enum ChecksumAlgorithm { - Crc64_Xor = 0; + Crc64_Xor = 0; } message RawChecksumRequest { - Context context = 1; - ChecksumAlgorithm algorithm = 2; - repeated KeyRange ranges = 3; + Context context = 1; + ChecksumAlgorithm algorithm = 2; + repeated KeyRange ranges = 3; } message RawChecksumResponse { - errorpb.Error region_error = 1; - string error = 2; - uint64 checksum = 3; - uint64 total_kvs = 4; - uint64 total_bytes = 5; + errorpb.Error region_error = 1; + string error = 2; + uint64 checksum = 3; + uint64 total_kvs = 4; + uint64 total_bytes = 5; } message CompactError { - oneof error { - CompactErrorInvalidStartKey err_invalid_start_key = 1; - CompactErrorPhysicalTableNotExist err_physical_table_not_exist = 2; - CompactErrorCompactInProgress err_compact_in_progress = 3; - CompactErrorTooManyPendingTasks err_too_many_pending_tasks = 4; - } + oneof error { + CompactErrorInvalidStartKey err_invalid_start_key = 1; + CompactErrorPhysicalTableNotExist err_physical_table_not_exist = 2; + CompactErrorCompactInProgress err_compact_in_progress = 3; + CompactErrorTooManyPendingTasks err_too_many_pending_tasks = 4; + } } message CompactErrorInvalidStartKey {} @@ -1510,47 +1679,132 @@ message CompactErrorCompactInProgress {} message CompactErrorTooManyPendingTasks {} message CompactRequest { - // If specified, the compaction will start from this start key. - // If unspecified, the compaction will start from beginning. - // NOTE 1: The start key should be never manually constructed. You should always use a key - // returned in CompactResponse. - // NOTE 2: the compaction range will be always restricted by physical_table_id. - bytes start_key = 1; - - // The physical table that will be compacted. - // - // TODO: this is information that TiKV doesn't need to know. - // See https://github.com/pingcap/kvproto/issues/912 - int64 physical_table_id = 2; - - // The logical table id of the compaction. When receiving parallel requests with the same - // logical table id, err_compact_in_progress will be returned. - // - // TODO: this is information that TiKV doesn't need to know. - // See https://github.com/pingcap/kvproto/issues/912 - int64 logical_table_id = 3; - - // API version of the request - APIVersion api_version = 7; - // Keyspace of the table located in. - uint32 keyspace_id = 8; + // If specified, the compaction will start from this start key. + // If unspecified, the compaction will start from beginning. + // NOTE 1: The start key should be never manually constructed. You should always use a key + // returned in CompactResponse. + // NOTE 2: the compaction range will be always restricted by physical_table_id. + bytes start_key = 1; + + // The physical table that will be compacted. + // + // TODO: this is information that TiKV doesn't need to know. + // See https://github.com/pingcap/kvproto/issues/912 + int64 physical_table_id = 2; + + // The logical table id of the compaction. When receiving parallel requests with the same + // logical table id, err_compact_in_progress will be returned. + // + // TODO: this is information that TiKV doesn't need to know. + // See https://github.com/pingcap/kvproto/issues/912 + int64 logical_table_id = 3; + + // API version of the request + APIVersion api_version = 7; + // Keyspace of the table located in. + uint32 keyspace_id = 8; } message CompactResponse { - CompactError error = 1; + CompactError error = 1; - // The compaction is done incrementally. If there are more data to compact, this field - // will be set. The client can request to compact more data according to the `compacted_end_key`. - bool has_remaining = 2; + // The compaction is done incrementally. If there are more data to compact, this field + // will be set. The client can request to compact more data according to the `compacted_end_key`. + bool has_remaining = 2; - bytes compacted_start_key = 3; - bytes compacted_end_key = 4; + bytes compacted_start_key = 3; + bytes compacted_end_key = 4; } message TiFlashSystemTableRequest { - string sql = 1; + string sql = 1; } message TiFlashSystemTableResponse { - bytes data = 1 [(gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", (gogoproto.nullable) = false]; -} + bytes data = 1 [ + (gogoproto.customtype) = "github.com/pingcap/kvproto/pkg/sharedbytes.SharedBytes", + (gogoproto.nullable) = false + ]; +} + +// Flush is introduced from the pipelined DML protocol. +// A Flush request writes some keys and values to TiKV, storing in LOCK and DEFAULT CF, just like a Prewrite request. +message FlushRequest { + Context context = 1; + repeated Mutation mutations = 2; + bytes primary_key = 3; + uint64 start_ts = 4; + uint64 min_commit_ts = 5; + // generation of the flush request. It is a monotonically increasing number in each transaction. + uint64 generation = 6; + uint64 lock_ttl = 7; + AssertionLevel assertion_level = 8; +} + +message FlushResponse { + errorpb.Error region_error = 1; + repeated KeyError errors = 2; + ExecDetailsV2 exec_details_v2 = 3; +} + +// BufferBatchGet is introduced from the pipelined DML protocol. +// It is similar to a BatchGet request, except that it can only read the data that has been flushed by itself. +message BufferBatchGetRequest { + Context context = 1; + repeated bytes keys = 2; + uint64 version = 3; +} + +message BufferBatchGetResponse { + errorpb.Error region_error = 1; + KeyError error = 2; + repeated KvPair pairs = 3; + // Time and scan details when processing the request. + ExecDetailsV2 exec_details_v2 = 4; +} + +// Actively request TiKV to report health feedback information. TiKV won't omit the health feedback information when sending the +// `BatchCommandsResponse` that contains this response. +// The health feedback information won't be replied in the response, but will be attached to `BatchCommandsResponse.health_feedback` field as usual. +// Only works when batch RPC is enabled. +message GetHealthFeedbackRequest { + Context context = 1; +} + +message GetHealthFeedbackResponse { + // The error field is added for keeping consistent. This request won't meet any region error as it's store level rather than region level. + errorpb.Error region_error = 1; + HealthFeedback health_feedback = 2; +} + +message HealthFeedback { + uint64 store_id = 1; + // The sequence number of the feedback message. + // It's defined as an incrementing integer, starting from the unix timestamp (milliseconds) at + // the time that the TiKV node is started. + // This can be useful for filtering out out-of-order feedback messages. + // Note that considering the possibility of system clock changing, this field doesn't guarantee + // uniqueness and monotonic if the TiKV node is restarted. + uint64 feedback_seq_no = 2; + // The slow_score calculated in raftstore module. Due to some limitations of slow score, this would + // be replaced by `SlowTrend` in the future. + int32 slow_score = 3; +} + +message BroadcastTxnStatusRequest { + Context context = 1; + repeated TxnStatus txn_status = 2; +} + +message TxnStatus { + uint64 start_ts = 1; + // a non-zero min_commit_ts indicates the transaction is ongoing + uint64 min_commit_ts = 2; + // a non-zero commit_ts indicates the transaction is committed + uint64 commit_ts = 3; + bool rolled_back = 4; + // The txn has unlocked all keys, implying that it can be removed from txn_status_cache. + bool is_completed = 5; +} + +message BroadcastTxnStatusResponse {} diff --git a/proto/logbackuppb.proto b/proto/logbackuppb.proto index 17e9820f..ff92667c 100644 --- a/proto/logbackuppb.proto +++ b/proto/logbackuppb.proto @@ -1,58 +1,73 @@ syntax = "proto3"; package logbackup; +import "errorpb.proto"; import "gogoproto/gogo.proto"; import "rustproto.proto"; -import "errorpb.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // The minimal information for identify a region. message RegionIdentity { - uint64 id = 1; - uint64 epoch_version = 2; - // We omitted epoch_conf_version because config change won't make range change. + uint64 id = 1; + uint64 epoch_version = 2; + // We omitted epoch_conf_version because config change won't make range change. } // The last flush ts with region information. message RegionCheckpoint { - errorpb.Error err = 1; - RegionIdentity region = 2; - uint64 checkpoint = 3; + errorpb.Error err = 1; + RegionIdentity region = 2; + uint64 checkpoint = 3; } message GetLastFlushTSOfRegionRequest { - repeated RegionIdentity regions = 1; + repeated RegionIdentity regions = 1; } message GetLastFlushTSOfRegionResponse { - repeated RegionCheckpoint checkpoints = 1; + repeated RegionCheckpoint checkpoints = 1; } message SubscribeFlushEventRequest { - string client_id = 1; + string client_id = 1; } message SubscribeFlushEventResponse { - repeated FlushEvent events = 1; + repeated FlushEvent events = 1; } message FlushEvent { - bytes start_key = 1; - bytes end_key = 2; - uint64 checkpoint = 3; + bytes start_key = 1; + bytes end_key = 2; + uint64 checkpoint = 3; +} + +message FlushNowRequest {} + +message FlushNowResponse { + repeated FlushResult results = 1; +} + +message FlushResult { + string task_name = 1; + bool success = 2; + string error_message = 3; } // The log backup service. -// Generally, most essential interfaces of log backup (say, checkpoint management, task management) are +// Generally, most essential interfaces of log backup (say, checkpoint management, task management) are // provided by adding some key in the embed etcd of PD. // This interface is mainly provided for the checkpoint advancer and debug usage. service LogBackup { - rpc GetLastFlushTSOfRegion(GetLastFlushTSOfRegionRequest) returns (GetLastFlushTSOfRegionResponse) {} - rpc SubscribeFlushEvent(SubscribeFlushEventRequest) returns (stream SubscribeFlushEventResponse) {} + rpc GetLastFlushTSOfRegion(GetLastFlushTSOfRegionRequest) returns (GetLastFlushTSOfRegionResponse) {} + rpc SubscribeFlushEvent(SubscribeFlushEventRequest) returns (stream SubscribeFlushEventResponse) {} + rpc FlushNow(FlushNowRequest) returns (FlushNowResponse) {} } diff --git a/proto/meta_storagepb.proto b/proto/meta_storagepb.proto index bae7ae57..40a8d2aa 100644 --- a/proto/meta_storagepb.proto +++ b/proto/meta_storagepb.proto @@ -4,96 +4,125 @@ package meta_storagepb; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // MetaStorage is the meta storage service. service MetaStorage { - rpc Watch(WatchRequest) returns (stream WatchResponse) {} + rpc Watch(WatchRequest) returns (stream WatchResponse) {} + + // Get is the same as etcd Range which might be implemented in a more common way + // so that we can use other storages to replace etcd in the future. + rpc Get(GetRequest) returns (GetResponse) {} - // Get is the same as etcd Range which might be implemented in a more common way - // so that we can use other storages to replace etcd in the future. - rpc Get(GetRequest) returns (GetResponse) {} + rpc Put(PutRequest) returns (PutResponse) {} - rpc Put(PutRequest) returns (PutResponse) {} + // Delete is the same as etcd Range which might be implemented in a more common way + // so that we can use other storages to replace etcd in the future. + rpc Delete(DeleteRequest) returns (DeleteResponse) {} } enum ErrorType { - OK = 0; - UNKNOWN = 1; - // required watch revision is smaller than current compact/min revision. - DATA_COMPACTED = 2; + OK = 0; + UNKNOWN = 1; + // required watch revision is smaller than current compact/min revision. + DATA_COMPACTED = 2; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // source is the source of the request. - string source = 2; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // source is the source of the request. + string source = 2; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; - int64 revision = 3; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; + int64 revision = 3; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message WatchRequest { - RequestHeader header = 1; - bytes key = 2; - bytes range_end = 3; - int64 start_revision = 4; - bool prev_kv = 5; + RequestHeader header = 1; + bytes key = 2; + bytes range_end = 3; + int64 start_revision = 4; + bool prev_kv = 5; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message WatchResponse { - ResponseHeader header = 1; - int64 compact_revision = 2; - repeated Event events = 3; + ResponseHeader header = 1; + int64 compact_revision = 2; + repeated Event events = 3; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message GetRequest { - RequestHeader header = 1; - bytes key = 2; - bytes range_end = 3; - int64 limit = 4; - int64 revision = 5; + RequestHeader header = 1; + bytes key = 2; + bytes range_end = 3; + int64 limit = 4; + int64 revision = 5; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message GetResponse { - ResponseHeader header = 1; - repeated KeyValue kvs = 2; - bool more = 3; - int64 count = 4; + ResponseHeader header = 1; + repeated KeyValue kvs = 2; + bool more = 3; + int64 count = 4; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message PutRequest { - RequestHeader header = 1; - bytes key = 2; - bytes value = 3; - int64 lease = 4; - bool prev_kv = 5; + RequestHeader header = 1; + bytes key = 2; + bytes value = 3; + int64 lease = 4; + bool prev_kv = 5; } // copied part of https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/etcdserverpb/rpc.proto message PutResponse { - ResponseHeader header = 1; - KeyValue prev_kv = 2; + ResponseHeader header = 1; + KeyValue prev_kv = 2; +} + +// copied from etcd https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/mvccpb/kv.proto +message DeleteRequest { + RequestHeader header = 1; + bytes key = 2; + // range_end is the key following the last key to delete for the range [key, range_end). + // If range_end is not given, the range is defined to contain only the key argument. + // If range_end is one bit larger than the given key, then the range is all the keys + // with the prefix (the given key). + // If range_end is '\0', the range is all keys greater than or equal to the key argument. + bytes range_end = 3; + + // If prev_kv is set, etcd gets the previous key-value pairs before deleting it. + // The previous key-value pairs will be returned in the delete response. + bool prev_kv = 4; +} + +message DeleteResponse { + ResponseHeader header = 1; + // deleted is the number of keys deleted by the delete range request. + int64 deleted = 2; + repeated KeyValue prev_kvs = 3; } // copied from etcd https://github.com/etcd-io/etcd/blob/7dfd29b0cc7ce25337276dce646ca2a65aa44b4d/api/mvccpb/kv.proto diff --git a/proto/metapb.proto b/proto/metapb.proto index ef3c8d45..4e38559e 100644 --- a/proto/metapb.proto +++ b/proto/metapb.proto @@ -5,25 +5,27 @@ import "encryptionpb.proto"; import "gogoproto/gogo.proto"; import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - message Cluster { - uint64 id = 1; - // max peer count for a region. - // pd will do the auto-balance if region peer count mismatches. - uint32 max_peer_count = 2; - // more attributes...... + uint64 id = 1; + // max peer count for a region. + // pd will do the auto-balance if region peer count mismatches. + uint32 max_peer_count = 2; + // more attributes...... } enum StoreState { - Up = 0; - Offline = 1; - Tombstone = 2; + Up = 0; + Offline = 1; + Tombstone = 2; } // NodeState is going to replace StoreState to make the state concept more clear. @@ -31,122 +33,129 @@ enum StoreState { // "Removing" is just like previous `Offline` which is more accurate. // "Removed" has the same meaning with `Tombstone`. enum NodeState { - Preparing = 0; - Serving = 1; - Removing = 2; - Removed = 3; + Preparing = 0; + Serving = 1; + Removing = 2; + Removed = 3; } // Case insensitive key/value for replica constraints. message StoreLabel { - string key = 1; - string value = 2; + string key = 1; + string value = 2; } message Store { - uint64 id = 1; - // Address to handle client requests (kv, cop, etc.) - string address = 2; - StoreState state = 3; - repeated StoreLabel labels = 4; - string version = 5; - // Address to handle peer requests (raft messages from other store). - // Empty means same as address. - string peer_address = 6; - // Status address provides the HTTP service for external components - string status_address = 7; - string git_hash = 8; - // The start timestamp of the current store - int64 start_timestamp = 9; - string deploy_path = 10; - // The last heartbeat timestamp of the store. - int64 last_heartbeat = 11; - // If the store is physically destroyed, which means it can never up again. - bool physically_destroyed = 12; - // NodeState is used to replace StoreState which will be deprecated in the future. - NodeState node_state = 13; + uint64 id = 1; + // Address to handle client requests (kv, cop, etc.) + string address = 2; + StoreState state = 3; + repeated StoreLabel labels = 4; + string version = 5; + // Address to handle peer requests (raft messages from other store). + // Empty means same as address. + string peer_address = 6; + // Status address provides the HTTP service for external components + string status_address = 7; + string git_hash = 8; + // The start timestamp of the current store + int64 start_timestamp = 9; + string deploy_path = 10; + // The last heartbeat timestamp of the store. + int64 last_heartbeat = 11; + // If the store is physically destroyed, which means it can never up again. + bool physically_destroyed = 12; + // NodeState is used to replace StoreState which will be deprecated in the future. + NodeState node_state = 13; } message RegionEpoch { - // Conf change version, auto increment when add or remove peer - uint64 conf_ver = 1; - // Region version, auto increment when split or merge - uint64 version = 2; + // Conf change version, auto increment when add or remove peer + uint64 conf_ver = 1; + // Region version, auto increment when split or merge + uint64 version = 2; } message BucketStats { - // total read in bytes of each bucket - repeated uint64 read_bytes = 1; + // total read in bytes of each bucket + repeated uint64 read_bytes = 1; - // total write in bytes of each bucket - repeated uint64 write_bytes = 2; + // total write in bytes of each bucket + repeated uint64 write_bytes = 2; - // total read qps of each bucket - repeated uint64 read_qps = 3; + // total read qps of each bucket + repeated uint64 read_qps = 3; - // total write qps of each bucket - repeated uint64 write_qps = 4; + // total write qps of each bucket + repeated uint64 write_qps = 4; - // total read keys of each bucket - repeated uint64 read_keys = 5; + // total read keys of each bucket + repeated uint64 read_keys = 5; - // total write keys of each bucket - repeated uint64 write_keys = 6; + // total write keys of each bucket + repeated uint64 write_keys = 6; } message Buckets { - uint64 region_id = 1; + uint64 region_id = 1; - // A hint indicate if keys have changed. - uint64 version = 2; + // A hint indicate if keys have changed. + uint64 version = 2; - // keys of buckets, include start/end key of region - repeated bytes keys = 3; + // keys of buckets, include start/end key of region + repeated bytes keys = 3; - // bucket stats - BucketStats stats = 4; + // bucket stats + BucketStats stats = 4; + + // The period in milliseconds that stats are collected with in + uint64 period_in_ms = 5; +} - // The period in milliseconds that stats are collected with in - uint64 period_in_ms = 5; +message BucketMeta { + // A hint indicate if keys have changed. + uint64 version = 1; + // keys of buckets, include start/end key of region + repeated bytes keys = 2; } message Region { - uint64 id = 1; - // Region key range [start_key, end_key). - bytes start_key = 2; - bytes end_key = 3; - RegionEpoch region_epoch = 4; - repeated Peer peers = 5; - // Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key. - // IV for end_key is calculated from (encryption_meta.iv + len(start_key)). - // The field is only used by PD and should be ignored otherwise. - // If encryption_meta is empty (i.e. nil), it means start_key and end_key are unencrypted. - encryptionpb.EncryptionMeta encryption_meta = 6; - // The flashback state indicates whether this region is in the flashback state. - // TODO: only check by `flashback_start_ts` in the future. Keep for compatibility now. - bool is_in_flashback = 7; - // The start_ts that the current flashback progress is using. - uint64 flashback_start_ts = 8; + uint64 id = 1; + // Region key range [start_key, end_key). + bytes start_key = 2; + bytes end_key = 3; + RegionEpoch region_epoch = 4; + repeated Peer peers = 5; + // Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key. + // IV for end_key is calculated from (encryption_meta.iv + len(start_key)). + // The field is only used by PD and should be ignored otherwise. + // If encryption_meta is empty (i.e. nil), it means start_key and end_key are unencrypted. + encryptionpb.EncryptionMeta encryption_meta = 6; + // The flashback state indicates whether this region is in the flashback state. + // TODO: only check by `flashback_start_ts` in the future. Keep for compatibility now. + bool is_in_flashback = 7; + // The start_ts that the current flashback progress is using. + uint64 flashback_start_ts = 8; } enum PeerRole { - // Voter -> Voter - Voter = 0; - // Learner/None -> Learner - Learner = 1; - // Learner/None -> Voter - IncomingVoter = 2; - // Voter -> Learner - DemotingVoter = 3; - // We forbid Voter -> None, it can introduce unavailability as discussed in - // etcd-io/etcd#7625 - // Learner -> None can be apply directly, doesn't need to be stored as - // joint state. + // Voter -> Voter + Voter = 0; + // Learner/None -> Learner + Learner = 1; + // Learner/None -> Voter + IncomingVoter = 2; + // Voter -> Learner + DemotingVoter = 3; + // We forbid Voter -> None, it can introduce unavailability as discussed in + // etcd-io/etcd#7625 + // Learner -> None can be apply directly, doesn't need to be stored as + // joint state. } message Peer { - uint64 id = 1; - uint64 store_id = 2; - PeerRole role = 3; - bool is_witness = 4; + uint64 id = 1; + uint64 store_id = 2; + PeerRole role = 3; + bool is_witness = 4; } diff --git a/proto/mpp.proto b/proto/mpp.proto index 268e218d..1035ac00 100644 --- a/proto/mpp.proto +++ b/proto/mpp.proto @@ -1,103 +1,110 @@ syntax = "proto3"; package mpp; -import "gogoproto/gogo.proto"; import "coprocessor.proto"; -import "metapb.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; +import "metapb.proto"; +import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; - -option java_package = "org.tikv.kvproto"; +option (rustproto.lite_runtime_all) = true; // TaskMeta contains meta of a mpp plan, including query's ts and task address. message TaskMeta { - uint64 start_ts = 1; // start ts of a query - int64 task_id = 2; // if task id is -1 , it indicates a tidb task. - int64 partition_id = 3; // Only used for hash partition - string address = 4; // target address of this task. - uint64 gather_id = 5; // used to distinguish different gathers in the mpp query. - uint64 query_ts = 6; // timestamp when start to execute query, used for TiFlash miniTSO schedule. - uint64 local_query_id = 7; // unique local query_id if tidb don't restart. So we can use gather_id + query_ts + local_query_id + server_id to represent a global unique query. - uint64 server_id = 8; // TiDB server id - int64 mpp_version = 9; // mpp version - uint32 keyspace_id = 10; // keyspace id of the request - string coordinator_address = 11; // coordinator_address of this query - bool report_execution_summary = 12; // Only when coordinator_address is not empty, this flag can be true. When set to true, TiFlash only report execution summary through ReportMPPTaskStatus service, don't include summaries in MppDataPacket - kvrpcpb.APIVersion api_version = 16; // API version of the request - string resource_group_name = 17; - uint64 connection_id = 18; // This is the session id between a client and tidb - string connection_alias = 19; // This is the session alias between a client and tidb + uint64 start_ts = 1; // start ts of a query + int64 task_id = 2; // if task id is -1 , it indicates a tidb task. + int64 partition_id = 3; // Only used for hash partition + string address = 4; // target address of this task. + uint64 gather_id = 5; // used to distinguish different gathers in the mpp query. + uint64 query_ts = 6; // timestamp when start to execute query, used for TiFlash miniTSO schedule. + uint64 local_query_id = 7; // unique local query_id if tidb don't restart. So we can use gather_id + query_ts + local_query_id + server_id to represent a global unique query. + uint64 server_id = 8; // TiDB server id + int64 mpp_version = 9; // mpp version + uint32 keyspace_id = 10; // keyspace id of the request + string coordinator_address = 11; // coordinator_address of this query + bool report_execution_summary = 12; // Only when coordinator_address is not empty, this flag can be true. When set to true, TiFlash only report execution summary through ReportMPPTaskStatus service, don't include summaries in MppDataPacket + kvrpcpb.APIVersion api_version = 16; // API version of the request + string resource_group_name = 17; + uint64 connection_id = 18; // This is the session id between a client and tidb + string connection_alias = 19; // This is the session alias between a client and tidb + string sql_digest = 20; + string plan_digest = 21; } -message IsAliveRequest { -} +message IsAliveRequest {} message IsAliveResponse { - bool available = 1; - int64 mpp_version = 2; + bool available = 1; + int64 mpp_version = 2; } // Dipsatch the task request to different tiflash servers. message DispatchTaskRequest { - TaskMeta meta = 1; - bytes encoded_plan = 2; - int64 timeout = 3; - repeated coprocessor.RegionInfo regions = 4; - // If this task contains table scan, we still need their region info. - int64 schema_ver = 5; - // Used for partition table scan - repeated coprocessor.TableRegions table_regions = 6; + TaskMeta meta = 1; + bytes encoded_plan = 2; + int64 timeout = 3; + repeated coprocessor.RegionInfo regions = 4; + // If this task contains table scan, we still need their region info. + int64 schema_ver = 5; + // Used for partition table scan + repeated coprocessor.TableRegions table_regions = 6; + // Shard infos for TiCI/FTS routing in MPP dispatch path. + repeated coprocessor.TableShardInfos table_shard_infos = 7; } // Get response of DispatchTaskRequest. message DispatchTaskResponse { - Error error = 1; - repeated metapb.Region retry_regions = 2; + Error error = 1; + repeated metapb.Region retry_regions = 2; } // CancelTaskRequest closes the execution of a task. message CancelTaskRequest { - TaskMeta meta = 1; - Error error = 2; + TaskMeta meta = 1; + Error error = 2; } message CancelTaskResponse { - Error error = 1; + Error error = 1; } // ReportTaskStatus reports the execution status of a task. // when TiFlash reports status to TiDB, ReportTaskStatusRequest serialize tipb.TiFlashExecutionInfo into data; message ReportTaskStatusRequest { - TaskMeta meta = 1; - bytes data = 2; - Error error = 3; + TaskMeta meta = 1; + bytes data = 2; + Error error = 3; } message ReportTaskStatusResponse { - Error error = 1; + Error error = 1; } // build connection between different tasks. Data is sent by the tasks that are closer to the data sources. message EstablishMPPConnectionRequest { - TaskMeta sender_meta = 1; // node closer to the source - TaskMeta receiver_meta = 2; // node closer to the tidb mpp gather. + TaskMeta sender_meta = 1; // node closer to the source + TaskMeta receiver_meta = 2; // node closer to the tidb mpp gather. } // when TiFlash sends data to TiDB, Data packets wrap tipb.SelectResponse, i.e., serialize tipb.SelectResponse into data; // when TiFlash sends data to TiFlash, data blocks are serialized into chunks, and the execution_summaries in tipb.SelectResponse are serialized into data only for the last packet. message MPPDataPacket { - bytes data = 1; - Error error = 2; - repeated bytes chunks = 3; - repeated uint64 stream_ids = 4; - int64 version = 5; // version of data packet format + bytes data = 1; + Error error = 2; + repeated bytes chunks = 3; + repeated uint64 stream_ids = 4; + int64 version = 5; // version of data packet format } message Error { - int32 code = 1; - string msg = 2; - int64 mpp_version = 3; + int32 code = 1; + string msg = 2; + int64 mpp_version = 3; } diff --git a/proto/pdpb.proto b/proto/pdpb.proto index c8fec849..5142625e 100644 --- a/proto/pdpb.proto +++ b/proto/pdpb.proto @@ -1,1127 +1,1423 @@ syntax = "proto3"; package pdpb; -import "metapb.proto"; import "eraftpb.proto"; +import "gogoproto/gogo.proto"; +import "metapb.proto"; import "raft_serverpb.proto"; import "replication_modepb.proto"; - -import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service PD { - // GetClusterInfo get the information of this cluster. It does not require - // the cluster_id in request matchs the id of this cluster. - rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} + // GetClusterInfo get the information of this cluster. It does not require + // the cluster_id in request matchs the id of this cluster. + rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {} + + // GetMembers get the member list of this cluster. It does not require + // the cluster_id in request matchs the id of this cluster. + rpc GetMembers(GetMembersRequest) returns (GetMembersResponse) {} + + rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} + + rpc Bootstrap(BootstrapRequest) returns (BootstrapResponse) {} - // GetMembers get the member list of this cluster. It does not require - // the cluster_id in request matchs the id of this cluster. - rpc GetMembers(GetMembersRequest) returns (GetMembersResponse) {} + rpc IsBootstrapped(IsBootstrappedRequest) returns (IsBootstrappedResponse) {} - rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} + rpc AllocID(AllocIDRequest) returns (AllocIDResponse) {} - rpc Bootstrap(BootstrapRequest) returns (BootstrapResponse) {} + rpc IsSnapshotRecovering(IsSnapshotRecoveringRequest) returns (IsSnapshotRecoveringResponse) {} - rpc IsBootstrapped(IsBootstrappedRequest) returns (IsBootstrappedResponse) {} + rpc GetStore(GetStoreRequest) returns (GetStoreResponse) {} - rpc AllocID(AllocIDRequest) returns (AllocIDResponse) {} + rpc PutStore(PutStoreRequest) returns (PutStoreResponse) {} - rpc IsSnapshotRecovering(IsSnapshotRecoveringRequest) returns (IsSnapshotRecoveringResponse) {} + rpc GetAllStores(GetAllStoresRequest) returns (GetAllStoresResponse) {} - rpc GetStore(GetStoreRequest) returns (GetStoreResponse) {} + rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} - rpc PutStore(PutStoreRequest) returns (PutStoreResponse) {} + rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} - rpc GetAllStores(GetAllStoresRequest) returns (GetAllStoresResponse) {} + rpc GetRegion(GetRegionRequest) returns (GetRegionResponse) {} - rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + rpc GetPrevRegion(GetRegionRequest) returns (GetRegionResponse) {} - rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} + rpc GetRegionByID(GetRegionByIDRequest) returns (GetRegionResponse) {} - rpc GetRegion(GetRegionRequest) returns (GetRegionResponse) {} + rpc QueryRegion(stream QueryRegionRequest) returns (stream QueryRegionResponse) {} - rpc GetPrevRegion(GetRegionRequest) returns (GetRegionResponse) {} + // Deprecated: use BatchScanRegions instead. + rpc ScanRegions(ScanRegionsRequest) returns (ScanRegionsResponse) {} - rpc GetRegionByID(GetRegionByIDRequest) returns (GetRegionResponse) {} + rpc BatchScanRegions(BatchScanRegionsRequest) returns (BatchScanRegionsResponse) {} - rpc ScanRegions(ScanRegionsRequest) returns (ScanRegionsResponse) {} + rpc AskSplit(AskSplitRequest) returns (AskSplitResponse) { + // Use AskBatchSplit instead. + option deprecated = true; + } - rpc AskSplit(AskSplitRequest) returns (AskSplitResponse) { - // Use AskBatchSplit instead. - option deprecated = true; - } + rpc ReportSplit(ReportSplitRequest) returns (ReportSplitResponse) { + // Use ResportBatchSplit instead. + option deprecated = true; + } - rpc ReportSplit(ReportSplitRequest) returns (ReportSplitResponse) { - // Use ResportBatchSplit instead. - option deprecated = true; - } + rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} - rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} + rpc ReportBatchSplit(ReportBatchSplitRequest) returns (ReportBatchSplitResponse) {} - rpc ReportBatchSplit(ReportBatchSplitRequest) returns (ReportBatchSplitResponse) {} + rpc GetClusterConfig(GetClusterConfigRequest) returns (GetClusterConfigResponse) {} - rpc GetClusterConfig(GetClusterConfigRequest) returns (GetClusterConfigResponse) {} + rpc PutClusterConfig(PutClusterConfigRequest) returns (PutClusterConfigResponse) {} - rpc PutClusterConfig(PutClusterConfigRequest) returns (PutClusterConfigResponse) {} + rpc ScatterRegion(ScatterRegionRequest) returns (ScatterRegionResponse) {} - rpc ScatterRegion(ScatterRegionRequest) returns (ScatterRegionResponse) {} + rpc GetGCSafePoint(GetGCSafePointRequest) returns (GetGCSafePointResponse) {} - rpc GetGCSafePoint(GetGCSafePointRequest) returns (GetGCSafePointResponse) {} + rpc UpdateGCSafePoint(UpdateGCSafePointRequest) returns (UpdateGCSafePointResponse) {} - rpc UpdateGCSafePoint(UpdateGCSafePointRequest) returns (UpdateGCSafePointResponse) {} + rpc UpdateServiceGCSafePoint(UpdateServiceGCSafePointRequest) returns (UpdateServiceGCSafePointResponse) {} - rpc UpdateServiceGCSafePoint(UpdateServiceGCSafePointRequest) returns (UpdateServiceGCSafePointResponse) {} + rpc GetGCSafePointV2(GetGCSafePointV2Request) returns (GetGCSafePointV2Response) {} - rpc GetGCSafePointV2(GetGCSafePointV2Request) returns (GetGCSafePointV2Response) {} + rpc WatchGCSafePointV2(WatchGCSafePointV2Request) returns (stream WatchGCSafePointV2Response) {} - rpc WatchGCSafePointV2(WatchGCSafePointV2Request) returns (stream WatchGCSafePointV2Response) {} + rpc UpdateGCSafePointV2(UpdateGCSafePointV2Request) returns (UpdateGCSafePointV2Response) {} - rpc UpdateGCSafePointV2(UpdateGCSafePointV2Request) returns (UpdateGCSafePointV2Response) {} + rpc UpdateServiceSafePointV2(UpdateServiceSafePointV2Request) returns (UpdateServiceSafePointV2Response) {} - rpc UpdateServiceSafePointV2(UpdateServiceSafePointV2Request) returns (UpdateServiceSafePointV2Response) {} + rpc GetAllGCSafePointV2(GetAllGCSafePointV2Request) returns (GetAllGCSafePointV2Response) {} - rpc GetAllGCSafePointV2(GetAllGCSafePointV2Request) returns (GetAllGCSafePointV2Response) {} + rpc AdvanceGCSafePoint(AdvanceGCSafePointRequest) returns (AdvanceGCSafePointResponse) {} - rpc SyncRegions(stream SyncRegionRequest) returns (stream SyncRegionResponse) {} + rpc AdvanceTxnSafePoint(AdvanceTxnSafePointRequest) returns (AdvanceTxnSafePointResponse) {} - rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} + rpc SetGCBarrier(SetGCBarrierRequest) returns (SetGCBarrierResponse) {} - rpc SyncMaxTS(SyncMaxTSRequest) returns (SyncMaxTSResponse) {} + rpc DeleteGCBarrier(DeleteGCBarrierRequest) returns (DeleteGCBarrierResponse) {} - rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} + rpc SetGlobalGCBarrier(SetGlobalGCBarrierRequest) returns (SetGlobalGCBarrierResponse) {} - rpc SplitAndScatterRegions(SplitAndScatterRegionsRequest) returns (SplitAndScatterRegionsResponse) {} + rpc DeleteGlobalGCBarrier(DeleteGlobalGCBarrierRequest) returns (DeleteGlobalGCBarrierResponse) {} - rpc GetDCLocationInfo(GetDCLocationInfoRequest) returns (GetDCLocationInfoResponse) {} + rpc GetGCState(GetGCStateRequest) returns (GetGCStateResponse) {} - rpc StoreGlobalConfig(StoreGlobalConfigRequest) returns (StoreGlobalConfigResponse) {} + rpc GetAllKeyspacesGCStates(GetAllKeyspacesGCStatesRequest) returns (GetAllKeyspacesGCStatesResponse) {} - rpc LoadGlobalConfig(LoadGlobalConfigRequest) returns (LoadGlobalConfigResponse) {} + rpc SyncRegions(stream SyncRegionRequest) returns (stream SyncRegionResponse) {} - rpc WatchGlobalConfig(WatchGlobalConfigRequest) returns (stream WatchGlobalConfigResponse) {} + rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} - rpc ReportBuckets(stream ReportBucketsRequest) returns (ReportBucketsResponse) {} + rpc SyncMaxTS(SyncMaxTSRequest) returns (SyncMaxTSResponse) {} - rpc ReportMinResolvedTS(ReportMinResolvedTsRequest) returns (ReportMinResolvedTsResponse) {} + rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} - rpc SetExternalTimestamp(SetExternalTimestampRequest) returns (SetExternalTimestampResponse) {} + rpc SplitAndScatterRegions(SplitAndScatterRegionsRequest) returns (SplitAndScatterRegionsResponse) {} - rpc GetExternalTimestamp(GetExternalTimestampRequest) returns (GetExternalTimestampResponse) {} + rpc GetDCLocationInfo(GetDCLocationInfoRequest) returns (GetDCLocationInfoResponse) {} - // Get the minimum timestamp across all keyspace groups from API server - // TODO: Currently, we need to ask API server to get the minimum timestamp. - // Once we support service discovery, we can remove it. - rpc GetMinTS (GetMinTSRequest) returns (GetMinTSResponse) {} + rpc StoreGlobalConfig(StoreGlobalConfigRequest) returns (StoreGlobalConfigResponse) {} + + rpc LoadGlobalConfig(LoadGlobalConfigRequest) returns (LoadGlobalConfigResponse) {} + + rpc WatchGlobalConfig(WatchGlobalConfigRequest) returns (stream WatchGlobalConfigResponse) {} + + rpc ReportBuckets(stream ReportBucketsRequest) returns (ReportBucketsResponse) {} + + rpc ReportMinResolvedTS(ReportMinResolvedTsRequest) returns (ReportMinResolvedTsResponse) {} + + rpc SetExternalTimestamp(SetExternalTimestampRequest) returns (SetExternalTimestampResponse) {} + + rpc GetExternalTimestamp(GetExternalTimestampRequest) returns (GetExternalTimestampResponse) {} + + // Get the minimum timestamp across all keyspace groups from API server + // TODO: Currently, we need to ask API server to get the minimum timestamp. + // Once we support service discovery, we can remove it. + rpc GetMinTS(GetMinTSRequest) returns (GetMinTSResponse) {} } message WatchGlobalConfigRequest { - string config_path = 1; - int64 revision = 2; + string config_path = 1; + int64 revision = 2; } message WatchGlobalConfigResponse { - repeated GlobalConfigItem changes = 1; - int64 revision = 2; - ResponseHeader header = 3; + repeated GlobalConfigItem changes = 1; + int64 revision = 2; + ResponseHeader header = 3; } message StoreGlobalConfigRequest { - repeated GlobalConfigItem changes = 1; - string config_path = 2; + repeated GlobalConfigItem changes = 1; + string config_path = 2; } message StoreGlobalConfigResponse { - Error error = 1; + Error error = 1; } message LoadGlobalConfigRequest { - repeated string names = 1; - string config_path = 2; + repeated string names = 1; + string config_path = 2; } message LoadGlobalConfigResponse { - repeated GlobalConfigItem items = 1; - int64 revision = 2; + repeated GlobalConfigItem items = 1; + int64 revision = 2; } enum EventType { - PUT = 0; - DELETE = 1; + PUT = 0; + DELETE = 1; } message GlobalConfigItem { - string name = 1; - // this field 'value' is replaced by the field 'payload'. - string value = 2; - Error error = 3; - EventType kind = 4; - // Since item value needs to support marshal of different struct types, - // it should be set to bytes instead of string. - bytes payload = 5; + string name = 1; + // this field 'value' is replaced by the field 'payload'. + string value = 2; + Error error = 3; + EventType kind = 4; + // Since item value needs to support marshal of different struct types, + // it should be set to bytes instead of string. + bytes payload = 5; } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // sender_id is the ID of the sender server, also member ID or etcd ID. - uint64 sender_id = 2; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // sender_id is the ID of the sender server, also member ID or etcd ID. + // sender_id is used in PD internal communication. + uint64 sender_id = 2; + // caller_id is the ID of the client which sends the request, such as tikv, + // tidb, cdc, etc. + string caller_id = 3; + // caller_component is the component of the client which sends the request, + // such as ddl, optimizer, etc. + string caller_component = 4; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; } enum ErrorType { - OK = 0; - UNKNOWN = 1; - NOT_BOOTSTRAPPED = 2; - STORE_TOMBSTONE = 3; - ALREADY_BOOTSTRAPPED = 4; - INCOMPATIBLE_VERSION = 5; - REGION_NOT_FOUND = 6; - GLOBAL_CONFIG_NOT_FOUND = 7; - DUPLICATED_ENTRY = 8; - ENTRY_NOT_FOUND = 9; - INVALID_VALUE = 10; - // required watch revision is smaller than current compact/min revision. - DATA_COMPACTED = 11; + OK = 0; + UNKNOWN = 1; + NOT_BOOTSTRAPPED = 2; + STORE_TOMBSTONE = 3; + ALREADY_BOOTSTRAPPED = 4; + INCOMPATIBLE_VERSION = 5; + REGION_NOT_FOUND = 6; + GLOBAL_CONFIG_NOT_FOUND = 7; + DUPLICATED_ENTRY = 8; + ENTRY_NOT_FOUND = 9; + INVALID_VALUE = 10; + // required watch revision is smaller than current compact/min revision. + DATA_COMPACTED = 11; + REGIONS_NOT_CONTAIN_ALL_KEY_RANGE = 12; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message TsoRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 count = 2; - string dc_location = 3; + uint32 count = 2; + string dc_location = 3; } message Timestamp { - int64 physical = 1; - int64 logical = 2; - // Number of suffix bits used for global distinction, - // PD client will use this to compute a TSO's logical part. - uint32 suffix_bits = 3; + int64 physical = 1; + int64 logical = 2; + // Number of suffix bits used for global distinction, + // PD client will use this to compute a TSO's logical part. + uint32 suffix_bits = 3; } message TsoResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint32 count = 2; - Timestamp timestamp = 3; + uint32 count = 2; + Timestamp timestamp = 3; } message BootstrapRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Store store = 2; - metapb.Region region = 3; + metapb.Store store = 2; + metapb.Region region = 3; } message BootstrapResponse { - ResponseHeader header = 1; - replication_modepb.ReplicationStatus replication_status = 2; + ResponseHeader header = 1; + replication_modepb.ReplicationStatus replication_status = 2; } message IsBootstrappedRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message IsBootstrappedResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - bool bootstrapped = 2; + bool bootstrapped = 2; } message AllocIDRequest { - RequestHeader header = 1; + RequestHeader header = 1; + + uint32 count = 2; } message AllocIDResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 id = 2; + uint64 id = 2; + uint32 count = 3; } message IsSnapshotRecoveringRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message IsSnapshotRecoveringResponse { - ResponseHeader header = 1; - bool marked = 2; + ResponseHeader header = 1; + bool marked = 2; } message GetStoreRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 store_id = 2; + uint64 store_id = 2; } message GetStoreResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - metapb.Store store = 2; - StoreStats stats = 3; + metapb.Store store = 2; + StoreStats stats = 3; } message PutStoreRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Store store = 2; + metapb.Store store = 2; } message PutStoreResponse { - ResponseHeader header = 1; - replication_modepb.ReplicationStatus replication_status = 2; + ResponseHeader header = 1; + replication_modepb.ReplicationStatus replication_status = 2; } message GetAllStoresRequest { - RequestHeader header = 1; - // Do NOT return tombstone stores if set to true. - bool exclude_tombstone_stores = 2; + RequestHeader header = 1; + // Do NOT return tombstone stores if set to true. + bool exclude_tombstone_stores = 2; } message GetAllStoresResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated metapb.Store stores = 2; + repeated metapb.Store stores = 2; } message GetRegionRequest { - RequestHeader header = 1; + RequestHeader header = 1; - bytes region_key = 2; - bool need_buckets = 3; + bytes region_key = 2; + bool need_buckets = 3; } message GetRegionResponse { - reserved 4; + reserved 4; - ResponseHeader header = 1; + ResponseHeader header = 1; - metapb.Region region = 2; - metapb.Peer leader = 3; - // Leader considers that these peers are down. - repeated PeerStats down_peers = 5; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 6; - // buckets isn't nil if GetRegion.* requests set need_buckets. - metapb.Buckets buckets = 7; + metapb.Region region = 2; + metapb.Peer leader = 3; + // Leader considers that these peers are down. + repeated PeerStats down_peers = 5; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 6; + // buckets isn't nil if GetRegion.* requests set need_buckets. + metapb.Buckets buckets = 7; } message GetRegionByIDRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 region_id = 2; - bool need_buckets = 3; + uint64 region_id = 2; + bool need_buckets = 3; } -// Use GetRegionResponse as the response of GetRegionByIDRequest. +message QueryRegionRequest { + RequestHeader header = 1; + + // Whether to include the buckets info within the response. + bool need_buckets = 2; + // The region IDs to query. + repeated uint64 ids = 3; + // The region keys to query. + repeated bytes keys = 4; + // The previous region keys to query. + repeated bytes prev_keys = 5; +} + +message QueryRegionResponse { + ResponseHeader header = 1; + // This array functions as a map corresponding to the region IDs, + // preserving the order of the input region keys, if they are present. + repeated uint64 key_id_map = 2; + // This array functions as a map corresponding to the previous region IDs, + // preserving the order of the input previous region keys, if they are present. + repeated uint64 prev_key_id_map = 3; + // RegionID -> RegionResponse + map regions_by_id = 4; +} + +message RegionResponse { + metapb.Region region = 1; + metapb.Peer leader = 2; + repeated PeerStats down_peers = 3; + repeated metapb.Peer pending_peers = 4; + metapb.Buckets buckets = 5; +} + +// Use GetRegionResponse as the response of GetRegionByIDRequest. +// Deprecated: use BatchScanRegionsRequest instead. message ScanRegionsRequest { - RequestHeader header = 1; + RequestHeader header = 1; - bytes start_key = 2; - int32 limit = 3; // no limit when limit <= 0. - bytes end_key = 4; // end_key is +inf when it is empty. + bytes start_key = 2; + int32 limit = 3; // no limit when limit <= 0. + bytes end_key = 4; // end_key is +inf when it is empty. } message Region { - metapb.Region region = 1; - metapb.Peer leader = 2; - // Leader considers that these peers are down. - repeated PeerStats down_peers = 3; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 4; + metapb.Region region = 1; + metapb.Peer leader = 2; + // Leader considers that these peers are down. + repeated PeerStats down_peers = 3; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 4; + // buckets isn't nil only when need_buckets is true. + metapb.Buckets buckets = 5; } message ScanRegionsResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; + + // Keep for backword compatibability. + repeated metapb.Region region_metas = 2; + repeated metapb.Peer leaders = 3; + + // Extended region info with down/pending peers. + repeated Region regions = 4; +} + +message KeyRange { + bytes start_key = 1; + bytes end_key = 2; // end_key is +inf when it is empty. +} + +message BatchScanRegionsRequest { + RequestHeader header = 1; + bool need_buckets = 2; + + repeated KeyRange ranges = 3; // the given ranges must be in order. + int32 limit = 4; // limit the total number of regions to scan. + // If contain_all_key_range is true, the output must contain all + // key ranges in the request. + // If the output does not contain all key ranges, the request is considered + // failed and returns an error(REGIONS_NOT_CONTAIN_ALL_KEY_RANGE). + bool contain_all_key_range = 5; +} - // Keep for backword compatibability. - repeated metapb.Region region_metas = 2; - repeated metapb.Peer leaders = 3; +message BatchScanRegionsResponse { + ResponseHeader header = 1; - // Extended region info with down/pending peers. - repeated Region regions = 4; + // the returned regions are flattened into a list, because the given ranges can located in the same range, we do not return duplicated regions then. + repeated Region regions = 2; } message GetClusterConfigRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetClusterConfigResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - metapb.Cluster cluster = 2; + metapb.Cluster cluster = 2; } message PutClusterConfigRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Cluster cluster = 2; + metapb.Cluster cluster = 2; } message PutClusterConfigResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message Member { - // name is the name of the PD member. - string name = 1; - // member_id is the unique id of the PD member. - uint64 member_id = 2; - repeated string peer_urls = 3; - repeated string client_urls = 4; - int32 leader_priority = 5; - string deploy_path = 6; - string binary_version = 7; - string git_hash = 8; - string dc_location = 9; + // name is the name of the PD member. + string name = 1; + // member_id is the unique id of the PD member. + uint64 member_id = 2; + repeated string peer_urls = 3; + repeated string client_urls = 4; + int32 leader_priority = 5; + string deploy_path = 6; + string binary_version = 7; + string git_hash = 8; + string dc_location = 9; } message GetMembersRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetMembersResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated Member members = 2; - Member leader = 3; - Member etcd_leader = 4; - map tso_allocator_leaders = 5; + repeated Member members = 2; + Member leader = 3; + Member etcd_leader = 4; + map tso_allocator_leaders = 5; } message GetClusterInfoRequest { - ResponseHeader header = 1; + ResponseHeader header = 1; } enum ServiceMode { - UNKNOWN_SVC_MODE = 0; - PD_SVC_MODE = 1; - API_SVC_MODE = 2; + UNKNOWN_SVC_MODE = 0; + PD_SVC_MODE = 1; + API_SVC_MODE = 2; } message GetClusterInfoResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated ServiceMode serviceModes = 2; - // If service mode is API_SVC_MODE, this field will be set to the - // registered tso service addresses. - repeated string tso_urls = 3; + repeated ServiceMode serviceModes = 2; + // If service mode is API_SVC_MODE, this field will be set to the + // registered tso service addresses. + repeated string tso_urls = 3; } message PeerStats { - metapb.Peer peer = 1; - uint64 down_seconds = 2; + metapb.Peer peer = 1; + uint64 down_seconds = 2; } message RegionHeartbeatRequest { - RequestHeader header = 1; - - metapb.Region region = 2; - // Leader Peer sending the heartbeat. - metapb.Peer leader = 3; - // Leader considers that these peers are down. - repeated PeerStats down_peers = 4; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 5; - // Bytes read/written during this period. - uint64 bytes_written = 6; - uint64 bytes_read = 7; - // Keys read/written during this period. - uint64 keys_written = 8; - uint64 keys_read = 9; - // Approximate region size. - uint64 approximate_size = 10; - reserved 11; - // Actually reported time interval - TimeInterval interval = 12; - // Approximate number of keys. - uint64 approximate_keys = 13; - // Term is the term of raft group. - uint64 term = 14; - replication_modepb.RegionReplicationStatus replication_status = 15; - // QueryStats reported write query stats, and there are read query stats in store heartbeat - QueryStats query_stats = 16; - // cpu_usage is the CPU time usage of the leader region since the last heartbeat, - // which is calculated by cpu_time_delta/heartbeat_reported_interval. - uint64 cpu_usage = 17; - // (Serverless) Approximate size of key-value pairs for billing. - // It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. - uint64 approximate_kv_size = 18; + RequestHeader header = 1; + + metapb.Region region = 2; + // Leader Peer sending the heartbeat. + metapb.Peer leader = 3; + // Leader considers that these peers are down. + repeated PeerStats down_peers = 4; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 5; + // Bytes read/written during this period. + uint64 bytes_written = 6; + uint64 bytes_read = 7; + // Keys read/written during this period. + uint64 keys_written = 8; + uint64 keys_read = 9; + // Approximate region size. + uint64 approximate_size = 10; + reserved 11; + // Actually reported time interval + TimeInterval interval = 12; + // Approximate number of keys. + uint64 approximate_keys = 13; + // Term is the term of raft group. + uint64 term = 14; + replication_modepb.RegionReplicationStatus replication_status = 15; + // QueryStats reported write query stats, and there are read query stats in store heartbeat + QueryStats query_stats = 16; + // cpu_usage is the total CPU time usage of the leader region since the last heartbeat, + // which is calculated by cpu_time_delta/heartbeat_reported_interval. + // Deprecated: use cpu_stats instead. + uint64 cpu_usage = 17 [deprecated = true]; + // cpu_stats reports CPU usage breakdown for the leader region by kind + // (e.g. unified read). + CPUStats cpu_stats = 21; + // Approximate size of row-based key-value pairs for billing. + // It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. + uint64 approximate_kv_size = 18; + // Approximate size of column-based key-value pairs for billing. + // It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. + uint64 approximate_columnar_kv_size = 19; + + // BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + metapb.BucketMeta bucket_meta = 20; } message ChangePeer { - metapb.Peer peer = 1; - eraftpb.ConfChangeType change_type = 2; + metapb.Peer peer = 1; + eraftpb.ConfChangeType change_type = 2; } message ChangePeerV2 { - // If changes is empty, it means that to exit joint state. - repeated ChangePeer changes = 1; + // If changes is empty, it means that to exit joint state. + repeated ChangePeer changes = 1; } message TransferLeader { - metapb.Peer peer = 1; - repeated metapb.Peer peers = 2; + metapb.Peer peer = 1; + repeated metapb.Peer peers = 2; } message Merge { - metapb.Region target = 1; + metapb.Region target = 1; } message SplitRegion { - CheckPolicy policy = 1; - repeated bytes keys = 2; + CheckPolicy policy = 1; + repeated bytes keys = 2; } message SwitchWitness { - uint64 peer_id = 1; - bool is_witness = 2; + uint64 peer_id = 1; + bool is_witness = 2; } message BatchSwitchWitness { - repeated SwitchWitness switch_witnesses = 1; + repeated SwitchWitness switch_witnesses = 1; } enum CheckPolicy { - SCAN = 0; - APPROXIMATE = 1; - USEKEY = 2; + SCAN = 0; + APPROXIMATE = 1; + USEKEY = 2; +} + +message ChangeSplit { + // auto_split_enabled configures whether the corresponding Region is allowed to be auto split by size or load. + bool auto_split_enabled = 1; } message RegionHeartbeatResponse { - ResponseHeader header = 1; - - // Notice, Pd only allows handling reported epoch >= current pd's. - // Leader peer reports region status with RegionHeartbeatRequest - // to pd regularly, pd will determine whether this region - // should do ChangePeer or not. - // E,g, max peer number is 3, region A, first only peer 1 in A. - // 1. Pd region state -> Peers (1), ConfVer (1). - // 2. Leader peer 1 reports region state to pd, pd finds the - // peer number is < 3, so first changes its current region - // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. - // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), - // pd updates its state -> Peers (1, 2), ConfVer (2). - // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange - // finished, pd stills responses ChangePeer Adding 2, of course, we must - // guarantee the second ChangePeer can't be applied in TiKV. - ChangePeer change_peer = 2; - // Pd can return transfer_leader to let TiKV does leader transfer itself. - TransferLeader transfer_leader = 3; - // ID of the region - uint64 region_id = 4; - metapb.RegionEpoch region_epoch = 5; - // Leader of the region at the moment of the corresponding request was made. - metapb.Peer target_peer = 6; - Merge merge = 7; - // PD sends split_region to let TiKV split a region into two regions. - SplitRegion split_region = 8; - // Multiple change peer operations atomically. - // Note: PD can use both ChangePeer and ChangePeerV2 at the same time - // (not in the same RegionHeartbeatResponse). - // Now, PD use ChangePeerV2 in following scenarios: - // 1. replacing peers - // 2. demoting voter directly - ChangePeerV2 change_peer_v2 = 9; - BatchSwitchWitness switch_witnesses = 10; + ResponseHeader header = 1; + + // Notice, Pd only allows handling reported epoch >= current pd's. + // Leader peer reports region status with RegionHeartbeatRequest + // to pd regularly, pd will determine whether this region + // should do ChangePeer or not. + // E,g, max peer number is 3, region A, first only peer 1 in A. + // 1. Pd region state -> Peers (1), ConfVer (1). + // 2. Leader peer 1 reports region state to pd, pd finds the + // peer number is < 3, so first changes its current region + // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. + // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), + // pd updates its state -> Peers (1, 2), ConfVer (2). + // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange + // finished, pd stills responses ChangePeer Adding 2, of course, we must + // guarantee the second ChangePeer can't be applied in TiKV. + ChangePeer change_peer = 2; + // Pd can return transfer_leader to let TiKV does leader transfer itself. + TransferLeader transfer_leader = 3; + // ID of the region + uint64 region_id = 4; + metapb.RegionEpoch region_epoch = 5; + // Leader of the region at the moment of the corresponding request was made. + metapb.Peer target_peer = 6; + Merge merge = 7; + // PD sends split_region to let TiKV split a region into two regions. + SplitRegion split_region = 8; + // Multiple change peer operations atomically. + // Note: PD can use both ChangePeer and ChangePeerV2 at the same time + // (not in the same RegionHeartbeatResponse). + // Now, PD use ChangePeerV2 in following scenarios: + // 1. replacing peers + // 2. demoting voter directly + ChangePeerV2 change_peer_v2 = 9; + BatchSwitchWitness switch_witnesses = 10; + ChangeSplit change_split = 11; } message AskSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region region = 2; + metapb.Region region = 2; } message AskSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - // We split the region into two, first uses the origin - // parent region id, and the second uses the new_region_id. - // We must guarantee that the new_region_id is global unique. - uint64 new_region_id = 2; - // The peer ids for the new split region. - repeated uint64 new_peer_ids = 3; + // We split the region into two, first uses the origin + // parent region id, and the second uses the new_region_id. + // We must guarantee that the new_region_id is global unique. + uint64 new_region_id = 2; + // The peer ids for the new split region. + repeated uint64 new_peer_ids = 3; } message ReportSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region left = 2; - metapb.Region right = 3; + metapb.Region left = 2; + metapb.Region right = 3; } message ReportSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; +} + +enum SplitReason { + ADMIN = 0; + SIZE = 1; + LOAD = 2; } message AskBatchSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region region = 2; - uint32 split_count = 3; + metapb.Region region = 2; + uint32 split_count = 3; + SplitReason reason = 4; } message SplitID { - uint64 new_region_id = 1; - repeated uint64 new_peer_ids = 2; + uint64 new_region_id = 1; + repeated uint64 new_peer_ids = 2; } message AskBatchSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - repeated SplitID ids = 2; + repeated SplitID ids = 2; } message ReportBatchSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - repeated metapb.Region regions = 2; + repeated metapb.Region regions = 2; } message ReportBatchSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message TimeInterval { - // The unix timestamp in seconds of the start of this period. - uint64 start_timestamp = 1; - // The unix timestamp in seconds of the end of this period. - uint64 end_timestamp = 2; + // The unix timestamp in seconds of the start of this period. + uint64 start_timestamp = 1; + // The unix timestamp in seconds of the end of this period. + uint64 end_timestamp = 2; } message RecordPair { - string key = 1; - uint64 value = 2; + string key = 1; + uint64 value = 2; } message PeerStat { - uint64 region_id = 1; - uint64 read_keys = 2; - uint64 read_bytes = 3; - QueryStats query_stats = 4; - uint64 written_keys = 5; - uint64 written_bytes = 6; + uint64 region_id = 1; + uint64 read_keys = 2; + uint64 read_bytes = 3; + QueryStats query_stats = 4; + uint64 written_keys = 5; + uint64 written_bytes = 6; + // cpu_stats is the CPU usage of the region's unified read pool since the last heartbeat, + // which is calculated by cpu_time_delta/heartbeat_reported_interval. + CPUStats cpu_stats = 7; } message StoreStats { - uint64 store_id = 1; - // Capacity for the store. - uint64 capacity = 2; - // Available size for the store. - uint64 available = 3; - // Total region count in this store. - uint32 region_count = 4; - // Current sending snapshot count. - uint32 sending_snap_count = 5; - // Current receiving snapshot count. - uint32 receiving_snap_count = 6; - // When the store is started (unix timestamp in seconds). - uint32 start_time = 7; - // How many region is applying snapshot. - uint32 applying_snap_count = 8; - // If the store is busy - bool is_busy = 9; - // Actually used space by db - uint64 used_size = 10; - // Bytes written for the store during this period. - uint64 bytes_written = 11; - // Keys written for the store during this period. - uint64 keys_written = 12; - // Bytes read for the store during this period. - uint64 bytes_read = 13; - // Keys read for the store during this period. - uint64 keys_read = 14; - // Actually reported time interval - TimeInterval interval = 15; - // Threads' CPU usages in the store - repeated RecordPair cpu_usages = 16; - // Threads' read disk I/O rates in the store - repeated RecordPair read_io_rates = 17; - // Threads' write disk I/O rates in the store - repeated RecordPair write_io_rates = 18; - // Operations' latencies in the store - repeated RecordPair op_latencies = 19; - // Hot peer stat in the store - repeated PeerStat peer_stats = 20; - // Store query stats - QueryStats query_stats = 21; - // Score that represents the speed of the store, ranges in [1, 100], lower is better. - uint64 slow_score = 22; - // Damaged regions on the store that need to be removed by PD. - repeated uint64 damaged_regions_id = 23; - // If the apply worker is busy, namely high apply wait duration - bool is_apply_busy = 24; - // Snapshot stats in the store - repeated SnapshotStat snapshot_stats = 25; - SlowTrend slow_trend = 26; - // If the grpc server is paused. - bool is_grpc_paused = 27; - // Total memory of the store in bytes. - uint64 total_memory = 28; - // Used memory of the store in bytes. - uint64 used_memory = 29; -} - -message SlowTrend{ - double cause_value = 1; - double cause_rate = 2; - double result_value = 3; - double result_rate = 4; -} - -message SnapshotStat{ - uint64 region_id = 1; - // Generate snapshot duration - uint64 generate_duration_sec = 2; - // Send snapshot duration - uint64 send_duration_sec = 3; - // |-- waiting --|-- generate --｜-- send --| - // |-----------total duration---------------| - // Total duration include waiting and executing duration - uint64 total_duration_sec = 4; - // Size is the transport data size - uint64 transport_size = 5; + uint64 store_id = 1; + // Capacity for the store. + uint64 capacity = 2; + // Available size for the store. + uint64 available = 3; + // Total region count in this store. + uint32 region_count = 4; + // Current sending snapshot count. + uint32 sending_snap_count = 5; + // Current receiving snapshot count. + uint32 receiving_snap_count = 6; + // When the store is started (unix timestamp in seconds). + uint32 start_time = 7; + // How many region is applying snapshot. + uint32 applying_snap_count = 8; + // If the store is busy + bool is_busy = 9; + // Actually used space by db + uint64 used_size = 10; + // Bytes written for the store during this period. + uint64 bytes_written = 11; + // Keys written for the store during this period. + uint64 keys_written = 12; + // Bytes read for the store during this period. + uint64 bytes_read = 13; + // Keys read for the store during this period. + uint64 keys_read = 14; + // Actually reported time interval + TimeInterval interval = 15; + // Threads' CPU usages in the store + repeated RecordPair cpu_usages = 16; + // Threads' read disk I/O rates in the store + repeated RecordPair read_io_rates = 17; + // Threads' write disk I/O rates in the store + repeated RecordPair write_io_rates = 18; + // Operations' latencies in the store + repeated RecordPair op_latencies = 19; + // Hot peer stat in the store + repeated PeerStat peer_stats = 20; + // Store query stats + QueryStats query_stats = 21; + // Score that represents the speed of the store, ranges in [1, 100], lower is better. + uint64 slow_score = 22; + // Damaged regions on the store that need to be removed by PD. + repeated uint64 damaged_regions_id = 23; + // If the apply worker is busy, namely high apply wait duration + bool is_apply_busy = 24; + // Snapshot stats in the store + repeated SnapshotStat snapshot_stats = 25; + SlowTrend slow_trend = 26; + // If the grpc server is paused. + bool is_grpc_paused = 27; + // Total memory of the store in bytes. + uint64 total_memory = 28; + // Used memory of the store in bytes. + uint64 used_memory = 29; + // Network_slow_scores indicate the network status between TiKV nodes, ranging from 1 to 100 (lower is better). + // StoreID -> score + map network_slow_scores = 30; + // The statistics about DFS uploads. + repeated DfsStatItem dfs = 31; + // True if the store is undergoing graceful shutdown. + bool is_stopping = 32; + + // Reserved for a downstream fork + reserved 1000 to 1199; +} + +message DfsStatScope { + // When true, the statistic is not tied to any keyspace. + bool is_global = 1; + // The keyspace of this statistic. Ignore when is_global is true. + uint32 keyspace_id = 2; + // The component that provides the statistic. + string component = 3; +} + +message DfsStatItem { + DfsStatScope scope = 1; + // Number of bytes written to DFS. + uint64 written_bytes = 2; + // Number of write requests sent to DFS. + uint64 write_requests = 3; +} + +message SlowTrend { + double cause_value = 1; + double cause_rate = 2; + double result_value = 3; + double result_rate = 4; +} + +message SnapshotStat { + uint64 region_id = 1; + // Generate snapshot duration + uint64 generate_duration_sec = 2; + // Send snapshot duration + uint64 send_duration_sec = 3; + // |-- waiting --|-- generate --｜-- send --| + // |-----------total duration---------------| + // Total duration include waiting and executing duration + uint64 total_duration_sec = 4; + // Size is the transport data size + uint64 transport_size = 5; } message PeerReport { - raft_serverpb.RaftLocalState raft_state = 1; - raft_serverpb.RegionLocalState region_state = 2; - bool is_force_leader = 3; - // The peer has proposed but uncommitted commit merge. - bool has_commit_merge = 4; + raft_serverpb.RaftLocalState raft_state = 1; + raft_serverpb.RegionLocalState region_state = 2; + bool is_force_leader = 3; + // The peer has proposed but uncommitted commit merge. + bool has_commit_merge = 4; + // raft applied index + uint64 applied_index = 5; } message StoreReport { - repeated PeerReport peer_reports = 1; - uint64 step = 2; + repeated PeerReport peer_reports = 1; + uint64 step = 2; } message StoreHeartbeatRequest { - RequestHeader header = 1; + RequestHeader header = 1; - StoreStats stats = 2; - // Detailed store report that is only filled up on PD's demand for online unsafe recovery. - StoreReport store_report = 3; - replication_modepb.StoreDRAutoSyncStatus dr_autosync_status = 4; + StoreStats stats = 2; + // Detailed store report that is only filled up on PD's demand for online unsafe recovery. + StoreReport store_report = 3; + replication_modepb.StoreDRAutoSyncStatus dr_autosync_status = 4; } message DemoteFailedVoters { - uint64 region_id = 1; - repeated metapb.Peer failed_voters = 2; + uint64 region_id = 1; + repeated metapb.Peer failed_voters = 2; } message ForceLeader { - // The store ids of the failed stores, TiKV uses it to decide if a peer is alive. - repeated uint64 failed_stores = 1; - // The region ids of the peer which is to be force leader. - repeated uint64 enter_force_leaders = 2; + // The store ids of the failed stores, TiKV uses it to decide if a peer is alive. + repeated uint64 failed_stores = 1; + // The region ids of the peer which is to be force leader. + repeated uint64 enter_force_leaders = 2; } message RecoveryPlan { - // Create empty regions to fill the key range hole. - repeated metapb.Region creates = 1; - // Update the meta of the regions, including peer lists, epoch and key range. - repeated metapb.Region updates = 2 [deprecated=true]; - // Tombstone the peers on the store locally. - repeated uint64 tombstones = 3; - // Issue conf change that demote voters on failed stores to learners on the regions. - repeated DemoteFailedVoters demotes = 4; - // Make the peers to be force leaders. - ForceLeader force_leader = 5; - // Step is an increasing number to note the round of recovery, - // It should be filled in the corresponding store report. - uint64 step = 6; + // Create empty regions to fill the key range hole. + repeated metapb.Region creates = 1; + // Update the meta of the regions, including peer lists, epoch and key range. + repeated metapb.Region updates = 2 [deprecated = true]; + // Tombstone the peers on the store locally. + repeated uint64 tombstones = 3; + // Issue conf change that demote voters on failed stores to learners on the regions. + repeated DemoteFailedVoters demotes = 4; + // Make the peers to be force leaders. + ForceLeader force_leader = 5; + // Step is an increasing number to note the round of recovery, + // It should be filled in the corresponding store report. + uint64 step = 6; } message AwakenRegions { - // Awake all regions if abnormal_stores is empty. - repeated uint64 abnormal_stores = 1; + // Awake all regions if abnormal_stores is empty. + repeated uint64 abnormal_stores = 1; } enum ControlGrpcEvent { - // Pause TiKV grpc server. - PAUSE = 0; - // Resume TiKV grpc server. - RESUME = 1; + // Pause TiKV grpc server. + PAUSE = 0; + // Resume TiKV grpc server. + RESUME = 1; } message ControlGrpc { - ControlGrpcEvent ctrl_event = 1; + ControlGrpcEvent ctrl_event = 1; } message StoreHeartbeatResponse { - ResponseHeader header = 1; - replication_modepb.ReplicationStatus replication_status = 2; - string cluster_version = 3; - - // Used by online unsafe recovery to request store report. - // Now it's substituted by reusing recovery_plan field. PD will send a empty - // recovery plan instead to request store report. - bool require_detailed_report = 4 [deprecated=true]; - // Operations of recovery. After the plan is executed, TiKV should attach the - // store report in store heartbeat. - RecoveryPlan recovery_plan = 5; - // Pd can return awaken_regions to let TiKV awaken hibernated regions itself. - AwakenRegions awaken_regions = 6; - // Pd can return operations to let TiKV forcely PAUSE | RESUME grpc server. - ControlGrpc control_grpc = 7; + ResponseHeader header = 1; + replication_modepb.ReplicationStatus replication_status = 2; + string cluster_version = 3; + + // Used by online unsafe recovery to request store report. + // Now it's substituted by reusing recovery_plan field. PD will send a empty + // recovery plan instead to request store report. + bool require_detailed_report = 4 [deprecated = true]; + // Operations of recovery. After the plan is executed, TiKV should attach the + // store report in store heartbeat. + RecoveryPlan recovery_plan = 5; + // Pd can return awaken_regions to let TiKV awaken hibernated regions itself. + AwakenRegions awaken_regions = 6; + // Pd can return operations to let TiKV forcely PAUSE | RESUME grpc server. + ControlGrpc control_grpc = 7; + // NodeState is going to mark the state of the store. + metapb.NodeState state = 8; } message ScatterRegionRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 region_id = 2 [deprecated=true]; + uint64 region_id = 2 [deprecated = true]; - // PD will use these region information if it can't find the region. - // For example, the region is just split and hasn't report to PD yet. - metapb.Region region = 3; - metapb.Peer leader = 4; + // PD will use these region information if it can't find the region. + // For example, the region is just split and hasn't report to PD yet. + metapb.Region region = 3; + metapb.Peer leader = 4; - // If group is defined, the regions with the same group would be scattered as a whole group. - // If not defined, the regions would be scattered in a cluster level. - string group = 5; + // If group is defined, the regions with the same group would be scattered as a whole group. + // If not defined, the regions would be scattered in a cluster level. + string group = 5; - // If regions_id is defined, the region_id would be ignored. - repeated uint64 regions_id = 6; - uint64 retry_limit = 7; - bool skip_store_limit = 8; + // If regions_id is defined, the region_id would be ignored. + repeated uint64 regions_id = 6; + uint64 retry_limit = 7; + bool skip_store_limit = 8; } message ScatterRegionResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 failed_regions_id = 3; } message GetGCSafePointRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetGCSafePointResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 safe_point = 2; + uint64 safe_point = 2; } message UpdateGCSafePointRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 safe_point = 2; + uint64 safe_point = 2; } message UpdateGCSafePointResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 new_safe_point = 2; + uint64 new_safe_point = 2; } message UpdateServiceGCSafePointRequest { - RequestHeader header = 1; + RequestHeader header = 1; - bytes service_id = 2; - int64 TTL = 3; - uint64 safe_point = 4; + bytes service_id = 2; + int64 TTL = 3; + uint64 safe_point = 4; } message UpdateServiceGCSafePointResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - bytes service_id = 2; - int64 TTL = 3; - uint64 min_safe_point = 4; + bytes service_id = 2; + int64 TTL = 3; + uint64 min_safe_point = 4; } message GetGCSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 keyspace_id = 2; + uint32 keyspace_id = 2; } message GetGCSafePointV2Response { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 safe_point = 2; + uint64 safe_point = 2; } message WatchGCSafePointV2Request { - RequestHeader header = 1; - int64 revision = 2; - + RequestHeader header = 1; + int64 revision = 2; } // SafePointEvent is for the rpc WatchGCSafePointV2. message SafePointEvent { - uint32 keyspace_id = 1; - uint64 safe_point = 2; - EventType type = 3; + uint32 keyspace_id = 1; + uint64 safe_point = 2; + EventType type = 3; } message WatchGCSafePointV2Response { - ResponseHeader header = 1; - repeated SafePointEvent events = 2; - int64 revision = 3; + ResponseHeader header = 1; + repeated SafePointEvent events = 2; + int64 revision = 3; } message UpdateGCSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 keyspace_id = 2; - uint64 safe_point = 3; + uint32 keyspace_id = 2; + uint64 safe_point = 3; } message UpdateGCSafePointV2Response { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 new_safe_point = 2; + uint64 new_safe_point = 2; } message UpdateServiceSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 keyspace_id = 2; - bytes service_id = 3; - uint64 safe_point = 4; - // Safe point will be set to expire on (PD Server time + TTL), - // pass in a ttl < 0 to remove target safe point; - // pass in MAX_INT64 to set a safe point that never expire. - // This should be set by component that may crash unexpectedly so that it doesn't block - // cluster garbage collection. - int64 ttl = 5; + uint32 keyspace_id = 2; + bytes service_id = 3; + uint64 safe_point = 4; + // Safe point will be set to expire on (PD Server time + TTL), + // pass in a ttl < 0 to remove target safe point; + // pass in MAX_INT64 to set a safe point that never expire. + // This should be set by component that may crash unexpectedly so that it doesn't block + // cluster garbage collection. + int64 ttl = 5; } message UpdateServiceSafePointV2Response { - ResponseHeader header = 1; + ResponseHeader header = 1; - bytes service_id = 2; - int64 ttl = 3; - uint64 min_safe_point = 4; + bytes service_id = 2; + int64 ttl = 3; + uint64 min_safe_point = 4; } message GetAllGCSafePointV2Request { - RequestHeader header = 1; + RequestHeader header = 1; } message GCSafePointV2 { - uint32 keyspace_id = 1; - uint64 gc_safe_point = 2; + uint32 keyspace_id = 1; + uint64 gc_safe_point = 2; } message GetAllGCSafePointV2Response { - ResponseHeader header = 1; - repeated GCSafePointV2 gc_safe_points = 2; - int64 revision = 3; + ResponseHeader header = 1; + repeated GCSafePointV2 gc_safe_points = 2; + int64 revision = 3; +} + +// A wrapper over keyspace_id. +// When a field is not specified in proto3, its value will be regarded as 0; however, keyspace_id = 0 is regarded as a valid keyspace (which +// is the "DEFAULT" keyspace). To distinguish unspecified keyspace (NullKeyspace, 0xffffffff) and the default keyspace in some APIs as well +// as preventing potential misuse, we wrap the keyspace_id into a message type which is nullable. +message KeyspaceScope { + uint32 keyspace_id = 1; +} + +message AdvanceGCSafePointRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + uint64 target = 3; +} + +message AdvanceGCSafePointResponse { + ResponseHeader header = 1; + + uint64 old_gc_safe_point = 2; + uint64 new_gc_safe_point = 3; +} + +message AdvanceTxnSafePointRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + uint64 target = 3; +} + +message AdvanceTxnSafePointResponse { + ResponseHeader header = 1; + + uint64 old_txn_safe_point = 2; + uint64 new_txn_safe_point = 3; + string blocker_description = 4; +} + +message SetGCBarrierRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + + string barrier_id = 3; + uint64 barrier_ts = 4; + int64 ttl_seconds = 5; +} + +message GCBarrierInfo { + string barrier_id = 1; + uint64 barrier_ts = 2; + int64 ttl_seconds = 3; +} + +message SetGCBarrierResponse { + ResponseHeader header = 1; + + GCBarrierInfo new_barrier_info = 2; +} + +message DeleteGCBarrierRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + + string barrier_id = 3; +} + +message DeleteGCBarrierResponse { + ResponseHeader header = 1; + + GCBarrierInfo deleted_barrier_info = 2; +} + +message SetGlobalGCBarrierRequest { + RequestHeader header = 1; + string barrier_id = 2; + uint64 barrier_ts = 3; + int64 ttl_seconds = 4; +} + +message SetGlobalGCBarrierResponse { + ResponseHeader header = 1; + GlobalGCBarrierInfo new_barrier_info = 2; +} + +message DeleteGlobalGCBarrierRequest { + RequestHeader header = 1; + string barrier_id = 2; +} + +message DeleteGlobalGCBarrierResponse { + ResponseHeader header = 1; + GlobalGCBarrierInfo deleted_barrier_info = 2; +} + +message GlobalGCBarrierInfo { + string barrier_id = 1; + uint64 barrier_ts = 2; + int64 ttl_seconds = 3; +} + +message GetGCStateRequest { + RequestHeader header = 1; + + KeyspaceScope keyspace_scope = 2; + bool exclude_gc_barriers = 3; +} + +message GCState { + KeyspaceScope keyspace_scope = 1; + + bool is_keyspace_level_gc = 2; + uint64 txn_safe_point = 3; + uint64 gc_safe_point = 4; + repeated GCBarrierInfo gc_barriers = 5; +} + +message GetGCStateResponse { + ResponseHeader header = 1; + + GCState gc_state = 2; +} + +message GetAllKeyspacesGCStatesRequest { + RequestHeader header = 1; + + bool exclude_gc_barriers = 2; + bool exclude_global_gc_barriers = 3; +} + +message GetAllKeyspacesGCStatesResponse { + ResponseHeader header = 1; + + repeated GCState gc_states = 2; + repeated GlobalGCBarrierInfo global_gc_barriers = 3; } message RegionStat { - // Bytes read/written during this period. - uint64 bytes_written = 1; - uint64 bytes_read = 2; - // Keys read/written during this period. - uint64 keys_written = 3; - uint64 keys_read = 4; + // Bytes read/written during this period. + uint64 bytes_written = 1; + uint64 bytes_read = 2; + // Keys read/written during this period. + uint64 keys_written = 3; + uint64 keys_read = 4; } -message SyncRegionRequest{ - RequestHeader header = 1; - Member member = 2; - // the follower PD will use the start index to locate historical changes - // that require synchronization. - uint64 start_index = 3; +message SyncRegionRequest { + RequestHeader header = 1; + Member member = 2; + // the follower PD will use the start index to locate historical changes + // that require synchronization. + uint64 start_index = 3; } message PeersStats { - repeated PeerStats peers = 1; + repeated PeerStats peers = 1; } message Peers { - repeated metapb.Peer peers = 1; + repeated metapb.Peer peers = 1; } -message SyncRegionResponse{ - ResponseHeader header = 1; - // the leader PD will send the repsonds include - // changed regions records and the index of the first record. - repeated metapb.Region regions = 2; - uint64 start_index = 3; - repeated RegionStat region_stats = 4; - repeated metapb.Peer region_leaders = 5; - // the buckets informations without stats. - repeated metapb.Buckets buckets = 6; - repeated PeersStats down_peers = 16; - repeated Peers pending_peers = 17; +message SyncRegionResponse { + ResponseHeader header = 1; + // the leader PD will send the repsonds include + // changed regions records and the index of the first record. + repeated metapb.Region regions = 2; + uint64 start_index = 3; + repeated RegionStat region_stats = 4; + repeated metapb.Peer region_leaders = 5; + // the buckets informations without stats. + repeated metapb.Buckets buckets = 6; + repeated PeersStats down_peers = 16; + repeated Peers pending_peers = 17; } message GetOperatorRequest { - RequestHeader header = 1; - uint64 region_id = 2; + RequestHeader header = 1; + uint64 region_id = 2; } enum OperatorStatus { - SUCCESS = 0; - TIMEOUT = 1; - CANCEL = 2; - REPLACE = 3; - RUNNING = 4; + SUCCESS = 0; + TIMEOUT = 1; + CANCEL = 2; + REPLACE = 3; + RUNNING = 4; } message GetOperatorResponse { - ResponseHeader header = 1; - uint64 region_id = 2; - bytes desc = 3; - OperatorStatus status = 4; - bytes kind = 5; + ResponseHeader header = 1; + uint64 region_id = 2; + bytes desc = 3; + OperatorStatus status = 4; + bytes kind = 5; } message SyncMaxTSRequest { - RequestHeader header = 1; - Timestamp max_ts = 2; - // If skip_check is true, the sync will try to write the max_ts without checking whether it's bigger. - bool skip_check = 3; + RequestHeader header = 1; + Timestamp max_ts = 2; + // If skip_check is true, the sync will try to write the max_ts without checking whether it's bigger. + bool skip_check = 3; } message SyncMaxTSResponse { - ResponseHeader header = 1; - Timestamp max_local_ts = 2; - repeated string synced_dcs = 3; + ResponseHeader header = 1; + Timestamp max_local_ts = 2; + repeated string synced_dcs = 3; } message SplitRegionsRequest { - RequestHeader header = 1; - repeated bytes split_keys = 2; - uint64 retry_limit = 3; + RequestHeader header = 1; + repeated bytes split_keys = 2; + uint64 retry_limit = 3; } message SplitRegionsResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; - repeated uint64 regions_id = 3; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 regions_id = 3; } message SplitAndScatterRegionsRequest { - RequestHeader header = 1; - repeated bytes split_keys = 2; - string group = 3; - uint64 retry_limit = 4; + RequestHeader header = 1; + repeated bytes split_keys = 2; + string group = 3; + uint64 retry_limit = 4; } message SplitAndScatterRegionsResponse { - ResponseHeader header = 1; - uint64 split_finished_percentage = 2; - uint64 scatter_finished_percentage = 3; - repeated uint64 regions_id = 4; + ResponseHeader header = 1; + uint64 split_finished_percentage = 2; + uint64 scatter_finished_percentage = 3; + repeated uint64 regions_id = 4; } message GetDCLocationInfoRequest { - RequestHeader header = 1; - string dc_location = 2; + RequestHeader header = 1; + string dc_location = 2; } message GetDCLocationInfoResponse { - ResponseHeader header = 1; - // suffix sign - int32 suffix = 2; - // max_ts will be included into this response if PD leader think the receiver needs, - // which it's set when the number of the max suffix bits changes. - Timestamp max_ts = 3; + ResponseHeader header = 1; + // suffix sign + int32 suffix = 2; + // max_ts will be included into this response if PD leader think the receiver needs, + // which it's set when the number of the max suffix bits changes. + Timestamp max_ts = 3; } message QueryStats { - uint64 GC = 1; - uint64 Get = 2; - uint64 Scan = 3; - uint64 Coprocessor = 4; - uint64 Delete = 5; - uint64 DeleteRange = 6; - uint64 Put = 7; - uint64 Prewrite = 8; - uint64 AcquirePessimisticLock = 9; - uint64 Commit = 10; - uint64 Rollback = 11; + uint64 GC = 1; + uint64 Get = 2; + uint64 Scan = 3; + uint64 Coprocessor = 4; + uint64 Delete = 5; + uint64 DeleteRange = 6; + uint64 Put = 7; + uint64 Prewrite = 8; + uint64 AcquirePessimisticLock = 9; + uint64 Commit = 10; + uint64 Rollback = 11; +} + +// CPU usage breakdown by kind. New kinds may be added in the future. +message CPUStats { + // UnifiedRead is the CPU usage of the unified read pool. + uint64 UnifiedRead = 1; + // Scheduler is the CPU usage of the scheduler pool, it contains `sched-pool`, `sched-high`, and `sched-pri`. + uint64 Scheduler = 2; } enum QueryKind { - Others = 0; - GC = 1; - Get = 2; - Scan = 3; - Coprocessor = 4; - Delete = 5; - DeleteRange = 6; - Put = 7; - Prewrite = 8; - AcquirePessimisticLock = 9; - Commit = 10; - Rollback = 11; + Others = 0; + GC = 1; + Get = 2; + Scan = 3; + Coprocessor = 4; + Delete = 5; + DeleteRange = 6; + Put = 7; + Prewrite = 8; + AcquirePessimisticLock = 9; + Commit = 10; + Rollback = 11; } message ReportBucketsRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.RegionEpoch region_epoch = 2; - metapb.Buckets buckets = 3; + metapb.RegionEpoch region_epoch = 2; + metapb.Buckets buckets = 3; } message ReportBucketsResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message ReportMinResolvedTsRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 store_id = 2; + uint64 store_id = 2; - uint64 min_resolved_ts = 3; + uint64 min_resolved_ts = 3; } message ReportMinResolvedTsResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message SetExternalTimestampRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint64 timestamp = 2; + uint64 timestamp = 2; } message SetExternalTimestampResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; } message GetExternalTimestampRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetExternalTimestampResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint64 timestamp = 2; + uint64 timestamp = 2; } message GetMinTSRequest { - RequestHeader header = 1; + RequestHeader header = 1; } message GetMinTSResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - Timestamp timestamp = 2; + Timestamp timestamp = 2; } diff --git a/proto/raft_cmdpb.proto b/proto/raft_cmdpb.proto index 155210b6..90121fb5 100644 --- a/proto/raft_cmdpb.proto +++ b/proto/raft_cmdpb.proto @@ -1,47 +1,53 @@ syntax = "proto3"; package raft_cmdpb; -import "metapb.proto"; -import "errorpb.proto"; import "eraftpb.proto"; -import "kvrpcpb.proto"; +import "errorpb.proto"; +import "gogoproto/gogo.proto"; import "import_sstpb.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "raft_serverpb.proto"; import "rustproto.proto"; -option (rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message GetRequest { - string cf = 1; - bytes key = 2; + string cf = 1; + bytes key = 2; } message GetResponse { - bytes value = 1; + bytes value = 1; } message PutRequest { - string cf = 1; - bytes key = 2; - bytes value = 3; + string cf = 1; + bytes key = 2; + bytes value = 3; } message PutResponse {} message DeleteRequest { - string cf = 1; - bytes key = 2; + string cf = 1; + bytes key = 2; } message DeleteResponse {} message DeleteRangeRequest { - string cf = 1; - bytes start_key = 2; - bytes end_key = 3; - bool notify_only = 4; + string cf = 1; + bytes start_key = 2; + bytes end_key = 3; + bool notify_only = 4; } message DeleteRangeResponse {} @@ -49,169 +55,169 @@ message DeleteRangeResponse {} message SnapRequest {} message SnapResponse { - metapb.Region region = 1; + metapb.Region region = 1; } message PrewriteRequest { - bytes key = 1; - bytes value = 2; - bytes lock = 3; + bytes key = 1; + bytes value = 2; + bytes lock = 3; } message PrewriteResponse {} message IngestSSTRequest { - import_sstpb.SSTMeta sst = 1; + import_sstpb.SSTMeta sst = 1; } message IngestSSTResponse {} message ReadIndexRequest { - // In replica read, leader uses start_ts and key_ranges to check memory locks. - uint64 start_ts = 1; - repeated kvrpcpb.KeyRange key_ranges = 2; + // In replica read, leader uses start_ts and key_ranges to check memory locks. + uint64 start_ts = 1; + repeated kvrpcpb.KeyRange key_ranges = 2; } -message ReadIndexResponse{ - uint64 read_index = 1; - // The memory lock blocking this read at the leader - kvrpcpb.LockInfo locked = 2; +message ReadIndexResponse { + uint64 read_index = 1; + // The memory lock blocking this read at the leader + kvrpcpb.LockInfo locked = 2; } enum CmdType { - Invalid = 0; - Get = 1; - Put = 3; - Delete = 4; - Snap = 5; - Prewrite = 6; - DeleteRange = 7; - IngestSST = 8; - ReadIndex = 9; + Invalid = 0; + Get = 1; + Put = 3; + Delete = 4; + Snap = 5; + Prewrite = 6; + DeleteRange = 7; + IngestSST = 8; + ReadIndex = 9; } message Request { - CmdType cmd_type = 1; - GetRequest get = 2; - PutRequest put = 4; - DeleteRequest delete = 5; - SnapRequest snap = 6; - PrewriteRequest prewrite = 7; - DeleteRangeRequest delete_range = 8; - IngestSSTRequest ingest_sst = 9; - ReadIndexRequest read_index = 10; + CmdType cmd_type = 1; + GetRequest get = 2; + PutRequest put = 4; + DeleteRequest delete = 5; + SnapRequest snap = 6; + PrewriteRequest prewrite = 7; + DeleteRangeRequest delete_range = 8; + IngestSSTRequest ingest_sst = 9; + ReadIndexRequest read_index = 10; } message Response { - CmdType cmd_type = 1; - GetResponse get = 2; - PutResponse put = 4; - DeleteResponse delete = 5; - SnapResponse snap = 6; - PrewriteResponse prewrite = 7; - DeleteRangeResponse delte_range = 8; - IngestSSTResponse ingest_sst = 9; - ReadIndexResponse read_index = 10; + CmdType cmd_type = 1; + GetResponse get = 2; + PutResponse put = 4; + DeleteResponse delete = 5; + SnapResponse snap = 6; + PrewriteResponse prewrite = 7; + DeleteRangeResponse delte_range = 8; + IngestSSTResponse ingest_sst = 9; + ReadIndexResponse read_index = 10; } message ChangePeerRequest { - // This can be only called in internal RaftStore now. - eraftpb.ConfChangeType change_type = 1; - metapb.Peer peer = 2; + // This can be only called in internal RaftStore now. + eraftpb.ConfChangeType change_type = 1; + metapb.Peer peer = 2; } message ChangePeerResponse { - metapb.Region region = 1; + metapb.Region region = 1; } message ChangePeerV2Request { - repeated ChangePeerRequest changes = 1; + repeated ChangePeerRequest changes = 1; } message ChangePeerV2Response { - metapb.Region region = 1; + metapb.Region region = 1; } message SplitRequest { - // This can be only called in internal RaftStore now. - // The split_key must be in the been splitting region. - bytes split_key = 1; - // We split the region into two, first uses the origin - // parent region id, and the second uses the new_region_id. - // We must guarantee that the new_region_id is global unique. - uint64 new_region_id = 2; - // The peer ids for the new split region. - repeated uint64 new_peer_ids = 3; - // If true, right region derive the origin region_id, - // left region use new_region_id. - // Will be ignored in batch split, use `BatchSplitRequest::right_derive` instead. - bool right_derive = 4 [deprecated=true]; - // It should be false iff the region split by user key such as split table or create partion table etc, - // the new region's will not share the source region size, so it's region size is zero. - // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, - // the new region's size will share the origin region, so it's region size is half of the source region. - bool share_source_region_size = 5; + // This can be only called in internal RaftStore now. + // The split_key must be in the been splitting region. + bytes split_key = 1; + // We split the region into two, first uses the origin + // parent region id, and the second uses the new_region_id. + // We must guarantee that the new_region_id is global unique. + uint64 new_region_id = 2; + // The peer ids for the new split region. + repeated uint64 new_peer_ids = 3; + // If true, right region derive the origin region_id, + // left region use new_region_id. + // Will be ignored in batch split, use `BatchSplitRequest::right_derive` instead. + bool right_derive = 4 [deprecated = true]; + // It should be false iff the region split by user key such as split table or create partion table etc, + // the new region's will not share the source region size, so it's region size is zero. + // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, + // the new region's size will share the origin region, so it's region size is half of the source region. + bool share_source_region_size = 5; } message SplitResponse { - metapb.Region left = 1; - metapb.Region right = 2; + metapb.Region left = 1; + metapb.Region right = 2; } message BatchSplitRequest { - repeated SplitRequest requests = 1; - // If true, the last region derive the origin region_id, - // other regions use new ids. - bool right_derive = 2; - // It should be false iff the region split by user key such as split table or create partion table etc, - // the new region's will not share the source region size, so it's region size is zero. - // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, - // the new region's size will share the origin region, so it's region size is half of the source region. - bool share_source_region_size = 3; + repeated SplitRequest requests = 1; + // If true, the last region derive the origin region_id, + // other regions use new ids. + bool right_derive = 2; + // It should be false iff the region split by user key such as split table or create partion table etc, + // the new region's will not share the source region size, so it's region size is zero. + // It should be true iff the region's load reaches the threshold such as size, keys, load check etc, + // the new region's size will share the origin region, so it's region size is half of the source region. + bool share_source_region_size = 3; } message BatchSplitResponse { - repeated metapb.Region regions = 1; + repeated metapb.Region regions = 1; } message CompactLogRequest { - uint64 compact_index = 1; - uint64 compact_term = 2; - uint64 voter_replicated_index = 3; + uint64 compact_index = 1; + uint64 compact_term = 2; + uint64 voter_replicated_index = 3; } message CompactLogResponse {} message TransferLeaderRequest { - metapb.Peer peer = 1; - repeated metapb.Peer peers = 2; + metapb.Peer peer = 1; + repeated metapb.Peer peers = 2; } message TransferLeaderResponse {} message ComputeHashRequest { - bytes context = 1; + bytes context = 1; } message VerifyHashRequest { - uint64 index = 1; - bytes hash = 2; + uint64 index = 1; + bytes hash = 2; - bytes context = 3; + bytes context = 3; } message VerifyHashResponse {} message PrepareMergeRequest { - uint64 min_index = 1; - metapb.Region target = 2; + uint64 min_index = 1; + metapb.Region target = 2; } message PrepareMergeResponse {} message PrepareFlashbackRequest { - // The start_ts that the current flashback progress is using. - uint64 start_ts = 1; + // The start_ts that the current flashback progress is using. + uint64 start_ts = 1; } message PrepareFlashbackResponse {} @@ -221,106 +227,107 @@ message FinishFlashbackRequest {} message FinishFlashbackResponse {} message CommitMergeRequest { - metapb.Region source = 1; - uint64 commit = 2; - repeated eraftpb.Entry entries = 3; - // Used in v2. When it's present, `source` and `commit` will not be set. - raft_serverpb.RegionLocalState source_state = 4; + metapb.Region source = 1; + uint64 commit = 2; + repeated eraftpb.Entry entries = 3; + // Used in v2. When it's present, `source` and `commit` will not be set. + raft_serverpb.RegionLocalState source_state = 4; + bytes source_meta = 100; - reserved 100 to 200; + reserved 101 to 200; } message CommitMergeResponse {} message RollbackMergeRequest { - uint64 commit = 1; + uint64 commit = 1; } message RollbackMergeResponse {} message SwitchWitnessRequest { - uint64 peer_id = 1; - bool is_witness = 2; + uint64 peer_id = 1; + bool is_witness = 2; } message BatchSwitchWitnessRequest { - repeated SwitchWitnessRequest switch_witnesses = 1; + repeated SwitchWitnessRequest switch_witnesses = 1; } message BatchSwitchWitnessResponse {} message UpdateGcPeerRequest { - repeated uint64 peer_id = 1; + repeated uint64 peer_id = 1; } enum AdminCmdType { - InvalidAdmin = 0; - ChangePeer = 1; - // Use `BatchSplit` instead. - Split = 2 [deprecated=true]; - CompactLog = 3; - TransferLeader = 4; - ComputeHash = 5; - VerifyHash = 6; - PrepareMerge = 7; - CommitMerge = 8; - RollbackMerge = 9; - BatchSplit = 10; - ChangePeerV2 = 11; - PrepareFlashback = 12; - FinishFlashback = 13; - BatchSwitchWitness = 14; - // Command that updates RegionLocalState.gc_peers - UpdateGcPeer = 15; + InvalidAdmin = 0; + ChangePeer = 1; + // Use `BatchSplit` instead. + Split = 2 [deprecated = true]; + CompactLog = 3; + TransferLeader = 4; + ComputeHash = 5; + VerifyHash = 6; + PrepareMerge = 7; + CommitMerge = 8; + RollbackMerge = 9; + BatchSplit = 10; + ChangePeerV2 = 11; + PrepareFlashback = 12; + FinishFlashback = 13; + BatchSwitchWitness = 14; + // Command that updates RegionLocalState.gc_peers + UpdateGcPeer = 15; } message AdminRequest { - AdminCmdType cmd_type = 1; - ChangePeerRequest change_peer = 2; - SplitRequest split = 3 [deprecated=true]; - CompactLogRequest compact_log = 4; - TransferLeaderRequest transfer_leader = 5; - VerifyHashRequest verify_hash = 6; - PrepareMergeRequest prepare_merge = 7; - CommitMergeRequest commit_merge = 8; - RollbackMergeRequest rollback_merge = 9; - BatchSplitRequest splits = 10; - ChangePeerV2Request change_peer_v2 = 11; - ComputeHashRequest compute_hash = 12; - PrepareFlashbackRequest prepare_flashback = 13; - FinishFlashbackRequest finish_flashback = 14; - BatchSwitchWitnessRequest switch_witnesses = 15; - UpdateGcPeerRequest update_gc_peers = 16; + AdminCmdType cmd_type = 1; + ChangePeerRequest change_peer = 2; + SplitRequest split = 3 [deprecated = true]; + CompactLogRequest compact_log = 4; + TransferLeaderRequest transfer_leader = 5; + VerifyHashRequest verify_hash = 6; + PrepareMergeRequest prepare_merge = 7; + CommitMergeRequest commit_merge = 8; + RollbackMergeRequest rollback_merge = 9; + BatchSplitRequest splits = 10; + ChangePeerV2Request change_peer_v2 = 11; + ComputeHashRequest compute_hash = 12; + PrepareFlashbackRequest prepare_flashback = 13; + FinishFlashbackRequest finish_flashback = 14; + BatchSwitchWitnessRequest switch_witnesses = 15; + UpdateGcPeerRequest update_gc_peers = 16; } message AdminResponse { - AdminCmdType cmd_type = 1; - ChangePeerResponse change_peer = 2; - SplitResponse split = 3 [deprecated=true]; - CompactLogResponse compact_log = 4; - TransferLeaderResponse transfer_leader = 5; - VerifyHashResponse verify_hash = 6; - PrepareMergeResponse prepare_merge = 7; - CommitMergeResponse commit_merge = 8; - RollbackMergeResponse rollback_merge = 9; - BatchSplitResponse splits = 10; - ChangePeerV2Response change_peer_v2 = 11; - PrepareFlashbackResponse prepare_flashback = 12; - FinishFlashbackResponse finish_flashback = 13; - BatchSwitchWitnessResponse switch_witnesses = 14; - // UpdateGcPeer doesn't need to be responded. Avoid consuming a tag number. + AdminCmdType cmd_type = 1; + ChangePeerResponse change_peer = 2; + SplitResponse split = 3 [deprecated = true]; + CompactLogResponse compact_log = 4; + TransferLeaderResponse transfer_leader = 5; + VerifyHashResponse verify_hash = 6; + PrepareMergeResponse prepare_merge = 7; + CommitMergeResponse commit_merge = 8; + RollbackMergeResponse rollback_merge = 9; + BatchSplitResponse splits = 10; + ChangePeerV2Response change_peer_v2 = 11; + PrepareFlashbackResponse prepare_flashback = 12; + FinishFlashbackResponse finish_flashback = 13; + BatchSwitchWitnessResponse switch_witnesses = 14; + // UpdateGcPeer doesn't need to be responded. Avoid consuming a tag number. } // For get the leader of the region. message RegionLeaderRequest {} message RegionLeaderResponse { - metapb.Peer leader = 1; + metapb.Peer leader = 1; } // For getting more information of the region. // We add some admin operations (ChangePeer, Split...) into the pb job list, -// then pd server will peek the first one, handle it and then pop it from the job lib. +// then pd server will peek the first one, handle it and then pop it from the job lib. // But sometimes, the pd server may crash before popping. When another pd server // starts and finds the job is running but not finished, it will first check whether // the raft server already has handled this job. @@ -330,75 +337,78 @@ message RegionLeaderResponse { message RegionDetailRequest {} message RegionDetailResponse { - metapb.Region region = 1; - metapb.Peer leader = 2; + metapb.Region region = 1; + metapb.Peer leader = 2; } - enum StatusCmdType { - InvalidStatus = 0; - RegionLeader = 1; - RegionDetail = 2; + InvalidStatus = 0; + RegionLeader = 1; + RegionDetail = 2; } message StatusRequest { - StatusCmdType cmd_type = 1; - RegionLeaderRequest region_leader = 2; - RegionDetailRequest region_detail = 3; + StatusCmdType cmd_type = 1; + RegionLeaderRequest region_leader = 2; + RegionDetailRequest region_detail = 3; } message StatusResponse { - StatusCmdType cmd_type = 1; - RegionLeaderResponse region_leader = 2; - RegionDetailResponse region_detail = 3; + StatusCmdType cmd_type = 1; + RegionLeaderResponse region_leader = 2; + RegionDetailResponse region_detail = 3; } message RaftRequestHeader { - uint64 region_id = 1; - metapb.Peer peer = 2; - // true for read linearization - bool read_quorum = 3; - // 16 bytes, to distinguish request. - bytes uuid = 4; - - metapb.RegionEpoch region_epoch = 5; - uint64 term = 6; - - bool sync_log = 7; - bool replica_read = 8; - - // Read requests can be responsed directly after the Raft applys to `applied_index`. - uint64 applied_index = 9; - - // Custom flags for this raft request. - uint64 flags = 10; - bytes flag_data = 11; - - kvrpcpb.CommandPri priority = 12; - string resource_group_name = 13; + uint64 region_id = 1; + metapb.Peer peer = 2; + // true for read linearization + bool read_quorum = 3; + // 16 bytes, to distinguish request. + bytes uuid = 4; + + metapb.RegionEpoch region_epoch = 5; + uint64 term = 6; + + bool sync_log = 7; + bool replica_read = 8; + + // Read requests can be responsed directly after the Raft applys to `applied_index`. + uint64 applied_index = 9; + + // Custom flags for this raft request. + uint64 flags = 10; + bytes flag_data = 11; + + kvrpcpb.CommandPri priority = 12; + string resource_group_name = 13; } message RaftResponseHeader { - errorpb.Error error = 1; - bytes uuid = 2; - uint64 current_term = 3; + errorpb.Error error = 1; + bytes uuid = 2; + uint64 current_term = 3; +} + +message CustomRequest { + bytes data = 1; } message RaftCmdRequest { - RaftRequestHeader header = 1; - // We can't enclose normal requests and administrator request - // at same time. - repeated Request requests = 2; - AdminRequest admin_request = 3; - StatusRequest status_request = 4; + RaftRequestHeader header = 1; + // We can't enclose normal requests and administrator request + // at same time. + repeated Request requests = 2; + AdminRequest admin_request = 3; + StatusRequest status_request = 4; + CustomRequest custom_request = 5; - reserved 5; - reserved 100 to 200; + reserved 100 to 200; } message RaftCmdResponse { - RaftResponseHeader header = 1; - repeated Response responses = 2; - AdminResponse admin_response = 3; - StatusResponse status_response = 4; + RaftResponseHeader header = 1; + repeated Response responses = 2; + AdminResponse admin_response = 3; + StatusResponse status_response = 4; } diff --git a/proto/raft_serverpb.proto b/proto/raft_serverpb.proto index 49d513ce..c212e5d9 100644 --- a/proto/raft_serverpb.proto +++ b/proto/raft_serverpb.proto @@ -1,80 +1,89 @@ syntax = "proto3"; package raft_serverpb; -import "eraftpb.proto"; -import "metapb.proto"; -import "kvrpcpb.proto"; import "disk_usage.proto"; import "encryptionpb.proto"; +import "eraftpb.proto"; +import "gogoproto/gogo.proto"; +import "kvrpcpb.proto"; +import "metapb.proto"; import "rustproto.proto"; -option (rustproto.lite_runtime_all) = true; - option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; message RaftMessage { - uint64 region_id = 1; - metapb.Peer from_peer = 2; - metapb.Peer to_peer = 3; - eraftpb.Message message = 4; - metapb.RegionEpoch region_epoch = 5; - // true means to_peer is a tombstone peer and it should remove itself. - bool is_tombstone = 6; - // Region key range [start_key, end_key). - bytes start_key = 7; - bytes end_key = 8; - // If it has value, to_peer should be removed if merge is never going to complete. - metapb.Region merge_target = 9; - ExtraMessage extra_msg = 10; - bytes extra_ctx = 11; - - disk_usage.DiskUsage disk_usage = 12; + uint64 region_id = 1; + metapb.Peer from_peer = 2; + metapb.Peer to_peer = 3; + eraftpb.Message message = 4; + metapb.RegionEpoch region_epoch = 5; + // true means to_peer is a tombstone peer and it should remove itself. + bool is_tombstone = 6; + // Region key range [start_key, end_key). + bytes start_key = 7; + bytes end_key = 8; + // If it has value, to_peer should be removed if merge is never going to complete. + metapb.Region merge_target = 9; + ExtraMessage extra_msg = 10; + bytes extra_ctx = 11; + + disk_usage.DiskUsage disk_usage = 12; } message RaftTruncatedState { - uint64 index = 1; - uint64 term = 2; + uint64 index = 1; + uint64 term = 2; } message SnapshotCFFile { - string cf = 1; - uint64 size = 2; - uint32 checksum = 3; + string cf = 1; + uint64 size = 2; + uint32 checksum = 3; } message SnapshotMeta { - repeated SnapshotCFFile cf_files = 1; - // true means this snapshot is triggered for load balance - bool for_balance = 2; - // true means this is an empty snapshot for witness - bool for_witness = 3; - // the timestamp second to generate snapshot - uint64 start = 4; - // the duration of generating snapshot - uint64 generate_duration_sec = 5; - // the path of the tablet snapshot, it should only be used for v1 to receive - // snapshot from v2 - string tablet_snap_path = 6; - // A hint of the latest commit index on leader when sending snapshot. - // It should only be used for v2 to send snapshot to v1. - // See https://github.com/pingcap/tiflash/issues/7568 - uint64 commit_index_hint = 7; + repeated SnapshotCFFile cf_files = 1; + // true means this snapshot is triggered for load balance + bool for_balance = 2; + // true means this is an empty snapshot for witness + bool for_witness = 3; + // the timestamp second to generate snapshot + uint64 start = 4; + // the duration of generating snapshot + uint64 generate_duration_sec = 5; + // the path of the tablet snapshot, it should only be used for v1 to receive + // snapshot from v2 + string tablet_snap_path = 6; + // A hint of the latest commit index on leader when sending snapshot. + // It should only be used for v2 to send snapshot to v1. + // See https://github.com/pingcap/tiflash/issues/7568 + uint64 commit_index_hint = 7; + + // Reserved for a downstream fork + reserved 1000 to 1199; } message SnapshotChunk { - RaftMessage message = 1; - bytes data = 2; + RaftMessage message = 1; + bytes data = 2; } message Done {} message TabletSnapshotFileMeta { - uint64 file_size = 1; - string file_name = 2; - // Some block data. Unencrypted. - bytes head_chunk = 3; - // trailing data including checksum. Unencrypted. - bytes trailing_chunk = 4; + uint64 file_size = 1; + string file_name = 2; + // Some block data. Unencrypted. + bytes head_chunk = 3; + // trailing data including checksum. Unencrypted. + bytes trailing_chunk = 4; } // Snapshot preview for server to decide whether skip some files. @@ -82,215 +91,230 @@ message TabletSnapshotFileMeta { // keep sending specified files. Only SST files can be skipped, all // other files should always be sent. message TabletSnapshotPreview { - repeated TabletSnapshotFileMeta metas = 1; - // There may be too many metas, use a flag to indicate all metas - // are sent. - bool end = 2; + repeated TabletSnapshotFileMeta metas = 1; + // There may be too many metas, use a flag to indicate all metas + // are sent. + bool end = 2; } message TabletSnapshotFileChunk { - uint64 file_size = 1; - string file_name = 2; - // Encrypted. - bytes data = 3; - // Initial vector if encryption is enabled. - bytes iv = 4; - encryptionpb.DataKey key = 5; + uint64 file_size = 1; + string file_name = 2; + // Encrypted. + bytes data = 3; + // Initial vector if encryption is enabled. + bytes iv = 4; + encryptionpb.DataKey key = 5; } message TabletSnapshotHead { - RaftMessage message = 1; - bool use_cache = 2; + RaftMessage message = 1; + bool use_cache = 2; } message TabletSnapshotEnd { - // Checksum of all data sent in `TabletSnapshotFileChunk.data` and - // `TabletSnapshotFileChunk.file_name`. - uint64 checksum = 1; + // Checksum of all data sent in `TabletSnapshotFileChunk.data` and + // `TabletSnapshotFileChunk.file_name`. + uint64 checksum = 1; } message TabletSnapshotRequest { - oneof payload { - TabletSnapshotHead head = 1; - TabletSnapshotPreview preview = 2; - TabletSnapshotFileChunk chunk = 3; - TabletSnapshotEnd end = 4; - } + oneof payload { + TabletSnapshotHead head = 1; + TabletSnapshotPreview preview = 2; + TabletSnapshotFileChunk chunk = 3; + TabletSnapshotEnd end = 4; + } } message AcceptedSnapshotFiles { - repeated string file_name = 1; + repeated string file_name = 1; } message TabletSnapshotResponse { - AcceptedSnapshotFiles files = 1; + AcceptedSnapshotFiles files = 1; } message KeyValue { - bytes key = 1; - bytes value = 2; + bytes key = 1; + bytes value = 2; } message RaftSnapshotData { - metapb.Region region = 1; - uint64 file_size = 2; - repeated KeyValue data = 3; - uint64 version = 4; - SnapshotMeta meta = 5; - repeated metapb.Peer removed_records = 6; - repeated MergedRecord merged_records = 7; + metapb.Region region = 1; + uint64 file_size = 2; + repeated KeyValue data = 3; + uint64 version = 4; + SnapshotMeta meta = 5; + repeated metapb.Peer removed_records = 6; + repeated MergedRecord merged_records = 7; } message StoreIdent { - uint64 cluster_id = 1; - uint64 store_id = 2; - kvrpcpb.APIVersion api_version = 3; + uint64 cluster_id = 1; + uint64 store_id = 2; + kvrpcpb.APIVersion api_version = 3; } message StoreRecoverState { - // Used for TiKV start recovery when WAL of KVDB was disabled. - // TiKV may read all relations between seqno and raft log index, and replay - // all raft logs which corresponding seqno smaller than the seqno here. - // After TiKV replays all raft logs and flushed KV data, the seqno here must - // be updated. - uint64 seqno = 1; + // Used for TiKV start recovery when WAL of KVDB was disabled. + // TiKV may read all relations between seqno and raft log index, and replay + // all raft logs which corresponding seqno smaller than the seqno here. + // After TiKV replays all raft logs and flushed KV data, the seqno here must + // be updated. + uint64 seqno = 1; } message RaftLocalState { - eraftpb.HardState hard_state = 1; - uint64 last_index = 2; + eraftpb.HardState hard_state = 1; + uint64 last_index = 2; } message RaftApplyState { - uint64 applied_index = 1; - uint64 last_commit_index = 3; - uint64 commit_index = 4; - uint64 commit_term = 5; - RaftTruncatedState truncated_state = 2; + uint64 applied_index = 1; + uint64 last_commit_index = 3; + uint64 commit_index = 4; + uint64 commit_term = 5; + RaftTruncatedState truncated_state = 2; } enum PeerState { - Normal = 0; - Applying = 1; - Tombstone = 2; - Merging = 3; - // Currently used for witness to non-witness conversion: When a witness - // has just become a non-witness, we need to set and persist this state, - // so that when the service restarts before applying snapshot, we can - // actively request snapshot when initializing this peer. - Unavailable = 4; + Normal = 0; + Applying = 1; + Tombstone = 2; + Merging = 3; + // Currently used for witness to non-witness conversion: When a witness + // has just become a non-witness, we need to set and persist this state, + // so that when the service restarts before applying snapshot, we can + // actively request snapshot when initializing this peer. + Unavailable = 4; } message MergeState { - uint64 min_index = 1; - metapb.Region target = 2; - uint64 commit = 3; + uint64 min_index = 1; + metapb.Region target = 2; + uint64 commit = 3; } message MergedRecord { - uint64 source_region_id = 1; - metapb.RegionEpoch source_epoch = 2; - // Peers of source region when merge is committed. - repeated metapb.Peer source_peers = 3; - // Removed peers (by confchange) of source region when merge is committed. - repeated metapb.Peer source_removed_records = 9; - uint64 target_region_id = 4; - metapb.RegionEpoch target_epoch = 5; - repeated metapb.Peer target_peers = 6; - // Commit merge index. - uint64 index = 7; - // Prepare merge index. - uint64 source_index = 8; + uint64 source_region_id = 1; + metapb.RegionEpoch source_epoch = 2; + // Peers of source region when merge is committed. + repeated metapb.Peer source_peers = 3; + // Removed peers (by confchange) of source region when merge is committed. + repeated metapb.Peer source_removed_records = 9; + uint64 target_region_id = 4; + metapb.RegionEpoch target_epoch = 5; + repeated metapb.Peer target_peers = 6; + // Commit merge index. + uint64 index = 7; + // Prepare merge index. + uint64 source_index = 8; } message RegionLocalState { - PeerState state = 1; - metapb.Region region = 2; - MergeState merge_state = 3; - // The apply index corresponding to the storage when it's initialized. - uint64 tablet_index = 4; - // Raft doesn't guarantee peer will be removed in the end. In v1, peer finds - // out its destiny by logs or broadcast; in v2, leader is responsible to - // ensure removed peers are destroyed. - // Note: only peers who has been part of this region can be in this list. - repeated metapb.Peer removed_records = 5; - // Merged peer can't be deleted like gc peers. Instead, leader needs to - // query target peer to decide whether source peer can be destroyed. - repeated MergedRecord merged_records = 6; + PeerState state = 1; + metapb.Region region = 2; + MergeState merge_state = 3; + // The apply index corresponding to the storage when it's initialized. + uint64 tablet_index = 4; + // Raft doesn't guarantee peer will be removed in the end. In v1, peer finds + // out its destiny by logs or broadcast; in v2, leader is responsible to + // ensure removed peers are destroyed. + // Note: only peers who has been part of this region can be in this list. + repeated metapb.Peer removed_records = 5; + // Merged peer can't be deleted like gc peers. Instead, leader needs to + // query target peer to decide whether source peer can be destroyed. + repeated MergedRecord merged_records = 6; } message RegionSequenceNumberRelation { - uint64 region_id = 1; - uint64 sequence_number = 2; - RaftApplyState apply_state = 3; - RegionLocalState region_state = 4; + uint64 region_id = 1; + uint64 sequence_number = 2; + RaftApplyState apply_state = 3; + RegionLocalState region_state = 4; } message AvailabilityContext { - uint64 from_region_id = 1; - metapb.RegionEpoch from_region_epoch = 2; - bool unavailable = 3; - bool trimmed = 4; + uint64 from_region_id = 1; + metapb.RegionEpoch from_region_epoch = 2; + bool unavailable = 3; + bool trimmed = 4; } enum ExtraMessageType { - MsgRegionWakeUp = 0; - MsgWantRollbackMerge = 1; - MsgCheckStalePeer = 2; - MsgCheckStalePeerResponse = 3; - // If leader is going to sleep, it will send requests to all its followers - // to make sure they all agree to sleep. - MsgHibernateRequest = 4; - MsgHibernateResponse = 5; - MsgRejectRaftLogCausedByMemoryUsage = 6; - MsgAvailabilityRequest = 7; - MsgAvailabilityResponse = 8; - MsgVoterReplicatedIndexRequest = 9; - MsgVoterReplicatedIndexResponse = 10; - // Message means that `from` is tombstone. Leader can then update removed_records. - MsgGcPeerRequest = 11; - MsgGcPeerResponse = 12; - MsgFlushMemtable = 13; - MsgRefreshBuckets = 14; + MsgRegionWakeUp = 0; + MsgWantRollbackMerge = 1; + MsgCheckStalePeer = 2; + MsgCheckStalePeerResponse = 3; + // If leader is going to sleep, it will send requests to all its followers + // to make sure they all agree to sleep. + MsgHibernateRequest = 4; + MsgHibernateResponse = 5; + MsgRejectRaftLogCausedByMemoryUsage = 6; + MsgAvailabilityRequest = 7; + MsgAvailabilityResponse = 8; + MsgVoterReplicatedIndexRequest = 9; + MsgVoterReplicatedIndexResponse = 10; + // Message means that `from` is tombstone. Leader can then update removed_records. + MsgGcPeerRequest = 11; + MsgGcPeerResponse = 12; + MsgFlushMemtable = 13; + MsgRefreshBuckets = 14; + // Messages for the snapshot gen precheck process. + MsgSnapGenPrecheckRequest = 15; + MsgSnapGenPrecheckResponse = 16; + // Used in transfer leader process for leader to inform follower to load the + // region into in-memory engine if the relevant region is cached. + MsgPreLoadRegionRequest = 17; + // Used in transfer leader process for follower to inform leader the completes + // of the region cache + MsgPreLoadRegionResponse = 18; } message FlushMemtable { - uint64 region_id = 1; + uint64 region_id = 1; } message RefreshBuckets { - uint64 version = 1; - repeated bytes keys = 2; - repeated uint64 sizes = 3; + uint64 version = 1; + repeated bytes keys = 2; + repeated uint64 sizes = 3; } message CheckGcPeer { - // The region ID who triggers the check and wait for report. It should be - // the ID of RaftMessage.from. - uint64 from_region_id = 1; - // The region ID to be checked if should be destroyed. - uint64 check_region_id = 2; - // The epoch of the region to be checked. - metapb.RegionEpoch check_region_epoch = 3; - // The peer to be checked. - metapb.Peer check_peer = 4; + // The region ID who triggers the check and wait for report. It should be + // the ID of RaftMessage.from. + uint64 from_region_id = 1; + // The region ID to be checked if should be destroyed. + uint64 check_region_id = 2; + // The epoch of the region to be checked. + metapb.RegionEpoch check_region_epoch = 3; + // The peer to be checked. + metapb.Peer check_peer = 4; } message ExtraMessage { - ExtraMessageType type = 1; - // It's merge related index. In `WantRollbackMerge`, it's prepare merge index. In - // `MsgGcPeerRequest`, it's the commit merge index. In `MsgVoterReplicatedIndexRequest` - // it's the voter_replicated_index. - uint64 index = 2; - // In `MsgCheckStalePeerResponse`, it's the peers that receiver can continue to query. - repeated metapb.Peer check_peers = 3; - bool wait_data = 4; - // Flag for forcely wake up hibernate regions if true. - bool forcely_awaken = 5; - CheckGcPeer check_gc_peer = 6; - FlushMemtable flush_memtable = 7; - // Used by `MsgAvailabilityRequest` and `MsgAvailabilityResponse` in v2. - AvailabilityContext availability_context = 8; - // notice the peer to refresh buckets version - RefreshBuckets refresh_buckets = 9; + ExtraMessageType type = 1; + // It's merge related index. In `WantRollbackMerge`, it's prepare merge index. In + // `MsgGcPeerRequest`, it's the commit merge index. In `MsgVoterReplicatedIndexRequest` + // it's the voter_replicated_index. + uint64 index = 2; + // In `MsgCheckStalePeerResponse`, it's the peers that receiver can continue to query. + repeated metapb.Peer check_peers = 3; + bool wait_data = 4; + // Flag for forcely wake up hibernate regions if true. + bool forcely_awaken = 5; + CheckGcPeer check_gc_peer = 6; + FlushMemtable flush_memtable = 7; + // Used by `MsgAvailabilityRequest` and `MsgAvailabilityResponse` in v2. + AvailabilityContext availability_context = 8; + // notice the peer to refresh buckets version + RefreshBuckets refresh_buckets = 9; + // snap_gen_precheck_passed is used to indicate the precheck result when + // a follower responds to a leader's snapshot gen precheck request. + bool snap_gen_precheck_passed = 10; + + // Reserved for a downstream fork + reserved 1000 to 1199; } diff --git a/proto/recoverdatapb.proto b/proto/recoverdatapb.proto index de7a1b54..ada1f2e6 100644 --- a/proto/recoverdatapb.proto +++ b/proto/recoverdatapb.proto @@ -4,65 +4,68 @@ package recover_data; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; // request to read region meata from a store message ReadRegionMetaRequest { - uint64 store_id = 1; + uint64 store_id = 1; } message Error { - string msg = 1; + string msg = 1; } message RegionMeta { - uint64 region_id = 1; - uint64 peer_id = 2; - uint64 last_log_term = 3; - uint64 last_index = 4; - uint64 commit_index = 5; - uint64 version = 6; - bool tombstone = 7; //reserved, it may be used in late phase for peer check - bytes start_key = 8; - bytes end_key = 9; + uint64 region_id = 1; + uint64 peer_id = 2; + uint64 last_log_term = 3; + uint64 last_index = 4; + uint64 commit_index = 5; + uint64 version = 6; + bool tombstone = 7; //reserved, it may be used in late phase for peer check + bytes start_key = 8; + bytes end_key = 9; } // command to store for recover region message RecoverRegionRequest { - uint64 region_id = 1; - bool as_leader = 2; // force region_id as leader - bool tombstone = 3; // set Peer to tombstoned in late phase + uint64 region_id = 1; + bool as_leader = 2; // force region_id as leader + bool tombstone = 3; // set Peer to tombstoned in late phase } message RecoverRegionResponse { - Error error = 1; - uint64 store_id = 2; + Error error = 1; + uint64 store_id = 2; } // wait apply to last index - message WaitApplyRequest { - uint64 store_id = 1; - } +message WaitApplyRequest { + uint64 store_id = 1; +} - message WaitApplyResponse { - Error error = 1; - } +message WaitApplyResponse { + Error error = 1; +} // resolve data by resolved_ts message ResolveKvDataRequest { - uint64 resolved_ts = 1; + uint64 resolved_ts = 1; } message ResolveKvDataResponse { - Error error = 1; - uint64 store_id = 2; - uint64 resolved_key_count = 3; // reserved for summary of restore - // cursor of delete key.commit_ts, reserved for progress of restore - // progress is (current_commit_ts - resolved_ts) / (backup_ts - resolved_ts) x 100% - uint64 current_commit_ts = 4; + Error error = 1; + uint64 store_id = 2; + uint64 resolved_key_count = 3; // reserved for summary of restore + // cursor of delete key.commit_ts, reserved for progress of restore + // progress is (current_commit_ts - resolved_ts) / (backup_ts - resolved_ts) x 100% + uint64 current_commit_ts = 4; } // a recovery workflow likes @@ -71,12 +74,12 @@ message ResolveKvDataResponse { // 3. BR wait all regions in tikv to apply to last index (no write during the recovery) // 4. BR resolved kv data service RecoverData { - // read region meta to ready region meta - rpc ReadRegionMeta(ReadRegionMetaRequest) returns (stream RegionMeta) {} - // execute the recovery command - rpc RecoverRegion(stream RecoverRegionRequest) returns (RecoverRegionResponse) {} - // wait all region apply to last index - rpc WaitApply(WaitApplyRequest) returns (WaitApplyResponse) {} - // execute delete data from kv db - rpc ResolveKvData(ResolveKvDataRequest) returns (stream ResolveKvDataResponse) {} -} \ No newline at end of file + // read region meta to ready region meta + rpc ReadRegionMeta(ReadRegionMetaRequest) returns (stream RegionMeta) {} + // execute the recovery command + rpc RecoverRegion(stream RecoverRegionRequest) returns (RecoverRegionResponse) {} + // wait all region apply to last index + rpc WaitApply(WaitApplyRequest) returns (WaitApplyResponse) {} + // execute delete data from kv db + rpc ResolveKvData(ResolveKvDataRequest) returns (stream ResolveKvDataResponse) {} +} diff --git a/proto/replication_modepb.proto b/proto/replication_modepb.proto index c60005b6..af31307b 100644 --- a/proto/replication_modepb.proto +++ b/proto/replication_modepb.proto @@ -1,62 +1,76 @@ syntax = "proto3"; package replication_modepb; +import "gogoproto/gogo.proto"; +import "rustproto.proto"; + +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; + enum ReplicationMode { - // The standard mode. Replicate logs to majority peer. - MAJORITY = 0; - // DR mode. Replicate logs among 2 DCs. - DR_AUTO_SYNC = 1; + // The standard mode. Replicate logs to majority peer. + MAJORITY = 0; + // DR mode. Replicate logs among 2 DCs. + DR_AUTO_SYNC = 1; } // The replication status sync from PD to TiKV. message ReplicationStatus { - ReplicationMode mode = 1; - DRAutoSync dr_auto_sync = 2; + ReplicationMode mode = 1; + DRAutoSync dr_auto_sync = 2; + + // Reserved for a downstream fork + reserved 1000 to 1199; } enum DRAutoSyncState { - // Raft logs need to sync between different DCs - SYNC = 0; - // Wait for switching to ASYNC. Stop sync raft logs between DCs. - ASYNC_WAIT = 1; - // Raft logs need to sync to majority peers - ASYNC = 2; - // Switching from ASYNC to SYNC mode - SYNC_RECOVER = 3; + // Raft logs need to sync between different DCs + SYNC = 0; + // Wait for switching to ASYNC. Stop sync raft logs between DCs. + ASYNC_WAIT = 1; + // Raft logs need to sync to majority peers + ASYNC = 2; + // Switching from ASYNC to SYNC mode + SYNC_RECOVER = 3; } // The status of dr-autosync mode. message DRAutoSync { - // The key of the label that used for distinguish different DC. - string label_key = 1; - DRAutoSyncState state = 2; - // Unique ID of the state, it increases after each state transfer. - uint64 state_id = 3; - // Duration to wait before switching to SYNC by force (in seconds) - int32 wait_sync_timeout_hint = 4; - // Stores should only sync messages with available stores when state is ASYNC or ASYNC_WAIT. - repeated uint64 available_stores = 5; - // Stores should forbid region split. - bool pause_region_split = 6; + // The key of the label that used for distinguish different DC. + string label_key = 1; + DRAutoSyncState state = 2; + // Unique ID of the state, it increases after each state transfer. + uint64 state_id = 3; + // Duration to wait before switching to SYNC by force (in seconds) + int32 wait_sync_timeout_hint = 4; + // Stores should only sync messages with available stores when state is ASYNC or ASYNC_WAIT. + repeated uint64 available_stores = 5; + // Stores should forbid region split. + bool pause_region_split = 6; } enum RegionReplicationState { - // The region's state is unknown - UNKNOWN = 0; - // Logs sync to majority peers - SIMPLE_MAJORITY = 1; - // Logs sync to different DCs - INTEGRITY_OVER_LABEL = 2; + // The region's state is unknown + UNKNOWN = 0; + // Logs sync to majority peers + SIMPLE_MAJORITY = 1; + // Logs sync to different DCs + INTEGRITY_OVER_LABEL = 2; } // The replication status sync from TiKV to PD. message RegionReplicationStatus { - RegionReplicationState state = 1; - // Unique ID of the state, it increases after each state transfer. - uint64 state_id = 2; + RegionReplicationState state = 1; + // Unique ID of the state, it increases after each state transfer. + uint64 state_id = 2; } message StoreDRAutoSyncStatus { - DRAutoSyncState state = 1; - uint64 state_id = 2; + DRAutoSyncState state = 1; + uint64 state_id = 2; } diff --git a/proto/resource_manager.proto b/proto/resource_manager.proto index aa7de748..302efc9d 100644 --- a/proto/resource_manager.proto +++ b/proto/resource_manager.proto @@ -4,8 +4,11 @@ package resource_manager; import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; @@ -23,27 +26,54 @@ service ResourceManager { rpc AcquireTokenBuckets(stream TokenBucketsRequest) returns (stream TokenBucketsResponse) {} } -message ListResourceGroupsRequest{} +// KeyspaceIDValue is a wrapper for the value of keyspace ID. +// Because the 0 value is a valid keyspace ID, we need to use a wrapper to distinguish it from the null keyspace ID. +message KeyspaceIDValue { + uint32 value = 1; +} + +message ListResourceGroupsRequest { + bool with_ru_stats = 1; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, the listed resource groups will be filtered + // by the given keyspace ID. + KeyspaceIDValue keyspace_id = 2; +} -message ListResourceGroupsResponse{ +message ListResourceGroupsResponse { Error error = 1; repeated ResourceGroup groups = 2; } message GetResourceGroupRequest { string resource_group_name = 1; + bool with_ru_stats = 2; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will try to get the resource group within + // the given keyspace ID. + KeyspaceIDValue keyspace_id = 3; } -message GetResourceGroupResponse{ +message GetResourceGroupResponse { Error error = 1; ResourceGroup group = 2; } message DeleteResourceGroupRequest { string resource_group_name = 1; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will try to delete the resource group within + // the given keyspace ID. + KeyspaceIDValue keyspace_id = 2; } -message DeleteResourceGroupResponse{ +message DeleteResourceGroupResponse { Error error = 1; string body = 2; } @@ -64,25 +94,31 @@ message TokenBucketsRequest { } message TokenBucketRequest { - message RequestRU { - repeated RequestUnitItem request_r_u = 1; - } - message RequestRawResource { - repeated RawResourceItem request_raw_resource = 1; - } - - string resource_group_name = 1; - oneof request { - // RU mode, group settings with WRU/RRU etc resource abstract unit. - RequestRU ru_items = 2; - // Raw mode, group settings with CPU/IO etc resource unit. - RequestRawResource raw_resource_items = 3; - } - // Aggregate statistics in group level. - Consumption consumption_since_last_request = 4; - // label background request. - bool is_background = 5; - bool is_tiflash = 6; + message RequestRU { + repeated RequestUnitItem request_r_u = 1; + } + message RequestRawResource { + repeated RawResourceItem request_raw_resource = 1; + } + + string resource_group_name = 1; + oneof request { + // RU mode, group settings with WRU/RRU etc resource abstract unit. + RequestRU ru_items = 2; + // Raw mode, group settings with CPU/IO etc resource unit. + RequestRawResource raw_resource_items = 3; + } + // Aggregate statistics in group level. + Consumption consumption_since_last_request = 4; + // label background request. + bool is_background = 5; + bool is_tiflash = 6; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will try to request the token bucket from + // the resource group within the given keyspace ID. + KeyspaceIDValue keyspace_id = 7; } message TokenBucketsResponse { @@ -96,6 +132,12 @@ message TokenBucketResponse { repeated GrantedRUTokenBucket granted_r_u_tokens = 2; // Raw mode repeated GrantedRawResourceTokenBucket granted_resource_tokens = 3; + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version, + // which can be safely ignored to keep compatibility. + // - If the keyspace ID is set to a valid value, it means this response is from the resource + // group within this keyspace ID. + KeyspaceIDValue keyspace_id = 4; } message GrantedRUTokenBucket { @@ -116,7 +158,7 @@ enum RequestUnitType { enum RawResourceType { CPU = 0; - IOReadFlow = 1; + IOReadFlow = 1; IOWriteFlow = 2; } @@ -129,6 +171,14 @@ message Consumption { double sql_layer_cpu_time_ms = 6; double kv_read_rpc_count = 7; double kv_write_rpc_count = 8; + uint64 read_cross_az_traffic_bytes = 9; + uint64 write_cross_az_traffic_bytes = 10; + // RUv2 is an experimental v2 RU calculation. + // For now it only records the consumption without actual token deduction. + double tikv_r_u_v2 = 11; + double tidb_r_u_v2 = 12; + // tiflash_r_u_v2 equals tiflash RRU + tiflash WRU. + double tiflash_r_u_v2 = 13; } message RequestUnitItem { @@ -142,12 +192,12 @@ message RawResourceItem { } enum GroupMode { - Unknown = 0; - RUMode = 1; - RawMode = 2; + Unknown = 0; + RUMode = 1; + RawMode = 2; } -// ResourceGroup the settings definitions. +// ResourceGroup the settings definitions. message ResourceGroup { string name = 1; GroupMode mode = 2; @@ -159,7 +209,16 @@ message ResourceGroup { uint32 priority = 5; // Runaway queries settings RunawaySettings runaway_settings = 6; + // Background task control settings. BackgroundSettings background_settings = 7; + // RU consumption statistics. + Consumption RUStats = 8; + // The keyspace ID that the resource group belongs to. + // There're two cases for this field: + // - If the keyspace ID is not set, it means this may be a message from an older version. + // To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + // - If the keyspace ID is set to a valid value, it will directly be used. + KeyspaceIDValue keyspace_id = 9; } message GroupRequestUnitSettings { @@ -176,7 +235,7 @@ message TokenBucket { TokenLimitSettings settings = 1; // Once used to reconfigure, the tokens is delta tokens. double tokens = 2; -} +} message TokenLimitSettings { uint64 fill_rate = 1; @@ -185,14 +244,15 @@ message TokenLimitSettings { } message Error { - string message = 1; + string message = 1; } enum RunawayAction { NoneAction = 0; - DryRun = 1; // do nothing - CoolDown = 2; // deprioritize the task - Kill = 3; // kill the task + DryRun = 1; // do nothing + CoolDown = 2; // deprioritize the task + Kill = 3; // kill the task + SwitchGroup = 4; // switch the task to another group } enum RunawayWatchType { @@ -204,30 +264,38 @@ enum RunawayWatchType { message RunawayRule { uint64 exec_elapsed_time_ms = 1; + int64 processed_keys = 2; + int64 request_unit = 3; } message RunawayWatch { // how long would the watch last int64 lasting_duration_ms = 1; - RunawayWatchType type = 2; + RunawayWatchType type = 2; } message RunawaySettings { - RunawayRule rule = 1; + RunawayRule rule = 1; RunawayAction action = 2; RunawayWatch watch = 3; + // When the runaway action is `SwitchGroup`, + // this field will be used to indicate which group to switch. + string switch_group_name = 4; } message BackgroundSettings { - repeated string job_types = 1; + // background task types. + repeated string job_types = 1; + // the percentage limit of total resource(cpu/io) that background tasks can use. + uint64 utilization_limit = 2; } message Participant { - // name is the unique name of the resource manager participant. - string name = 1; - // id is the unique id of the resource manager participant. - uint64 id = 2; - // listen_urls is the serivce endpoint list in the url format. - // listen_urls[0] is primary service endpoint. - repeated string listen_urls = 3; + // name is the unique name of the resource manager participant. + string name = 1; + // id is the unique id of the resource manager participant. + uint64 id = 2; + // listen_urls is the serivce endpoint list in the url format. + // listen_urls[0] is primary service endpoint. + repeated string listen_urls = 3; } diff --git a/proto/resource_usage_agent.proto b/proto/resource_usage_agent.proto index 4250ec25..576ab926 100644 --- a/proto/resource_usage_agent.proto +++ b/proto/resource_usage_agent.proto @@ -5,25 +5,27 @@ package resource_usage_agent; import "gogoproto/gogo.proto"; import "rustproto.proto"; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // ResourceUsageAgent is the service for storing resource usage records. service ResourceUsageAgent { - // Report the resource usage records. By default, the records with the same - // resource group tag will be batched by minute. - rpc Report(stream ResourceUsageRecord) returns (EmptyResponse) {} + // Report the resource usage records. By default, the records with the same + // resource group tag will be batched by minute. + rpc Report(stream ResourceUsageRecord) returns (EmptyResponse) {} } // TiKV implements ResourceMeteringPubSub service for clients to subscribe to resource metering records. service ResourceMeteringPubSub { - // Clients subscribe to resource metering records through this RPC, and TiKV periodically (e.g. per minute) - // publishes resource metering records to clients via gRPC stream. - rpc Subscribe(ResourceMeteringRequest) returns (stream ResourceUsageRecord) {} + // Clients subscribe to resource metering records through this RPC, and TiKV periodically (e.g. per minute) + // publishes resource metering records to clients via gRPC stream. + rpc Subscribe(ResourceMeteringRequest) returns (stream ResourceUsageRecord) {} } message ResourceMeteringRequest {} @@ -31,20 +33,31 @@ message ResourceMeteringRequest {} message EmptyResponse {} message ResourceUsageRecord { - oneof record_oneof { - GroupTagRecord record = 1; - } + oneof record_oneof { + GroupTagRecord record = 1; + RegionRecord region_record = 2; + } } // GroupTagRecord is a set of resource usage data grouped by resource_group_tag. message GroupTagRecord { - bytes resource_group_tag = 1; - repeated GroupTagRecordItem items = 2; + bytes resource_group_tag = 1; + repeated GroupTagRecordItem items = 2; } message GroupTagRecordItem { - uint64 timestamp_sec = 1; - uint32 cpu_time_ms = 2; - uint32 read_keys = 3; - uint32 write_keys = 4; + uint64 timestamp_sec = 1; + uint32 cpu_time_ms = 2; + uint32 read_keys = 3; + uint32 write_keys = 4; + uint64 network_in_bytes = 5; + uint64 network_out_bytes = 6; + uint64 logical_read_bytes = 7; + uint64 logical_write_bytes = 8; +} + +// RegionRecord is a set of resource usage data grouped by region. +message RegionRecord { + uint64 region_id = 1; + repeated GroupTagRecordItem items = 2; } diff --git a/proto/schedulingpb.proto b/proto/schedulingpb.proto index 83b7de46..9167a3ae 100644 --- a/proto/schedulingpb.proto +++ b/proto/schedulingpb.proto @@ -1,197 +1,217 @@ syntax = "proto3"; package schedulingpb; -import "pdpb.proto"; import "gogoproto/gogo.proto"; -import "rustproto.proto"; import "metapb.proto"; +import "pdpb.proto"; +import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service Scheduling { - rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {} + + rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} - rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {} + rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} - rpc SplitRegions(SplitRegionsRequest) returns (SplitRegionsResponse) {} + rpc ScatterRegions(ScatterRegionsRequest) returns (ScatterRegionsResponse) {} - rpc ScatterRegions(ScatterRegionsRequest) returns (ScatterRegionsResponse) {} + rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} - rpc GetOperator(GetOperatorRequest) returns (GetOperatorResponse) {} - - rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} + rpc AskBatchSplit(AskBatchSplitRequest) returns (AskBatchSplitResponse) {} + + rpc RegionBuckets(stream RegionBucketsRequest) returns (stream RegionBucketsResponse) {} } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // sender_id is the ID of the sender server. - uint64 sender_id = 2; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // sender_id is the ID of the sender server. + uint64 sender_id = 2; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; } enum ErrorType { - OK = 0; - UNKNOWN = 1; - NOT_BOOTSTRAPPED = 2; - ALREADY_BOOTSTRAPPED = 3; - INVALID_VALUE = 4; - CLUSTER_MISMATCHED = 5; + OK = 0; + UNKNOWN = 1; + NOT_BOOTSTRAPPED = 2; + ALREADY_BOOTSTRAPPED = 3; + INVALID_VALUE = 4; + CLUSTER_MISMATCHED = 5; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message Participant { - // name is the unique name of the scheduling participant. - string name = 1; - // id is the unique id of the scheduling participant. - uint64 id = 2; - // listen_urls is the serivce endpoint list in the url format. - // listen_urls[0] is primary service endpoint. - repeated string listen_urls = 3; + // name is the unique name of the scheduling participant. + string name = 1; + // id is the unique id of the scheduling participant. + uint64 id = 2; + // listen_urls is the serivce endpoint list in the url format. + // listen_urls[0] is primary service endpoint. + repeated string listen_urls = 3; } message StoreHeartbeatRequest { - RequestHeader header = 1; - pdpb.StoreStats stats = 2; + RequestHeader header = 1; + pdpb.StoreStats stats = 2; } message StoreHeartbeatResponse { - ResponseHeader header = 1; - string cluster_version = 2; + ResponseHeader header = 1; + string cluster_version = 2; } message RegionHeartbeatRequest { - RequestHeader header = 1; - - metapb.Region region = 2; - // Leader Peer sending the heartbeat. - metapb.Peer leader = 3; - // Term is the term of raft group. - uint64 term = 4; - // Leader considers that these peers are down. - repeated pdpb.PeerStats down_peers = 5; - // Pending peers are the peers that the leader can't consider as - // working followers. - repeated metapb.Peer pending_peers = 6; - // Bytes read/written during this period. - uint64 bytes_written = 7; - uint64 bytes_read = 8; - // Keys read/written during this period. - uint64 keys_written = 9; - uint64 keys_read = 10; - // Approximate region size. - uint64 approximate_size = 11; - // Approximate number of keys. - uint64 approximate_keys = 12; - // QueryStats reported write query stats, and there are read query stats in store heartbeat - pdpb.QueryStats query_stats = 13; - // Actually reported time interval - pdpb.TimeInterval interval = 14; + RequestHeader header = 1; + + metapb.Region region = 2; + // Leader Peer sending the heartbeat. + metapb.Peer leader = 3; + // Term is the term of raft group. + uint64 term = 4; + // Leader considers that these peers are down. + repeated pdpb.PeerStats down_peers = 5; + // Pending peers are the peers that the leader can't consider as + // working followers. + repeated metapb.Peer pending_peers = 6; + // Bytes read/written during this period. + uint64 bytes_written = 7; + uint64 bytes_read = 8; + // Keys read/written during this period. + uint64 keys_written = 9; + uint64 keys_read = 10; + // Approximate region size. + uint64 approximate_size = 11; + // Approximate number of keys. + uint64 approximate_keys = 12; + // QueryStats reported write query stats, and there are read query stats in store heartbeat + pdpb.QueryStats query_stats = 13; + // Actually reported time interval + pdpb.TimeInterval interval = 14; + // BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + metapb.BucketMeta bucket_meta = 15; } message RegionHeartbeatResponse { - ResponseHeader header = 1; - - // ID of the region - uint64 region_id = 2; - metapb.RegionEpoch region_epoch = 3; - // Leader of the region at the moment of the corresponding request was made. - metapb.Peer target_peer = 4; - // Notice, Pd only allows handling reported epoch >= current pd's. - // Leader peer reports region status with RegionHeartbeatRequest - // to pd regularly, pd will determine whether this region - // should do ChangePeer or not. - // E,g, max peer number is 3, region A, first only peer 1 in A. - // 1. Pd region state -> Peers (1), ConfVer (1). - // 2. Leader peer 1 reports region state to pd, pd finds the - // peer number is < 3, so first changes its current region - // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. - // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), - // pd updates its state -> Peers (1, 2), ConfVer (2). - // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange - // finished, pd stills responses ChangePeer Adding 2, of course, we must - // guarantee the second ChangePeer can't be applied in TiKV. - pdpb.ChangePeer change_peer = 5; - // Pd can return transfer_leader to let TiKV does leader transfer itself. - pdpb.TransferLeader transfer_leader = 6; - pdpb.Merge merge = 7; - // PD sends split_region to let TiKV split a region into two regions. - pdpb.SplitRegion split_region = 8; - // Multiple change peer operations atomically. - // Note: PD can use both ChangePeer and ChangePeerV2 at the same time - // (not in the same RegionHeartbeatResponse). - // Now, PD use ChangePeerV2 in following scenarios: - // 1. replacing peers - // 2. demoting voter directly - pdpb.ChangePeerV2 change_peer_v2 = 9; - pdpb.BatchSwitchWitness switch_witnesses = 10; + ResponseHeader header = 1; + + // ID of the region + uint64 region_id = 2; + metapb.RegionEpoch region_epoch = 3; + // Leader of the region at the moment of the corresponding request was made. + metapb.Peer target_peer = 4; + // Notice, Pd only allows handling reported epoch >= current pd's. + // Leader peer reports region status with RegionHeartbeatRequest + // to pd regularly, pd will determine whether this region + // should do ChangePeer or not. + // E,g, max peer number is 3, region A, first only peer 1 in A. + // 1. Pd region state -> Peers (1), ConfVer (1). + // 2. Leader peer 1 reports region state to pd, pd finds the + // peer number is < 3, so first changes its current region + // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2. + // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2), + // pd updates its state -> Peers (1, 2), ConfVer (2). + // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange + // finished, pd stills responses ChangePeer Adding 2, of course, we must + // guarantee the second ChangePeer can't be applied in TiKV. + pdpb.ChangePeer change_peer = 5; + // Pd can return transfer_leader to let TiKV does leader transfer itself. + pdpb.TransferLeader transfer_leader = 6; + pdpb.Merge merge = 7; + // PD sends split_region to let TiKV split a region into two regions. + pdpb.SplitRegion split_region = 8; + // Multiple change peer operations atomically. + // Note: PD can use both ChangePeer and ChangePeerV2 at the same time + // (not in the same RegionHeartbeatResponse). + // Now, PD use ChangePeerV2 in following scenarios: + // 1. replacing peers + // 2. demoting voter directly + pdpb.ChangePeerV2 change_peer_v2 = 9; + pdpb.BatchSwitchWitness switch_witnesses = 10; + pdpb.ChangeSplit change_split = 11; } message ScatterRegionsRequest { - RequestHeader header = 1; - // If group is defined, the regions with the same group would be scattered as a whole group. - // If not defined, the regions would be scattered in a cluster level. - string group = 2; + RequestHeader header = 1; + // If group is defined, the regions with the same group would be scattered as a whole group. + // If not defined, the regions would be scattered in a cluster level. + string group = 2; - // If regions_id is defined, the region_id would be ignored. - repeated uint64 regions_id = 3; - uint64 retry_limit = 4; - bool skip_store_limit = 5; + // If regions_id is defined, the region_id would be ignored. + repeated uint64 regions_id = 3; + uint64 retry_limit = 4; + bool skip_store_limit = 5; } message ScatterRegionsResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 failed_regions_id = 3; } message SplitRegionsRequest { - RequestHeader header = 1; - repeated bytes split_keys = 2; - uint64 retry_limit = 3; + RequestHeader header = 1; + repeated bytes split_keys = 2; + uint64 retry_limit = 3; } message SplitRegionsResponse { - ResponseHeader header = 1; - uint64 finished_percentage = 2; - repeated uint64 regions_id = 3; + ResponseHeader header = 1; + uint64 finished_percentage = 2; + repeated uint64 regions_id = 3; } message GetOperatorRequest { - RequestHeader header = 1; - uint64 region_id = 2; + RequestHeader header = 1; + uint64 region_id = 2; } message GetOperatorResponse { - ResponseHeader header = 1; - uint64 region_id = 2; - bytes desc = 3; - pdpb.OperatorStatus status = 4; - bytes kind = 5; + ResponseHeader header = 1; + uint64 region_id = 2; + bytes desc = 3; + pdpb.OperatorStatus status = 4; + bytes kind = 5; } message AskBatchSplitRequest { - RequestHeader header = 1; + RequestHeader header = 1; - metapb.Region region = 2; - uint32 split_count = 3; + metapb.Region region = 2; + uint32 split_count = 3; + pdpb.SplitReason reason = 4; } message AskBatchSplitResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; + + repeated pdpb.SplitID ids = 2; +} + +message RegionBucketsRequest { + RequestHeader header = 1; + + metapb.RegionEpoch region_epoch = 2; + metapb.Buckets buckets = 3; +} - repeated pdpb.SplitID ids = 2; +message RegionBucketsResponse { + ResponseHeader header = 1; } diff --git a/proto/tikvpb.proto b/proto/tikvpb.proto index f48e9d14..4d1986ba 100644 --- a/proto/tikvpb.proto +++ b/proto/tikvpb.proto @@ -2,258 +2,304 @@ syntax = "proto3"; package tikvpb; import "coprocessor.proto"; +import "disaggregated.proto"; +import "gogoproto/gogo.proto"; import "kvrpcpb.proto"; import "mpp.proto"; import "raft_serverpb.proto"; -import "disaggregated.proto"; - -import "gogoproto/gogo.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - // Key/value store API for TiKV. service Tikv { - // Commands using a transactional interface. - rpc KvGet(kvrpcpb.GetRequest) returns (kvrpcpb.GetResponse) {} - rpc KvScan(kvrpcpb.ScanRequest) returns (kvrpcpb.ScanResponse) {} - rpc KvPrewrite(kvrpcpb.PrewriteRequest) returns (kvrpcpb.PrewriteResponse) {} - rpc KvPessimisticLock(kvrpcpb.PessimisticLockRequest) returns (kvrpcpb.PessimisticLockResponse) {} - rpc KVPessimisticRollback(kvrpcpb.PessimisticRollbackRequest) returns (kvrpcpb.PessimisticRollbackResponse) {} - rpc KvTxnHeartBeat(kvrpcpb.TxnHeartBeatRequest) returns (kvrpcpb.TxnHeartBeatResponse) {} - rpc KvCheckTxnStatus(kvrpcpb.CheckTxnStatusRequest) returns (kvrpcpb.CheckTxnStatusResponse) {} - rpc KvCheckSecondaryLocks(kvrpcpb.CheckSecondaryLocksRequest) returns (kvrpcpb.CheckSecondaryLocksResponse) {} - rpc KvCommit(kvrpcpb.CommitRequest) returns (kvrpcpb.CommitResponse) {} - rpc KvImport(kvrpcpb.ImportRequest) returns (kvrpcpb.ImportResponse) {} - rpc KvCleanup(kvrpcpb.CleanupRequest) returns (kvrpcpb.CleanupResponse) {} - rpc KvBatchGet(kvrpcpb.BatchGetRequest) returns (kvrpcpb.BatchGetResponse) {} - rpc KvBatchRollback(kvrpcpb.BatchRollbackRequest) returns (kvrpcpb.BatchRollbackResponse) {} - rpc KvScanLock(kvrpcpb.ScanLockRequest) returns (kvrpcpb.ScanLockResponse) {} - rpc KvResolveLock(kvrpcpb.ResolveLockRequest) returns (kvrpcpb.ResolveLockResponse) {} - rpc KvGC(kvrpcpb.GCRequest) returns (kvrpcpb.GCResponse) {} - rpc KvDeleteRange(kvrpcpb.DeleteRangeRequest) returns (kvrpcpb.DeleteRangeResponse) {} - rpc KvPrepareFlashbackToVersion(kvrpcpb.PrepareFlashbackToVersionRequest) returns (kvrpcpb.PrepareFlashbackToVersionResponse) {} - rpc KvFlashbackToVersion(kvrpcpb.FlashbackToVersionRequest) returns (kvrpcpb.FlashbackToVersionResponse) {} - - // Raw commands; no transaction support. - rpc RawGet(kvrpcpb.RawGetRequest) returns (kvrpcpb.RawGetResponse) {} - rpc RawBatchGet(kvrpcpb.RawBatchGetRequest) returns (kvrpcpb.RawBatchGetResponse) {} - rpc RawPut(kvrpcpb.RawPutRequest) returns (kvrpcpb.RawPutResponse) {} - rpc RawBatchPut(kvrpcpb.RawBatchPutRequest) returns (kvrpcpb.RawBatchPutResponse) {} - rpc RawDelete(kvrpcpb.RawDeleteRequest) returns (kvrpcpb.RawDeleteResponse) {} - rpc RawBatchDelete(kvrpcpb.RawBatchDeleteRequest) returns (kvrpcpb.RawBatchDeleteResponse) {} - rpc RawScan(kvrpcpb.RawScanRequest) returns (kvrpcpb.RawScanResponse) {} - rpc RawDeleteRange(kvrpcpb.RawDeleteRangeRequest) returns (kvrpcpb.RawDeleteRangeResponse) {} - rpc RawBatchScan(kvrpcpb.RawBatchScanRequest) returns (kvrpcpb.RawBatchScanResponse) {} - // Get TTL of the key. Returns 0 if TTL is not set for the key. - rpc RawGetKeyTTL(kvrpcpb.RawGetKeyTTLRequest) returns (kvrpcpb.RawGetKeyTTLResponse) {} - // Compare if the value in database equals to `RawCASRequest.previous_value` before putting the new value. If not, this request will have no effect and the value in the database will be returned. - rpc RawCompareAndSwap(kvrpcpb.RawCASRequest) returns (kvrpcpb.RawCASResponse) {} - rpc RawChecksum(kvrpcpb.RawChecksumRequest) returns (kvrpcpb.RawChecksumResponse) {} - - // Store commands (sent to a each TiKV node in a cluster, rather than a certain region). - rpc UnsafeDestroyRange(kvrpcpb.UnsafeDestroyRangeRequest) returns (kvrpcpb.UnsafeDestroyRangeResponse) {} - rpc RegisterLockObserver(kvrpcpb.RegisterLockObserverRequest) returns (kvrpcpb.RegisterLockObserverResponse) {} - rpc CheckLockObserver(kvrpcpb.CheckLockObserverRequest) returns (kvrpcpb.CheckLockObserverResponse) {} - rpc RemoveLockObserver(kvrpcpb.RemoveLockObserverRequest) returns (kvrpcpb.RemoveLockObserverResponse) {} - rpc PhysicalScanLock(kvrpcpb.PhysicalScanLockRequest) returns (kvrpcpb.PhysicalScanLockResponse) {} - - // Commands for executing SQL in the TiKV coprocessor (i.e., 'pushed down' to TiKV rather than - // executed in TiDB). - rpc Coprocessor(coprocessor.Request) returns (coprocessor.Response) {} - rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {} - rpc BatchCoprocessor(coprocessor.BatchRequest) returns (stream coprocessor.BatchResponse) {} - - // Command for executing custom user requests in TiKV coprocessor_v2. - rpc RawCoprocessor(kvrpcpb.RawCoprocessorRequest) returns (kvrpcpb.RawCoprocessorResponse) {} - - // Raft commands (sent between TiKV nodes). - rpc Raft(stream raft_serverpb.RaftMessage) returns (raft_serverpb.Done) {} - rpc BatchRaft(stream BatchRaftMessage) returns (raft_serverpb.Done) {} - rpc Snapshot(stream raft_serverpb.SnapshotChunk) returns (raft_serverpb.Done) {} - rpc TabletSnapshot(stream raft_serverpb.TabletSnapshotRequest) returns (stream raft_serverpb.TabletSnapshotResponse) {} - - // Sent from PD or TiDB to a TiKV node. - rpc SplitRegion (kvrpcpb.SplitRegionRequest) returns (kvrpcpb.SplitRegionResponse) {} - // Sent from TiFlash or TiKV to a TiKV node. - rpc ReadIndex(kvrpcpb.ReadIndexRequest) returns (kvrpcpb.ReadIndexResponse) {} - - // Commands for debugging transactions. - rpc MvccGetByKey(kvrpcpb.MvccGetByKeyRequest) returns (kvrpcpb.MvccGetByKeyResponse) {} - rpc MvccGetByStartTs(kvrpcpb.MvccGetByStartTsRequest) returns (kvrpcpb.MvccGetByStartTsResponse) {} - - // Batched commands. - rpc BatchCommands(stream BatchCommandsRequest) returns (stream BatchCommandsResponse) {} - - // These are for mpp execution. - rpc DispatchMPPTask(mpp.DispatchTaskRequest) returns (mpp.DispatchTaskResponse) {} - rpc CancelMPPTask(mpp.CancelTaskRequest) returns (mpp.CancelTaskResponse) {} - rpc EstablishMPPConnection(mpp.EstablishMPPConnectionRequest) returns (stream mpp.MPPDataPacket) {} - rpc IsAlive(mpp.IsAliveRequest) returns (mpp.IsAliveResponse) {} - rpc ReportMPPTaskStatus(mpp.ReportTaskStatusRequest) returns (mpp.ReportTaskStatusResponse) {} - - /// CheckLeader sends all information (includes region term and epoch) to other stores. - /// Once a store receives a request, it checks term and epoch for each region, and sends the regions whose - /// term and epoch match with local information in the store. - /// After the client collected all responses from all stores, it checks if got a quorum of responses from - /// other stores for every region, and decides to advance resolved ts from these regions. - rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns (kvrpcpb.CheckLeaderResponse); - - /// Get the minimal `safe_ts` from regions at the store - rpc GetStoreSafeTS(kvrpcpb.StoreSafeTSRequest) returns (kvrpcpb.StoreSafeTSResponse); - - /// Get the information about lock waiting from TiKV. - rpc GetLockWaitInfo(kvrpcpb.GetLockWaitInfoRequest) returns (kvrpcpb.GetLockWaitInfoResponse); - - /// Compact a specified key range. This request is not restricted to raft leaders and will not be replicated. - /// It only compacts data on this node. - /// TODO: Currently this RPC is designed to be only compatible with TiFlash. - /// Shall be move out in https://github.com/pingcap/kvproto/issues/912 - rpc Compact(kvrpcpb.CompactRequest) returns (kvrpcpb.CompactResponse); - /// Get the information about history lock waiting from TiKV. - rpc GetLockWaitHistory(kvrpcpb.GetLockWaitHistoryRequest) returns (kvrpcpb.GetLockWaitHistoryResponse); - - /// Get system table from TiFlash - rpc GetTiFlashSystemTable(kvrpcpb.TiFlashSystemTableRequest) returns (kvrpcpb.TiFlashSystemTableResponse) {} - - // These are for TiFlash disaggregated architecture - /// Try to lock a S3 object, atomically - rpc tryAddLock(disaggregated.TryAddLockRequest) returns (disaggregated.TryAddLockResponse) {}; - /// Try to delete a S3 object, atomically - rpc tryMarkDelete(disaggregated.TryMarkDeleteRequest) returns (disaggregated.TryMarkDeleteResponse) {}; - /// Build the disaggregated task on TiFlash write node - rpc EstablishDisaggTask(disaggregated.EstablishDisaggTaskRequest) returns (disaggregated.EstablishDisaggTaskResponse) {} - /// Cancel the disaggregated task on TiFlash write node - rpc CancelDisaggTask(disaggregated.CancelDisaggTaskRequest) returns (disaggregated.CancelDisaggTaskResponse) {} - /// Exchange page data between TiFlash write node and compute node - rpc FetchDisaggPages(disaggregated.FetchDisaggPagesRequest) returns (stream disaggregated.PagesPacket) {} - /// Compute node get configuration from Write node - rpc GetDisaggConfig(disaggregated.GetDisaggConfigRequest) returns (disaggregated.GetDisaggConfigResponse) {} + // Commands using a transactional interface. + rpc KvGet(kvrpcpb.GetRequest) returns (kvrpcpb.GetResponse) {} + rpc KvScan(kvrpcpb.ScanRequest) returns (kvrpcpb.ScanResponse) {} + rpc KvPrewrite(kvrpcpb.PrewriteRequest) returns (kvrpcpb.PrewriteResponse) {} + rpc KvPessimisticLock(kvrpcpb.PessimisticLockRequest) returns (kvrpcpb.PessimisticLockResponse) {} + rpc KVPessimisticRollback(kvrpcpb.PessimisticRollbackRequest) returns (kvrpcpb.PessimisticRollbackResponse) {} + rpc KvTxnHeartBeat(kvrpcpb.TxnHeartBeatRequest) returns (kvrpcpb.TxnHeartBeatResponse) {} + rpc KvCheckTxnStatus(kvrpcpb.CheckTxnStatusRequest) returns (kvrpcpb.CheckTxnStatusResponse) {} + rpc KvCheckSecondaryLocks(kvrpcpb.CheckSecondaryLocksRequest) returns (kvrpcpb.CheckSecondaryLocksResponse) {} + rpc KvCommit(kvrpcpb.CommitRequest) returns (kvrpcpb.CommitResponse) {} + rpc KvImport(kvrpcpb.ImportRequest) returns (kvrpcpb.ImportResponse) {} + rpc KvCleanup(kvrpcpb.CleanupRequest) returns (kvrpcpb.CleanupResponse) {} + rpc KvBatchGet(kvrpcpb.BatchGetRequest) returns (kvrpcpb.BatchGetResponse) {} + rpc KvBatchRollback(kvrpcpb.BatchRollbackRequest) returns (kvrpcpb.BatchRollbackResponse) {} + rpc KvScanLock(kvrpcpb.ScanLockRequest) returns (kvrpcpb.ScanLockResponse) {} + rpc KvResolveLock(kvrpcpb.ResolveLockRequest) returns (kvrpcpb.ResolveLockResponse) {} + rpc KvGC(kvrpcpb.GCRequest) returns (kvrpcpb.GCResponse) {} + rpc KvDeleteRange(kvrpcpb.DeleteRangeRequest) returns (kvrpcpb.DeleteRangeResponse) {} + rpc KvPrepareFlashbackToVersion(kvrpcpb.PrepareFlashbackToVersionRequest) returns (kvrpcpb.PrepareFlashbackToVersionResponse) {} + rpc KvFlashbackToVersion(kvrpcpb.FlashbackToVersionRequest) returns (kvrpcpb.FlashbackToVersionResponse) {} + rpc KvFlush(kvrpcpb.FlushRequest) returns (kvrpcpb.FlushResponse) {} + rpc KvBufferBatchGet(kvrpcpb.BufferBatchGetRequest) returns (kvrpcpb.BufferBatchGetResponse) {} + + // Raw commands; no transaction support. + rpc RawGet(kvrpcpb.RawGetRequest) returns (kvrpcpb.RawGetResponse) {} + rpc RawBatchGet(kvrpcpb.RawBatchGetRequest) returns (kvrpcpb.RawBatchGetResponse) {} + rpc RawPut(kvrpcpb.RawPutRequest) returns (kvrpcpb.RawPutResponse) {} + rpc RawBatchPut(kvrpcpb.RawBatchPutRequest) returns (kvrpcpb.RawBatchPutResponse) {} + rpc RawDelete(kvrpcpb.RawDeleteRequest) returns (kvrpcpb.RawDeleteResponse) {} + rpc RawBatchDelete(kvrpcpb.RawBatchDeleteRequest) returns (kvrpcpb.RawBatchDeleteResponse) {} + rpc RawScan(kvrpcpb.RawScanRequest) returns (kvrpcpb.RawScanResponse) {} + rpc RawDeleteRange(kvrpcpb.RawDeleteRangeRequest) returns (kvrpcpb.RawDeleteRangeResponse) {} + rpc RawBatchScan(kvrpcpb.RawBatchScanRequest) returns (kvrpcpb.RawBatchScanResponse) {} + // Get TTL of the key. Returns 0 if TTL is not set for the key. + rpc RawGetKeyTTL(kvrpcpb.RawGetKeyTTLRequest) returns (kvrpcpb.RawGetKeyTTLResponse) {} + // Compare if the value in database equals to `RawCASRequest.previous_value` before putting the new value. If not, this request will have no effect and the value in the database will be returned. + rpc RawCompareAndSwap(kvrpcpb.RawCASRequest) returns (kvrpcpb.RawCASResponse) {} + rpc RawChecksum(kvrpcpb.RawChecksumRequest) returns (kvrpcpb.RawChecksumResponse) {} + + // Store commands (sent to a each TiKV node in a cluster, rather than a certain region). + rpc UnsafeDestroyRange(kvrpcpb.UnsafeDestroyRangeRequest) returns (kvrpcpb.UnsafeDestroyRangeResponse) {} + rpc RegisterLockObserver(kvrpcpb.RegisterLockObserverRequest) returns (kvrpcpb.RegisterLockObserverResponse) {} + rpc CheckLockObserver(kvrpcpb.CheckLockObserverRequest) returns (kvrpcpb.CheckLockObserverResponse) {} + rpc RemoveLockObserver(kvrpcpb.RemoveLockObserverRequest) returns (kvrpcpb.RemoveLockObserverResponse) {} + rpc PhysicalScanLock(kvrpcpb.PhysicalScanLockRequest) returns (kvrpcpb.PhysicalScanLockResponse) {} + + // Commands for executing SQL in the TiKV coprocessor (i.e., 'pushed down' to TiKV rather than + // executed in TiDB). + rpc Coprocessor(coprocessor.Request) returns (coprocessor.Response) {} + rpc CoprocessorStream(coprocessor.Request) returns (stream coprocessor.Response) {} + rpc BatchCoprocessor(coprocessor.BatchRequest) returns (stream coprocessor.BatchResponse) {} + + // Command send by remote coprocessor to TiKV for executing coprocessor request. + rpc DelegateCoprocessor(coprocessor.DelegateRequest) returns (coprocessor.DelegateResponse) {} + + // Command for executing custom user requests in TiKV coprocessor_v2. + rpc RawCoprocessor(kvrpcpb.RawCoprocessorRequest) returns (kvrpcpb.RawCoprocessorResponse) {} + + // Raft commands (sent between TiKV nodes). + rpc Raft(stream raft_serverpb.RaftMessage) returns (raft_serverpb.Done) {} + rpc BatchRaft(stream BatchRaftMessage) returns (raft_serverpb.Done) {} + rpc Snapshot(stream raft_serverpb.SnapshotChunk) returns (raft_serverpb.Done) {} + rpc TabletSnapshot(stream raft_serverpb.TabletSnapshotRequest) returns (stream raft_serverpb.TabletSnapshotResponse) {} + + // Sent from PD or TiDB to a TiKV node. + rpc SplitRegion(kvrpcpb.SplitRegionRequest) returns (kvrpcpb.SplitRegionResponse) {} + // Sent from TiFlash or TiKV to a TiKV node. + rpc ReadIndex(kvrpcpb.ReadIndexRequest) returns (kvrpcpb.ReadIndexResponse) {} + + // Commands for debugging transactions. + rpc MvccGetByKey(kvrpcpb.MvccGetByKeyRequest) returns (kvrpcpb.MvccGetByKeyResponse) {} + rpc MvccGetByStartTs(kvrpcpb.MvccGetByStartTsRequest) returns (kvrpcpb.MvccGetByStartTsResponse) {} + + // Batched commands. + rpc BatchCommands(stream BatchCommandsRequest) returns (stream BatchCommandsResponse) {} + + // These are for mpp execution. + rpc DispatchMPPTask(mpp.DispatchTaskRequest) returns (mpp.DispatchTaskResponse) {} + rpc CancelMPPTask(mpp.CancelTaskRequest) returns (mpp.CancelTaskResponse) {} + rpc EstablishMPPConnection(mpp.EstablishMPPConnectionRequest) returns (stream mpp.MPPDataPacket) {} + rpc IsAlive(mpp.IsAliveRequest) returns (mpp.IsAliveResponse) {} + rpc ReportMPPTaskStatus(mpp.ReportTaskStatusRequest) returns (mpp.ReportTaskStatusResponse) {} + + /// CheckLeader sends all information (includes region term and epoch) to other stores. + /// Once a store receives a request, it checks term and epoch for each region, and sends the regions whose + /// term and epoch match with local information in the store. + /// After the client collected all responses from all stores, it checks if got a quorum of responses from + /// other stores for every region, and decides to advance resolved ts from these regions. + rpc CheckLeader(kvrpcpb.CheckLeaderRequest) returns (kvrpcpb.CheckLeaderResponse); + + /// Get the minimal `safe_ts` from regions at the store + rpc GetStoreSafeTS(kvrpcpb.StoreSafeTSRequest) returns (kvrpcpb.StoreSafeTSResponse); + + /// Get the information about lock waiting from TiKV. + rpc GetLockWaitInfo(kvrpcpb.GetLockWaitInfoRequest) returns (kvrpcpb.GetLockWaitInfoResponse); + + /// Compact a specified key range. This request is not restricted to raft leaders and will not be replicated. + /// It only compacts data on this node. + /// TODO: Currently this RPC is designed to be only compatible with TiFlash. + /// Shall be move out in https://github.com/pingcap/kvproto/issues/912 + rpc Compact(kvrpcpb.CompactRequest) returns (kvrpcpb.CompactResponse); + /// Get the information about history lock waiting from TiKV. + rpc GetLockWaitHistory(kvrpcpb.GetLockWaitHistoryRequest) returns (kvrpcpb.GetLockWaitHistoryResponse); + + /// Get system table from TiFlash + rpc GetTiFlashSystemTable(kvrpcpb.TiFlashSystemTableRequest) returns (kvrpcpb.TiFlashSystemTableResponse) {} + /// Get estimate tici count from TiFlash + rpc GetEstimateTiCICount(coprocessor.TiCIEstimateCountRequest) returns (coprocessor.TiCIEstimateCountResponse) {} + + // These are for TiFlash disaggregated architecture + /// Try to lock a S3 object, atomically + rpc tryAddLock(disaggregated.TryAddLockRequest) returns (disaggregated.TryAddLockResponse) {} + /// Try to delete a S3 object, atomically + rpc tryMarkDelete(disaggregated.TryMarkDeleteRequest) returns (disaggregated.TryMarkDeleteResponse) {} + /// Build the disaggregated task on TiFlash write node + rpc EstablishDisaggTask(disaggregated.EstablishDisaggTaskRequest) returns (disaggregated.EstablishDisaggTaskResponse) {} + /// Cancel the disaggregated task on TiFlash write node + rpc CancelDisaggTask(disaggregated.CancelDisaggTaskRequest) returns (disaggregated.CancelDisaggTaskResponse) {} + /// Exchange page data between TiFlash write node and compute node + rpc FetchDisaggPages(disaggregated.FetchDisaggPagesRequest) returns (stream disaggregated.PagesPacket) {} + /// Compute node get configuration from Write node + rpc GetDisaggConfig(disaggregated.GetDisaggConfigRequest) returns (disaggregated.GetDisaggConfigResponse) {} + + /// Get health feedback info from the TiKV node. + rpc GetHealthFeedback(kvrpcpb.GetHealthFeedbackRequest) returns (kvrpcpb.GetHealthFeedbackResponse) {} + /// Broadcast the transaction status to all TiKV nodes + rpc BroadcastTxnStatus(kvrpcpb.BroadcastTxnStatusRequest) returns (kvrpcpb.BroadcastTxnStatusResponse) {} +} + +// VersionedKv provides versioned coprocessor APIs for TiCI lookup. +// +// Invariants: +// - For `VersionedCoprocessor`, callers should fill `coprocessor.Request.versioned_ranges` +// (each `VersionedKeyRange.range` must be a point range) and keep `coprocessor.Request.ranges` empty. +service VersionedKv { + rpc VersionedCoprocessor(coprocessor.Request) returns (coprocessor.Response) {} } message BatchCommandsRequest { - repeated Request requests = 1; - repeated uint64 request_ids = 2; - - message Request { - oneof cmd { - kvrpcpb.GetRequest Get = 1; - kvrpcpb.ScanRequest Scan = 2; - kvrpcpb.PrewriteRequest Prewrite = 3; - kvrpcpb.CommitRequest Commit = 4; - kvrpcpb.ImportRequest Import = 5; - kvrpcpb.CleanupRequest Cleanup = 6; - kvrpcpb.BatchGetRequest BatchGet = 7; - kvrpcpb.BatchRollbackRequest BatchRollback = 8; - kvrpcpb.ScanLockRequest ScanLock = 9; - kvrpcpb.ResolveLockRequest ResolveLock = 10; - kvrpcpb.GCRequest GC = 11; - kvrpcpb.DeleteRangeRequest DeleteRange = 12; - - kvrpcpb.RawGetRequest RawGet = 13; - kvrpcpb.RawBatchGetRequest RawBatchGet = 14; - kvrpcpb.RawPutRequest RawPut = 15; - kvrpcpb.RawBatchPutRequest RawBatchPut = 16; - kvrpcpb.RawDeleteRequest RawDelete = 17; - kvrpcpb.RawBatchDeleteRequest RawBatchDelete = 18; - kvrpcpb.RawScanRequest RawScan = 19; - kvrpcpb.RawDeleteRangeRequest RawDeleteRange = 20; - kvrpcpb.RawBatchScanRequest RawBatchScan = 21; - - coprocessor.Request Coprocessor = 22; - kvrpcpb.PessimisticLockRequest PessimisticLock = 23; - kvrpcpb.PessimisticRollbackRequest PessimisticRollback = 24; - - kvrpcpb.CheckTxnStatusRequest CheckTxnStatus = 25; - kvrpcpb.TxnHeartBeatRequest TxnHeartBeat = 26; - - kvrpcpb.CheckSecondaryLocksRequest CheckSecondaryLocks = 33; - - kvrpcpb.RawCoprocessorRequest RawCoprocessor = 34; - - kvrpcpb.FlashbackToVersionRequest FlashbackToVersion = 35; - kvrpcpb.PrepareFlashbackToVersionRequest PrepareFlashbackToVersion = 36; - - // For some test cases. - BatchCommandsEmptyRequest Empty = 255; - } - - // Reserved for deprecated VerKV - reserved 27 to 32; + repeated Request requests = 1; + repeated uint64 request_ids = 2; + // Unix epoch timestamp in nanoseconds recorded by the client when this + // batch request is sent to TiKV. + uint64 client_send_time_ns = 3; + + message Request { + oneof cmd { + kvrpcpb.GetRequest Get = 1; + kvrpcpb.ScanRequest Scan = 2; + kvrpcpb.PrewriteRequest Prewrite = 3; + kvrpcpb.CommitRequest Commit = 4; + kvrpcpb.ImportRequest Import = 5; + kvrpcpb.CleanupRequest Cleanup = 6; + kvrpcpb.BatchGetRequest BatchGet = 7; + kvrpcpb.BatchRollbackRequest BatchRollback = 8; + kvrpcpb.ScanLockRequest ScanLock = 9; + kvrpcpb.ResolveLockRequest ResolveLock = 10; + kvrpcpb.GCRequest GC = 11; + kvrpcpb.DeleteRangeRequest DeleteRange = 12; + + kvrpcpb.RawGetRequest RawGet = 13; + kvrpcpb.RawBatchGetRequest RawBatchGet = 14; + kvrpcpb.RawPutRequest RawPut = 15; + kvrpcpb.RawBatchPutRequest RawBatchPut = 16; + kvrpcpb.RawDeleteRequest RawDelete = 17; + kvrpcpb.RawBatchDeleteRequest RawBatchDelete = 18; + kvrpcpb.RawScanRequest RawScan = 19; + kvrpcpb.RawDeleteRangeRequest RawDeleteRange = 20; + kvrpcpb.RawBatchScanRequest RawBatchScan = 21; + + coprocessor.Request Coprocessor = 22; + kvrpcpb.PessimisticLockRequest PessimisticLock = 23; + kvrpcpb.PessimisticRollbackRequest PessimisticRollback = 24; + + kvrpcpb.CheckTxnStatusRequest CheckTxnStatus = 25; + kvrpcpb.TxnHeartBeatRequest TxnHeartBeat = 26; + + kvrpcpb.CheckSecondaryLocksRequest CheckSecondaryLocks = 33; + + kvrpcpb.RawCoprocessorRequest RawCoprocessor = 34; + + kvrpcpb.FlashbackToVersionRequest FlashbackToVersion = 35; + kvrpcpb.PrepareFlashbackToVersionRequest PrepareFlashbackToVersion = 36; + kvrpcpb.FlushRequest Flush = 37; + kvrpcpb.BufferBatchGetRequest BufferBatchGet = 38; + + kvrpcpb.GetHealthFeedbackRequest GetHealthFeedback = 39; + + kvrpcpb.BroadcastTxnStatusRequest BroadcastTxnStatus = 40; + // For some test cases. + BatchCommandsEmptyRequest Empty = 255; } + + // Reserved for deprecated VerKV + reserved 27 to 32; + + // Reserved for a downstream fork + reserved 1000 to 1199; + } } message BatchCommandsResponse { - repeated Response responses = 1; - repeated uint64 request_ids = 2; - // 280 means TiKV gRPC cpu usage is 280%. - uint64 transport_layer_load = 3; - - message Response { - oneof cmd { - kvrpcpb.GetResponse Get = 1; - kvrpcpb.ScanResponse Scan = 2; - kvrpcpb.PrewriteResponse Prewrite = 3; - kvrpcpb.CommitResponse Commit = 4; - kvrpcpb.ImportResponse Import = 5; - kvrpcpb.CleanupResponse Cleanup = 6; - kvrpcpb.BatchGetResponse BatchGet = 7; - kvrpcpb.BatchRollbackResponse BatchRollback = 8; - kvrpcpb.ScanLockResponse ScanLock = 9; - kvrpcpb.ResolveLockResponse ResolveLock = 10; - kvrpcpb.GCResponse GC = 11; - kvrpcpb.DeleteRangeResponse DeleteRange = 12; - - kvrpcpb.RawGetResponse RawGet = 13; - kvrpcpb.RawBatchGetResponse RawBatchGet = 14; - kvrpcpb.RawPutResponse RawPut = 15; - kvrpcpb.RawBatchPutResponse RawBatchPut = 16; - kvrpcpb.RawDeleteResponse RawDelete = 17; - kvrpcpb.RawBatchDeleteResponse RawBatchDelete = 18; - kvrpcpb.RawScanResponse RawScan = 19; - kvrpcpb.RawDeleteRangeResponse RawDeleteRange = 20; - kvrpcpb.RawBatchScanResponse RawBatchScan = 21; - - coprocessor.Response Coprocessor = 22; - kvrpcpb.PessimisticLockResponse PessimisticLock = 23; - kvrpcpb.PessimisticRollbackResponse PessimisticRollback = 24; - - kvrpcpb.CheckTxnStatusResponse CheckTxnStatus = 25; - kvrpcpb.TxnHeartBeatResponse TxnHeartBeat = 26; - - kvrpcpb.CheckSecondaryLocksResponse CheckSecondaryLocks = 33; - - kvrpcpb.RawCoprocessorResponse RawCoprocessor = 34; - - kvrpcpb.FlashbackToVersionResponse FlashbackToVersion = 35; - kvrpcpb.PrepareFlashbackToVersionResponse PrepareFlashbackToVersion = 36; - - // For some test cases. - BatchCommandsEmptyResponse Empty = 255; - } - - // Reserved for deprecated VerKV - reserved 27 to 32; + repeated Response responses = 1; + repeated uint64 request_ids = 2; + // 280 means TiKV gRPC cpu usage is 280%. + uint64 transport_layer_load = 3; + kvrpcpb.HealthFeedback health_feedback = 4; + // Unix epoch timestamp in nanoseconds recorded by TiKV when this batch + // response is ready to be sent back to the client. + uint64 tikv_send_time_ns = 5; + + message Response { + oneof cmd { + kvrpcpb.GetResponse Get = 1; + kvrpcpb.ScanResponse Scan = 2; + kvrpcpb.PrewriteResponse Prewrite = 3; + kvrpcpb.CommitResponse Commit = 4; + kvrpcpb.ImportResponse Import = 5; + kvrpcpb.CleanupResponse Cleanup = 6; + kvrpcpb.BatchGetResponse BatchGet = 7; + kvrpcpb.BatchRollbackResponse BatchRollback = 8; + kvrpcpb.ScanLockResponse ScanLock = 9; + kvrpcpb.ResolveLockResponse ResolveLock = 10; + kvrpcpb.GCResponse GC = 11; + kvrpcpb.DeleteRangeResponse DeleteRange = 12; + + kvrpcpb.RawGetResponse RawGet = 13; + kvrpcpb.RawBatchGetResponse RawBatchGet = 14; + kvrpcpb.RawPutResponse RawPut = 15; + kvrpcpb.RawBatchPutResponse RawBatchPut = 16; + kvrpcpb.RawDeleteResponse RawDelete = 17; + kvrpcpb.RawBatchDeleteResponse RawBatchDelete = 18; + kvrpcpb.RawScanResponse RawScan = 19; + kvrpcpb.RawDeleteRangeResponse RawDeleteRange = 20; + kvrpcpb.RawBatchScanResponse RawBatchScan = 21; + + coprocessor.Response Coprocessor = 22; + kvrpcpb.PessimisticLockResponse PessimisticLock = 23; + kvrpcpb.PessimisticRollbackResponse PessimisticRollback = 24; + + kvrpcpb.CheckTxnStatusResponse CheckTxnStatus = 25; + kvrpcpb.TxnHeartBeatResponse TxnHeartBeat = 26; + + kvrpcpb.CheckSecondaryLocksResponse CheckSecondaryLocks = 33; + + kvrpcpb.RawCoprocessorResponse RawCoprocessor = 34; + + kvrpcpb.FlashbackToVersionResponse FlashbackToVersion = 35; + kvrpcpb.PrepareFlashbackToVersionResponse PrepareFlashbackToVersion = 36; + + kvrpcpb.FlushResponse Flush = 37; + kvrpcpb.BufferBatchGetResponse BufferBatchGet = 38; + + kvrpcpb.GetHealthFeedbackResponse GetHealthFeedback = 39; + + kvrpcpb.BroadcastTxnStatusResponse BroadcastTxnStatus = 40; + + // For some test cases. + BatchCommandsEmptyResponse Empty = 255; } + + // Reserved for deprecated VerKV + reserved 27 to 32; + } } message BatchRaftMessage { - repeated raft_serverpb.RaftMessage msgs = 1; + repeated raft_serverpb.RaftMessage msgs = 1; + // Used for measure the send duration. + uint64 last_observed_time = 13; } message BatchCommandsEmptyRequest { - // ID of the test request. - uint64 test_id = 1; - // TiKV needs to delay at least such a time to response the client. - uint64 delay_time = 2; + // ID of the test request. + uint64 test_id = 1; + // TiKV needs to delay at least such a time to response the client. + uint64 delay_time = 2; } message BatchCommandsEmptyResponse { - // ID of the test request. - uint64 test_id = 1; + // ID of the test request. + uint64 test_id = 1; } diff --git a/proto/tracepb.proto b/proto/tracepb.proto index c229454c..9065cb4d 100644 --- a/proto/tracepb.proto +++ b/proto/tracepb.proto @@ -1,60 +1,70 @@ syntax = "proto3"; package tracepb; +import "gogoproto/gogo.proto"; +import "rustproto.proto"; + option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (rustproto.lite_runtime_all) = true; service TraceRecordPubSub { - // Subscribe the Trace records generated on this service. The service will periodically (e.g. per minute) - // publishes Trace records to clients via gRPC stream. - rpc Subscribe(TraceRecordRequest) returns (stream TraceRecord) {} + // Subscribe the Trace records generated on this service. The service will periodically (e.g. per minute) + // publishes Trace records to clients via gRPC stream. + rpc Subscribe(TraceRecordRequest) returns (stream TraceRecord) {} } message TraceRecordRequest {} message TraceRecord { - oneof record_oneof { - Report report = 1; - NotifyCollect notify_collect = 2; - } + oneof record_oneof { + Report report = 1; + NotifyCollect notify_collect = 2; + } } message RemoteParentSpan { - // A unique id to identify the request. It's usually a UUID. - uint64 trace_id = 1; - // The span of remote caller that is awaiting the request. - uint64 span_id = 2; + // A unique id to identify the request. It's usually a UUID. + uint64 trace_id = 1; + // The span of remote caller that is awaiting the request. + uint64 span_id = 2; } // The context of the request to be traced. message TraceContext { - repeated RemoteParentSpan remote_parent_spans = 1; - // Report the trace records only if the duration of handling the request exceeds the threshold. - uint32 duration_threshold_ms = 2; + repeated RemoteParentSpan remote_parent_spans = 1; + // Report the trace records only if the duration of handling the request exceeds the threshold. + uint32 duration_threshold_ms = 2; } // Report the spans collected when handling a request on a service. message Report { - repeated RemoteParentSpan remote_parent_spans = 1; - repeated Span spans = 2; + repeated RemoteParentSpan remote_parent_spans = 1; + repeated Span spans = 2; } // Notify the subscriber to persis the spans of the trace. message NotifyCollect { - uint64 trace_id = 1; + uint64 trace_id = 1; } message Span { - // The unique span id within the spans with the same `trace_id`. - // The most significant 32 bits should be random number generated by each service instance. - uint64 span_id = 1; - uint64 parent_id = 2; - uint64 begin_unix_ns = 3; - uint64 duration_ns = 4; - string event = 5; - repeated Property properties = 6; + // The unique span id within the spans with the same `trace_id`. + // The most significant 32 bits should be random number generated by each service instance. + uint64 span_id = 1; + uint64 parent_id = 2; + uint64 begin_unix_ns = 3; + uint64 duration_ns = 4; + string event = 5; + repeated Property properties = 6; } message Property { - string key = 1; - string value = 2; + string key = 1; + string value = 2; } diff --git a/proto/tsopb.proto b/proto/tsopb.proto index 061bcadc..7a8f6a48 100644 --- a/proto/tsopb.proto +++ b/proto/tsopb.proto @@ -1,126 +1,134 @@ syntax = "proto3"; package tsopb; -import "pdpb.proto"; - import "gogoproto/gogo.proto"; +import "pdpb.proto"; import "rustproto.proto"; -option (gogoproto.sizer_all) = true; +option java_package = "org.tikv.kvproto"; +option (gogoproto.goproto_sizecache_all) = false; +option (gogoproto.goproto_unkeyed_all) = false; +option (gogoproto.goproto_unrecognized_all) = false; option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; option (gogoproto.unmarshaler_all) = true; option (rustproto.lite_runtime_all) = true; -option java_package = "org.tikv.kvproto"; - service TSO { - rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} - // Find the keyspace group that the keyspace belongs to by keyspace id. - rpc FindGroupByKeyspaceID (FindGroupByKeyspaceIDRequest) returns (FindGroupByKeyspaceIDResponse) {} - // Get the minimum timestamp across all keyspace groups served by the TSO server who receives - // and handle the request. If the TSO server/pod is not serving any keyspace group, return - // an empty timestamp, and the client needs to skip the empty timestamps when collecting - // the min timestamp from all TSO servers/pods. - rpc GetMinTS (GetMinTSRequest) returns (GetMinTSResponse) {} + rpc Tso(stream TsoRequest) returns (stream TsoResponse) {} + // Find the keyspace group that the keyspace belongs to by keyspace id. + rpc FindGroupByKeyspaceID(FindGroupByKeyspaceIDRequest) returns (FindGroupByKeyspaceIDResponse) {} + // Get the minimum timestamp across all keyspace groups served by the TSO server who receives + // and handle the request. If the TSO server/pod is not serving any keyspace group, return + // an empty timestamp, and the client needs to skip the empty timestamps when collecting + // the min timestamp from all TSO servers/pods. + rpc GetMinTS(GetMinTSRequest) returns (GetMinTSResponse) {} } message RequestHeader { - // cluster_id is the ID of the cluster which be sent to. - uint64 cluster_id = 1; - // sender_id is the ID of the sender server. - uint64 sender_id = 2; - - // keyspace_id is the unique id of the tenant/keyspace. - uint32 keyspace_id = 3; - // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. - uint32 keyspace_group_id = 4; + // cluster_id is the ID of the cluster which be sent to. + uint64 cluster_id = 1; + // sender_id is the ID of the sender server. + uint64 sender_id = 2; + + // keyspace_id is the unique id of the tenant/keyspace. + uint32 keyspace_id = 3; + // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. + uint32 keyspace_group_id = 4; + // callee_id is the ID of the server which the client expects to receive the request. + // such as tso-0, tso-1, pd-0, pd-1 etc. + // This field is used to check if the request is sent to the expected server. + // If it is not matched, the server will return an error. + string callee_id = 5; } message ResponseHeader { - // cluster_id is the ID of the cluster which sent the response. - uint64 cluster_id = 1; - Error error = 2; - - // keyspace_id is the unique id of the tenant/keyspace as the response receiver. - uint32 keyspace_id = 3; - // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. - uint32 keyspace_group_id = 4; + // cluster_id is the ID of the cluster which sent the response. + uint64 cluster_id = 1; + Error error = 2; + + // keyspace_id is the unique id of the tenant/keyspace as the response receiver. + uint32 keyspace_id = 3; + // keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. + uint32 keyspace_group_id = 4; } enum ErrorType { - OK = 0; - UNKNOWN = 1; - NOT_BOOTSTRAPPED = 2; - ALREADY_BOOTSTRAPPED = 3; - INVALID_VALUE = 4; - CLUSTER_MISMATCHED = 5; + OK = 0; + UNKNOWN = 1; + NOT_BOOTSTRAPPED = 2; + ALREADY_BOOTSTRAPPED = 3; + INVALID_VALUE = 4; + CLUSTER_MISMATCHED = 5; } message Error { - ErrorType type = 1; - string message = 2; + ErrorType type = 1; + string message = 2; } message TsoRequest { - RequestHeader header = 1; + RequestHeader header = 1; - uint32 count = 2; - string dc_location = 3; + uint32 count = 2; + string dc_location = 3; } message TsoResponse { - ResponseHeader header = 1; + ResponseHeader header = 1; - uint32 count = 2; - pdpb.Timestamp timestamp = 3; + uint32 count = 2; + pdpb.Timestamp timestamp = 3; } message Participant { - // name is the unique name of the TSO participant. - string name = 1; - // id is the unique id of the TSO participant. - uint64 id = 2; - // listen_urls is the serivce endpoint list in the url format. - // listen_urls[0] is primary service endpoint. - repeated string listen_urls = 3; + // name is the unique name of the TSO participant. + string name = 1; + // id is the unique id of the TSO participant. + uint64 id = 2; + // listen_urls is the serivce endpoint list in the url format. + // listen_urls[0] is primary service endpoint. + repeated string listen_urls = 3; } message KeyspaceGroupMember { - string address = 1; - bool is_primary = 2; + string address = 1; + bool is_primary = 2; } message SplitState { - uint32 split_source = 1; + uint32 split_source = 1; } message KeyspaceGroup { - uint32 id = 1; - string user_kind = 2; - SplitState split_state = 3; - repeated KeyspaceGroupMember members = 4; + uint32 id = 1; + string user_kind = 2; + SplitState split_state = 3; + repeated KeyspaceGroupMember members = 4; } - + message FindGroupByKeyspaceIDRequest { - RequestHeader header = 1; - uint32 keyspace_id = 2; + RequestHeader header = 1; + uint32 keyspace_id = 2; + uint64 mod_revision = 3; } message FindGroupByKeyspaceIDResponse { - ResponseHeader header = 1; - KeyspaceGroup keyspace_group = 2; + ResponseHeader header = 1; + KeyspaceGroup keyspace_group = 2; + uint64 mod_revision = 3; } message GetMinTSRequest { - RequestHeader header = 1; - string dc_location = 2; + RequestHeader header = 1; + string dc_location = 2; } message GetMinTSResponse { - ResponseHeader header = 1; - pdpb.Timestamp timestamp = 2; - // the count of keyspace group primaries that the TSO server/pod is serving - uint32 keyspace_groups_serving = 3; - // the total count of keyspace groups - uint32 keyspace_groups_total = 4; + ResponseHeader header = 1; + pdpb.Timestamp timestamp = 2; + // the count of keyspace group primaries that the TSO server/pod is serving + uint32 keyspace_groups_serving = 3; + // the total count of keyspace groups + uint32 keyspace_groups_total = 4; } diff --git a/src/generated/backup.rs b/src/generated/backup.rs index a283075f..1d4d331d 100644 --- a/src/generated/backup.rs +++ b/src/generated/backup.rs @@ -1,4 +1,36 @@ // This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrepareSnapshotBackupRequest { + /// The type of the request. + #[prost(enumeration = "PrepareSnapshotBackupRequestType", tag = "1")] + pub ty: i32, + /// The regions related to the request. + /// If the request is "UpdateLease" or "Finish", it should be empty. + #[prost(message, repeated, tag = "2")] + pub regions: ::prost::alloc::vec::Vec, + /// The lease duration for "UpdateLease" request. + #[prost(uint64, tag = "3")] + pub lease_in_seconds: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrepareSnapshotBackupResponse { + /// The type of the event. + #[prost(enumeration = "PrepareSnapshotBackupEventType", tag = "1")] + pub ty: i32, + /// The region bind to the type. + /// For "WaitApplyDone". + #[prost(message, optional, tag = "2")] + pub region: ::core::option::Option, + /// The error of the execution. + #[prost(message, optional, tag = "3")] + pub error: ::core::option::Option, + /// Whether the last lease is valid. + /// For "UpdateLease" and "Finish". + #[prost(bool, tag = "4")] + pub last_lease_is_valid: bool, +} /// The message save the metadata of a backup. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -39,6 +71,10 @@ pub struct BackupMeta { /// An index to files contains Schemas. #[prost(message, optional, tag = "14")] pub schema_index: ::core::option::Option, + /// Backup metadata compatibility version for restore. + /// 0 means the backup was created before this field existed. + #[prost(uint32, tag = "26")] + pub backup_schema_version: u32, /// If in raw kv mode, `start_versions`, `end_versions` and `schemas` will be /// ignored, and the backup data's range is represented by raw_ranges. #[prost(bool, tag = "8")] @@ -96,6 +132,20 @@ pub struct BackupRange { #[prost(message, repeated, tag = "3")] pub files: ::prost::alloc::vec::Vec, } +/// The message saves the physical table checksum +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableMeta { + /// physical id is either table id or partition id + #[prost(int64, tag = "1")] + pub physical_id: i64, + #[prost(uint64, tag = "2")] + pub crc64xor: u64, + #[prost(uint64, tag = "3")] + pub total_kvs: u64, + #[prost(uint64, tag = "4")] + pub total_bytes: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct File { @@ -111,12 +161,16 @@ pub struct File { pub start_version: u64, #[prost(uint64, tag = "6")] pub end_version: u64, + /// recorded the total statistics across multiple tables #[prost(uint64, tag = "7")] pub crc64xor: u64, #[prost(uint64, tag = "8")] pub total_kvs: u64, #[prost(uint64, tag = "9")] pub total_bytes: u64, + /// recorded when the SST kvs are from different tables + #[prost(message, repeated, tag = "13")] + pub table_metas: ::prost::alloc::vec::Vec, #[prost(string, tag = "10")] pub cf: ::prost::alloc::string::String, #[prost(uint64, tag = "11")] @@ -157,6 +211,43 @@ pub struct PlacementPolicy { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatsBlock { + /// The physical id which the stats json data belongs to. + #[prost(int64, tag = "1")] + pub physical_id: i64, + /// The marshal data of the part of json_table. Currently it is the partition table. + #[prost(bytes = "vec", tag = "2")] + pub json_table: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatsFile { + /// A set of files that contains json_tables. + #[prost(message, repeated, tag = "1")] + pub blocks: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatsFileIndex { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "2")] + pub sha256: ::prost::alloc::vec::Vec, + /// the size of the file content encrypted + #[prost(uint64, tag = "3")] + pub size_enc: u64, + /// the size of the original file content + #[prost(uint64, tag = "4")] + pub size_ori: u64, + /// cipher_iv is used for AES cipher + #[prost(bytes = "vec", tag = "5")] + pub cipher_iv: ::prost::alloc::vec::Vec, + /// If the size of the stats is very small, save the data into the metafile + #[prost(bytes = "vec", tag = "6")] + pub inline_data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Schema { #[prost(bytes = "vec", tag = "1")] pub db: ::prost::alloc::vec::Vec, @@ -174,6 +265,20 @@ pub struct Schema { /// #[prost(bytes = "vec", tag = "7")] pub stats: ::prost::alloc::vec::Vec, + /// An index of files that contains stats data + #[prost(message, repeated, tag = "8")] + pub stats_index: ::prost::alloc::vec::Vec, + /// is_merge_option_allowed indicates whether merge option is allowed for this schema. + /// For backward compatibility, this field defaults to false when not present. + #[prost(bool, tag = "9")] + pub is_merge_option_allowed: bool, + /// partition_merge_option_allowed indicates whether merge option is allowed for each partition. + /// is_merge_option_allowed is table level, and the partition_merge_option_allowed is partition level. + #[prost(map = "string, bool", tag = "10")] + pub partition_merge_option_allowed: ::std::collections::HashMap< + ::prost::alloc::string::String, + bool, + >, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -192,6 +297,9 @@ pub struct PitrTableMap { pub id_map: ::core::option::Option, #[prost(message, repeated, tag = "3")] pub partitions: ::prost::alloc::vec::Vec, + /// whether this table has been filtered out + #[prost(bool, tag = "4")] + pub filtered_out: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -202,6 +310,13 @@ pub struct PitrDbMap { pub id_map: ::core::option::Option, #[prost(message, repeated, tag = "3")] pub tables: ::prost::alloc::vec::Vec, + /// whether this db has been filtered out + #[prost(bool, tag = "4")] + pub filtered_out: bool, + /// whether this db has been reused + /// skip restoring the metakv of database in log restore if it is true + #[prost(bool, tag = "5")] + pub reused: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -244,6 +359,26 @@ pub mod error { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct StreamBackupTaskSecurityConfig { + #[prost(oneof = "stream_backup_task_security_config::Encryption", tags = "1, 2")] + pub encryption: ::core::option::Option< + stream_backup_task_security_config::Encryption, + >, +} +/// Nested message and enum types in `StreamBackupTaskSecurityConfig`. +pub mod stream_backup_task_security_config { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Encryption { + /// not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig + #[prost(message, tag = "1")] + PlaintextDataKey(super::CipherInfo), + #[prost(message, tag = "2")] + MasterKeyConfig(super::MasterKeyConfig), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct CipherInfo { #[prost(enumeration = "super::encryptionpb::EncryptionMethod", tag = "1")] pub cipher_type: i32, @@ -252,6 +387,20 @@ pub struct CipherInfo { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct MasterKeyConfig { + #[prost(enumeration = "super::encryptionpb::EncryptionMethod", tag = "1")] + pub encryption_type: i32, + #[prost(message, repeated, tag = "2")] + pub master_keys: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SortedSubRanges { + #[prost(message, repeated, tag = "1")] + pub sub_ranges: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct BackupRequest { #[prost(uint64, tag = "1")] pub cluster_id: u64, @@ -299,6 +448,10 @@ pub struct BackupRequest { /// only the remaining sub ranges of the range need to be backed up this time. #[prost(message, repeated, tag = "16")] pub sub_ranges: ::prost::alloc::vec::Vec, + /// only ranges in the same groups can be merged into one SST file. + /// the ranges in each groups are sorted. + #[prost(message, repeated, tag = "21")] + pub sorted_sub_ranges_groups: ::prost::alloc::vec::Vec, /// replica_read indicates whether to use replica read for backup. /// If it is false, the backup will only read data from leader. /// If it is true, the backup will read data from both leader and follower. @@ -325,7 +478,7 @@ pub struct StreamBackupTaskInfo { pub start_ts: u64, #[prost(uint64, tag = "3")] pub end_ts: u64, - /// Misc meta datas. + /// Misc meta data. /// The name of the task, also the ID of the task. #[prost(string, tag = "4")] pub name: ::prost::alloc::string::String, @@ -336,6 +489,9 @@ pub struct StreamBackupTaskInfo { /// compression type #[prost(enumeration = "CompressionType", tag = "6")] pub compression_type: i32, + /// security config for backup files + #[prost(message, optional, tag = "7")] + pub security_config: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -684,7 +840,7 @@ pub struct DataFileGroup { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct DataFileInfo { - /// SHA256 of the file. + /// Checksum of the plaintext file, i.e., pre-compression, pre-encryption. #[prost(bytes = "vec", tag = "1")] pub sha256: ::prost::alloc::vec::Vec, /// Path of the file. @@ -738,6 +894,34 @@ pub struct DataFileInfo { /// The compression type for the file. #[prost(enumeration = "CompressionType", tag = "18")] pub compression_type: i32, + /// The CRC64XOR of all contents of the file. + /// In older versions, this might be empty. + #[prost(uint64, tag = "19")] + pub crc64xor: u64, + /// The original region range the log file belongs to. + /// In older versions, this might be empty. + /// This may not be accurative: file may contain keys less than this. + #[prost(bytes = "vec", tag = "20")] + pub region_start_key: ::prost::alloc::vec::Vec, + /// The original region range the log file belongs to. + /// In older versions, this might be empty. + /// This may not be accurative: file may contain keys greater than this. + #[prost(bytes = "vec", tag = "21")] + pub region_end_key: ::prost::alloc::vec::Vec, + /// The region epoch that the log file belongs to. + /// In older versions, this might be empty. + /// If a region get split or merged during observing, the file may contain multi epoches. + /// This may not be complete: file may contain records from other versions. + /// + /// If there is exactly one epoch, + /// `region_start_key` and `region_end_key` must match this epoch. + #[prost(message, repeated, tag = "22")] + pub region_epoch: ::prost::alloc::vec::Vec, + /// Encryption information of this data file, not set if plaintext. + #[prost(message, optional, tag = "23")] + pub file_encryption_info: ::core::option::Option< + super::encryptionpb::FileEncryptionInfo, + >, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -755,6 +939,321 @@ pub struct StreamBackupError { #[prost(uint64, tag = "4")] pub store_id: u64, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Span { + #[prost(uint64, tag = "1")] + pub offset: u64, + #[prost(uint64, tag = "2")] + pub length: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SpansOfFile { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub spans: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileSubcompactionMeta { + /// The whole size of this subcompaction. + /// It should be the physical size of all input logs. + #[prost(uint64, tag = "1")] + pub size: u64, + /// The region that we perform the subcompaction. + #[prost(uint64, tag = "2")] + pub region_id: u64, + #[prost(string, tag = "3")] + pub cf: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub input_max_ts: u64, + #[prost(uint64, tag = "5")] + pub input_min_ts: u64, + /// The whole compaction's configure. + #[prost(uint64, tag = "6")] + pub compact_from_ts: u64, + #[prost(uint64, tag = "7")] + pub compact_until_ts: u64, + /// The key range of the input logs. + /// They are encoded user key. (i.e. mem-comparable, no 'z' prefix) + #[prost(bytes = "vec", tag = "8")] + pub min_key: ::prost::alloc::vec::Vec, + /// The largest key from the input. + #[prost(bytes = "vec", tag = "9")] + pub max_key: ::prost::alloc::vec::Vec, + /// The type of operations in the file. (Write or Delete) + #[prost(enumeration = "FileType", tag = "10")] + pub ty: i32, + #[prost(int64, tag = "11")] + pub table_id: i64, + /// The input log files of this. + #[prost(message, repeated, tag = "12")] + pub sources: ::prost::alloc::vec::Vec, +} +/// A subcompaction from a compaction. +/// Subcompactions are actions that collect then sort key values from one region. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileSubcompaction { + /// The metadata of the subcompaction. + #[prost(message, optional, tag = "1")] + pub meta: ::core::option::Option, + /// The generated SSTs. + #[prost(message, repeated, tag = "2")] + pub sst_outputs: ::prost::alloc::vec::Vec, + /// The hint for the history of the region that contains the keys being compacted. + /// This may be helpful for splitting, but it does not guarantee completeness and accuracy. + #[prost(message, repeated, tag = "3")] + pub region_meta_hints: ::prost::alloc::vec::Vec, +} +/// You may get region id from `LogFileSubcompactionMeta`. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionMetaHint { + /// The left boundary of the region of this epoch. + /// It is an byte-comparable encoded key without TS. + #[prost(bytes = "vec", tag = "1")] + pub start_key: ::prost::alloc::vec::Vec, + /// The right boundary of the region of the epoch. + #[prost(bytes = "vec", tag = "2")] + pub end_key: ::prost::alloc::vec::Vec, + /// The region epoch. + #[prost(message, optional, tag = "3")] + pub region_epoch: ::core::option::Option, +} +/// Batched version of `LogFileSubcompaction`. +/// So we can store many subcompactions to one file, to reduce the number of file. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileSubcompactions { + #[prost(message, repeated, tag = "1")] + pub subcompactions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogFileCompaction { + /// The selector of compaction source. + /// Files contains ANY record greater than this (and less than `until_ts`) will be selected. + /// The source and artificats may contain records out of the `from_ts` and `until_ts` range. + /// When restoring, please use `input_min_ts` to filter out not needed compactions instead. + #[prost(uint64, tag = "1")] + pub compaction_from_ts: u64, + /// The selector of compaction source. + /// Files contains ANY record less than this (and not less than `from_ts`) will be selected. + /// The source and artificats may contain records out of the `from_ts` and `until_ts` range. + /// When restoring, please use `input_max_ts` to filter out not needed compactions instead. + #[prost(uint64, tag = "2")] + pub compaction_until_ts: u64, + /// The user provided name for the compaction. + #[prost(string, tag = "3")] + pub name: ::prost::alloc::string::String, + /// The statistics or other useful information for human. + #[prost(string, tag = "4")] + pub comments: ::prost::alloc::string::String, + /// relative path to the generated subcompaction metas. ("artificates") + #[prost(string, tag = "5")] + pub artifacts: ::prost::alloc::string::String, + /// the XOR of all artificate's hash. + #[prost(uint64, tag = "6")] + pub artifacts_hash: u64, + /// relative path to the subcompaction generated SST files. + /// this is a hint for garbage-collecting, so we don't need to read the metas one by one. + /// Those SSTs cannot be directly used, what you need is probably in `artificates`. + #[prost(string, tag = "7")] + pub generated_files: ::prost::alloc::string::String, + /// The smallest TS of record contained among all selected files of this compaction. + #[prost(uint64, tag = "8")] + pub input_min_ts: u64, + /// The largest TS of record contained among all selected files of this compaction. + #[prost(uint64, tag = "9")] + pub input_max_ts: u64, + /// Total size of generated compact-log-backup cmeta files. + #[prost(uint64, tag = "10")] + pub generated_cmeta_files_total_size: u64, + /// Total size of generated compact-log-backup SST files. + #[prost(uint64, tag = "11")] + pub generated_sst_files_total_size: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MetaEdit { + /// Path to the meta file. + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + /// Delete the physical files (MetaFileGroup) in the meta file. + #[prost(string, repeated, tag = "2")] + pub delete_physical_files: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Delete the logical files (MetaFileInfo) in the meta file. + /// Note: Even the operation have been performed in the meta, + /// this modification should be kept as long as the corresponding physical + /// file not deleted. Or we may cannot know when to delete the physical file. + /// Then the file will be leak until truncated. + #[prost(message, repeated, tag = "3")] + pub delete_logical_files: ::prost::alloc::vec::Vec, + /// Whether to delete the metadata itself. + #[prost(bool, tag = "4")] + pub destruct_self: bool, + /// Whether all data kv files have been compacted but meta kv file still remains. + /// this field used to determine whether we can ignore the physical data kv log files in a compaction. + #[prost(bool, tag = "5")] + pub all_data_files_compacted: bool, +} +/// RewrittenTableID records a sort of modification over the SSTs during restoring. +/// +/// When "backing up" this "restored" SST, we want to backup the original SST before any rewrite. +/// But in some cases, SSTs are not restored "as is", TiKV may rewrite or truncate its +/// content by the client's request. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RewrittenTableId { + /// The table ID in the backup data. + #[prost(int64, tag = "1")] + pub ancestor_upstream: i64, + /// The rewritten table ID during restoring. + #[prost(int64, tag = "2")] + pub upstream: i64, +} +/// IngestedSSTs is created by a client that wants to put data to the cluster by `Ingest` APIs, +/// these write cannot be directly recorded by log backup. The client should put a migration +/// that contains the SSTs it uploaded. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestedSsTs { + /// The table IDs rewritten during restoring. + #[prost(message, repeated, tag = "1")] + pub rewritten_tables: ::prost::alloc::vec::Vec, + /// The SST files restored. + #[prost(message, repeated, tag = "2")] + pub files: ::prost::alloc::vec::Vec, + /// Treating the whole batch of SSTs as an huge atomic + /// write as this timestamp. + /// That is, when user wants to PiTR to sometime after this, + /// everything restored will present after the PiTR. + /// Otherwise nothing will present. + #[prost(uint64, tag = "3")] + pub as_if_ts: u64, + /// The hint of the common prefix of the files. + /// Used for speed up garbage collecting. + #[prost(string, tag = "4")] + pub files_prefix_hint: ::prost::alloc::string::String, + /// Whether this batch have finished. + /// If not, `as_if_ts` should be ignored and this should only + /// be restored when there is a finished backup with the same + /// `backup_uuid`. + #[prost(bool, tag = "5")] + pub finished: bool, + /// When checkpoint enabled, one restoration may be separated to many + /// of `IngestedSST`s. + /// `IngestedSST`s sharing the same UUID should be treated as one. + /// That is: + /// + /// * The `as_if_ts` should be the largest one among all finished ingested SST batches. + /// * When restoring, all batches sharing the same uuid should be restored or not. + #[prost(bytes = "vec", tag = "6")] + pub backup_uuid: ::prost::alloc::vec::Vec, +} +/// An extended version of `SpansOfFile`, added more metadata for the +/// execution of delayed deletion. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteSpansOfFile { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub spans: ::prost::alloc::vec::Vec, + /// This is a hint for garbage collecting. + /// So we can easily check whether a set of spans consists the whole file. + #[prost(uint64, tag = "3")] + pub whole_file_length: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Migration { + #[prost(string, tag = "5")] + pub creator: ::prost::alloc::string::String, + #[prost(enumeration = "MigrationVersion", tag = "6")] + pub version: i32, + #[prost(message, repeated, tag = "1")] + pub edit_meta: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "2")] + pub truncated_to: u64, + #[prost(message, repeated, tag = "3")] + pub compactions: ::prost::alloc::vec::Vec, + /// Delete all files matches this prefix. + #[prost(string, repeated, tag = "4")] + pub destruct_prefix: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// The ingested SSTs. + /// It is mutable during restoring, hence a reference. + #[prost(string, repeated, tag = "7")] + pub ingested_sst_paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum PrepareSnapshotBackupRequestType { + /// Update the lease of suspending some commands (Which may impact the + /// disk snapshot backup) to be proposed. + /// As long as the lease is kept, those commands may not be proposed. + /// If the client has gone, the lease will be destroyed after its TTL. + UpdateLease = 0, + /// Wait a region apply to the last index. + WaitApply = 1, + /// Hint before closing the stream. Server will go back to normal after this. + /// This should also return a final "UpdateLeaseResult" to the client, which + /// indices whether the last lease is valid. + Finish = 2, +} +impl PrepareSnapshotBackupRequestType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + PrepareSnapshotBackupRequestType::UpdateLease => "UpdateLease", + PrepareSnapshotBackupRequestType::WaitApply => "WaitApply", + PrepareSnapshotBackupRequestType::Finish => "Finish", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UpdateLease" => Some(Self::UpdateLease), + "WaitApply" => Some(Self::WaitApply), + "Finish" => Some(Self::Finish), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum PrepareSnapshotBackupEventType { + /// A region has finished wait apply. + WaitApplyDone = 0, + /// A lease has been updated. + UpdateLeaseResult = 1, +} +impl PrepareSnapshotBackupEventType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + PrepareSnapshotBackupEventType::WaitApplyDone => "WaitApplyDone", + PrepareSnapshotBackupEventType::UpdateLeaseResult => "UpdateLeaseResult", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "WaitApplyDone" => Some(Self::WaitApplyDone), + "UpdateLeaseResult" => Some(Self::UpdateLeaseResult), + _ => None, + } + } +} /// sst files or log files compression algorithm /// for log files, unknown means not use compression algorithm #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] @@ -875,6 +1374,38 @@ impl FileType { } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum MigrationVersion { + /// The initial version. Somehow "UNKNOWN". + M0 = 0, + /// Added `creator` and `version`. + M1 = 1, + /// Added `ingested_sst_paths`. + M2 = 2, +} +impl MigrationVersion { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + MigrationVersion::M0 => "M0", + MigrationVersion::M1 => "M1", + MigrationVersion::M2 => "M2", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "M0" => Some(Self::M0), + "M1" => Some(Self::M1), + "M2" => Some(Self::M2), + _ => None, + } + } +} /// Generated client implementations. pub mod backup_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -1010,6 +1541,37 @@ pub mod backup_client { .insert(GrpcMethod::new("backup.Backup", "CheckPendingAdminOp")); self.inner.server_streaming(req, path, codec).await } + /// PrepareSnapshotBackup is an advanced version of preparing snapshot backup. + /// Check the defination of `PrepareSnapshotBackupRequest` for more. + pub async fn prepare_snapshot_backup( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::PrepareSnapshotBackupRequest, + >, + ) -> std::result::Result< + tonic::Response< + tonic::codec::Streaming, + >, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/backup.Backup/PrepareSnapshotBackup", + ); + let mut req = request.into_streaming_request(); + req.extensions_mut() + .insert(GrpcMethod::new("backup.Backup", "PrepareSnapshotBackup")); + self.inner.streaming(req, path, codec).await + } /// prepare is used for file-copy backup. before we start the backup for a TiKV. /// we need invoke this function to generate the SST files map. or we get nothing to backup. pub async fn prepare( diff --git a/src/generated/cdcpb.rs b/src/generated/cdcpb.rs index e80572e2..5d2a993f 100644 --- a/src/generated/cdcpb.rs +++ b/src/generated/cdcpb.rs @@ -31,6 +31,15 @@ pub struct ClusterIdMismatch { #[prost(uint64, tag = "2")] pub request: u64, } +/// Congested is an error variable that +/// tells people that the TiKV-CDC is congested. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Congested { + /// The region ID that triggers the congestion. + #[prost(uint64, tag = "1")] + pub region_id: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Error { @@ -48,6 +57,8 @@ pub struct Error { pub cluster_id_mismatch: ::core::option::Option, #[prost(message, optional, tag = "7")] pub server_is_busy: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub congested: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -92,6 +103,9 @@ pub mod event { pub commit_ts: u64, #[prost(enumeration = "LogType", tag = "3")] pub r#type: i32, + /// generation is for pipelined DML protocol. See kvrpcpb.FlushRequest.generation. + #[prost(uint64, tag = "10")] + pub generation: u64, #[prost(enumeration = "row::OpType", tag = "4")] pub op_type: i32, #[prost(bytes = "vec", tag = "5")] diff --git a/src/generated/coprocessor.rs b/src/generated/coprocessor.rs index 907f9697..16676352 100644 --- a/src/generated/coprocessor.rs +++ b/src/generated/coprocessor.rs @@ -8,6 +8,16 @@ pub struct KeyRange { #[prost(bytes = "vec", tag = "2")] pub end: ::prost::alloc::vec::Vec, } +/// KeyRange with an attached read_ts (version). +/// It is used by TiCI versioned lookup. Callers must ensure `range` is a point range. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct VersionedKeyRange { + #[prost(message, optional, tag = "1")] + pub range: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub read_ts: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { @@ -44,6 +54,33 @@ pub struct Request { /// This is the session alias between a client and tidb #[prost(string, tag = "13")] pub connection_alias: ::prost::alloc::string::String, + /// Shard infos for FTS index, used by TiFlash reading TiCI. + #[prost(message, repeated, tag = "14")] + pub table_shard_infos: ::prost::alloc::vec::Vec, + /// Versioned point ranges for TiCI lookup. + /// When `versioned_ranges` is non-empty, all `versioned_ranges\[i\].range` must be point range. + #[prost(message, repeated, tag = "15")] + pub versioned_ranges: ::prost::alloc::vec::Vec, + /// max_keys_read is 0 when disabled, otherwise limits the number of storage + /// engine keys scanned per coprocessor task. It is a per-task hard ceiling + /// applied uniformly to any coprocessor request, used to bound worst-case + /// scan amplification regardless of how the request is paginated. + #[prost(uint64, tag = "16")] + pub max_keys_read: u64, + /// paging_size_bytes is 0 when disabled, otherwise it should be a positive number. + /// When set, within a paged coprocessor request, TiKV stops scanning the current + /// page once accumulated scanned bytes reach this limit and returns the page + /// boundary so the next page can resume from there. + /// + /// Unlike max_keys_read, which is a per-task hard ceiling on keys scanned, + /// paging_size_bytes is an independent rate-control dimension intended for + /// Resource Control: it bounds the byte volume of a single page so that RU + /// pre-charging in PD's resource controller can be performed at byte + /// granularity, complementing the existing row-count-based paging + /// (paging_size). The two fields target different scenarios and may be set + /// independently. + #[prost(uint64, tag = "17")] + pub paging_size_bytes: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -92,6 +129,54 @@ pub struct RegionInfo { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ShardInfo { + #[prost(uint64, tag = "1")] + pub shard_id: u64, + #[prost(uint64, tag = "2")] + pub shard_epoch: u64, + #[prost(message, repeated, tag = "3")] + pub ranges: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableShardInfos { + /// The executor ID is used to identify the tici executor. + #[prost(string, tag = "1")] + pub executor_id: ::prost::alloc::string::String, + /// The shard_infos contains the shard information for each tici executor. + #[prost(message, repeated, tag = "2")] + pub shard_infos: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TiCiEstimateCountRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub start_ts: u64, + #[prost(int64, tag = "3")] + pub table_id: i64, + #[prost(int64, tag = "4")] + pub index_id: i64, + #[prost(bytes = "vec", tag = "5")] + pub fts_query_info: ::prost::alloc::vec::Vec, + #[prost(string, tag = "6")] + pub time_zone_name: ::prost::alloc::string::String, + #[prost(int64, tag = "7")] + pub time_zone_offset: i64, + #[prost(message, repeated, tag = "8")] + pub shard_infos: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TiCiEstimateCountResponse { + #[prost(uint64, tag = "1")] + pub est_count: u64, + #[prost(string, tag = "2")] + pub other_error: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TableRegions { #[prost(int64, tag = "1")] pub physical_table_id: i64, @@ -125,6 +210,9 @@ pub struct BatchRequest { /// This is the session alias between a client and tidb #[prost(string, tag = "10")] pub connection_alias: ::prost::alloc::string::String, + /// Shard infos for FTS index, used by TiFlash reading TiCI. + #[prost(message, repeated, tag = "11")] + pub table_shard_infos: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -137,6 +225,8 @@ pub struct BatchResponse { pub exec_details: ::core::option::Option, #[prost(message, repeated, tag = "4")] pub retry_regions: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub retry_shards: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -151,6 +241,13 @@ pub struct StoreBatchTask { pub ranges: ::prost::alloc::vec::Vec, #[prost(uint64, tag = "5")] pub task_id: u64, + /// Versioned point ranges for TiCI lookup. + /// When `versioned_ranges` is non-empty, all `versioned_ranges\[i\].range` must be point range. + #[prost(message, repeated, tag = "6")] + pub versioned_ranges: ::prost::alloc::vec::Vec, + /// Bucket metadata version used to validate this child task. + #[prost(uint64, tag = "7")] + pub buckets_version: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -168,3 +265,38 @@ pub struct StoreBatchTaskResponse { #[prost(message, optional, tag = "6")] pub exec_details_v2: ::core::option::Option, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DelegateRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub start_ts: u64, + #[prost(message, repeated, tag = "3")] + pub ranges: ::prost::alloc::vec::Vec, + /// Used for avoid redundant mem-table copying. + /// If the sequence is the same, tikv-server will not return the mem-table. + #[prost(uint64, tag = "4")] + pub mem_table_sequence: u64, + /// Used for avoid redundant snapshot copying. + /// If the sequence is the same, tikv-server will not return the snapshot. + #[prost(uint64, tag = "5")] + pub snapshot_sequence: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DelegateResponse { + #[prost(bytes = "vec", tag = "1")] + pub mem_table_data: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "2")] + pub snapshot: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "3")] + pub region_error: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub locked: ::core::option::Option, + #[prost(string, tag = "5")] + pub other_error: ::prost::alloc::string::String, + /// Used for avoid redundant mem-table copying. + #[prost(uint64, tag = "6")] + pub mem_table_sequence: u64, +} diff --git a/src/generated/deadlock.rs b/src/generated/deadlock.rs index ef9c61e3..da4c957a 100644 --- a/src/generated/deadlock.rs +++ b/src/generated/deadlock.rs @@ -32,11 +32,31 @@ pub struct WaitForEntry { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReplaceLockByKeyItem { + #[prost(uint64, tag = "1")] + pub key_hash: u64, + #[prost(bytes = "vec", tag = "2")] + pub key: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "3")] + pub old_lock_ts: u64, + #[prost(uint64, tag = "4")] + pub new_lock_ts: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReplaceLocksByKeysRequest { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct DeadlockRequest { #[prost(enumeration = "DeadlockRequestType", tag = "1")] pub tp: i32, #[prost(message, optional, tag = "2")] pub entry: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub replace_locks_by_keys: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -44,13 +64,16 @@ pub struct DeadlockResponse { /// The same entry sent by DeadlockRequest, identifies the sender. #[prost(message, optional, tag = "1")] pub entry: ::core::option::Option, - /// The key hash of the lock that is hold by the waiting transaction. + /// The key hash of the lock that is hold by the waiting transaction. The hash of the `deadlock_key` field. #[prost(uint64, tag = "2")] pub deadlock_key_hash: u64, /// The other entries of the dead lock circle. The current entry is in `entry` field and not /// included in this field. #[prost(message, repeated, tag = "3")] pub wait_chain: ::prost::alloc::vec::Vec, + /// The key of the lock that is hold by the waiting transaction. + #[prost(bytes = "vec", tag = "4")] + pub deadlock_key: ::prost::alloc::vec::Vec, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/src/generated/encryptionpb.rs b/src/generated/encryptionpb.rs index f7d7a924..6a944b9d 100644 --- a/src/generated/encryptionpb.rs +++ b/src/generated/encryptionpb.rs @@ -108,6 +108,53 @@ pub struct MasterKeyKms { /// KMS endpoint. Normally not needed. #[prost(string, tag = "4")] pub endpoint: ::prost::alloc::string::String, + /// optional, used to set up azure master key backend + #[prost(message, optional, tag = "5")] + pub azure_kms: ::core::option::Option, + /// optional, used to set up gcp master key backend + #[prost(message, optional, tag = "6")] + pub gcp_kms: ::core::option::Option, + /// optional, used to set up aws master key backend + #[prost(message, optional, tag = "7")] + pub aws_kms: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AzureKms { + #[prost(string, tag = "1")] + pub tenant_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub client_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub client_secret: ::prost::alloc::string::String, + /// Key vault to encrypt/decrypt data key. + #[prost(string, tag = "4")] + pub key_vault_url: ::prost::alloc::string::String, + /// optional hsm used to generate data key + #[prost(string, tag = "5")] + pub hsm_name: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub hsm_url: ::prost::alloc::string::String, + #[prost(string, tag = "7")] + pub client_certificate: ::prost::alloc::string::String, + #[prost(string, tag = "8")] + pub client_certificate_path: ::prost::alloc::string::String, + #[prost(string, tag = "9")] + pub client_certificate_password: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GcpKms { + #[prost(string, tag = "1")] + pub credential: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AwsKms { + #[prost(string, tag = "1")] + pub access_key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub secret_access_key: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -134,6 +181,44 @@ pub struct EncryptedContent { #[prost(bytes = "vec", tag = "5")] pub ciphertext_key: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FileEncryptionInfo { + /// file encryption method + #[prost(enumeration = "EncryptionMethod", tag = "3")] + pub encryption_method: i32, + /// iv to encrypt the file by data key + #[prost(bytes = "vec", tag = "4")] + pub file_iv: ::prost::alloc::vec::Vec, + /// file checksum after encryption, optional if using GCM + #[prost(bytes = "vec", tag = "5")] + pub checksum: ::prost::alloc::vec::Vec, + #[prost(oneof = "file_encryption_info::Mode", tags = "1, 2")] + pub mode: ::core::option::Option, +} +/// Nested message and enum types in `FileEncryptionInfo`. +pub mod file_encryption_info { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Mode { + #[prost(message, tag = "1")] + PlainTextDataKey(super::PlainTextDataKey), + #[prost(message, tag = "2")] + MasterKeyBased(super::MasterKeyBased), + } +} +/// not recommended in production. +/// user needs to pass back the same data key for restore. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PlainTextDataKey {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MasterKeyBased { + /// encrypted data key with metadata + #[prost(message, repeated, tag = "1")] + pub data_key_encrypted_content: ::prost::alloc::vec::Vec, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum EncryptionMethod { diff --git a/src/generated/errorpb.rs b/src/generated/errorpb.rs index 4ed789b0..ddba0b60 100644 --- a/src/generated/errorpb.rs +++ b/src/generated/errorpb.rs @@ -207,6 +207,14 @@ pub struct MismatchPeerId { #[prost(uint64, tag = "2")] pub store_peer_id: u64, } +/// UndeterminedResult is the error variant that tells the result is not determined yet. +/// For example, the raft protocol timed out and the apply result is unknown. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UndeterminedResult { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, +} /// Error wraps all region errors, indicates an error encountered by a request. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -263,4 +271,7 @@ pub struct Error { /// BucketVersionNotMatch is the error variant that tells the request buckets version is not match. #[prost(message, optional, tag = "21")] pub bucket_version_not_match: ::core::option::Option, + /// UndeterminedResult is the error variant that tells the result is not determined yet. + #[prost(message, optional, tag = "22")] + pub undetermined_result: ::core::option::Option, } diff --git a/src/generated/import_sstpb.rs b/src/generated/import_sstpb.rs index 4f8a1274..f1169ae0 100644 --- a/src/generated/import_sstpb.rs +++ b/src/generated/import_sstpb.rs @@ -90,8 +90,15 @@ pub struct RewriteRule { pub old_key_prefix: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "2")] pub new_key_prefix: ::prost::alloc::vec::Vec, + /// (Optional) Rewrite all keys in the range to use this timestamp. #[prost(uint64, tag = "3")] pub new_timestamp: u64, + /// (Optional) Skip keys with timestamps greater than this during download, useful for compacted SST backups. + #[prost(uint64, tag = "4")] + pub ignore_after_timestamp: u64, + /// (Optional) Skip write CF keys with timestamps less than this during download. Default CF keys are preserved. + #[prost(uint64, tag = "5")] + pub ignore_before_timestamp: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -185,6 +192,8 @@ pub struct DownloadRequest { /// same order, otherwise the RPC request will fail. #[prost(message, optional, tag = "13")] pub rewrite_rule: ::core::option::Option, + #[prost(message, repeated, tag = "20")] + pub sorted_rewrite_rules: ::prost::alloc::vec::Vec, #[prost(message, optional, tag = "14")] pub storage_backend: ::core::option::Option, /// The identity for the stroage backend. @@ -246,6 +255,12 @@ pub struct SetDownloadSpeedLimitRequest { /// The download speed limit (bytes/second). Set to 0 for unlimited speed. #[prost(uint64, tag = "1")] pub speed_limit: u64, + /// The download speed limit task id. + #[prost(string, tag = "2")] + pub task_id: ::prost::alloc::string::String, + /// The download speed limit ttl. Set to 0 means it will never expire. + #[prost(uint64, tag = "3")] + pub ttl_seconds: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -464,6 +479,11 @@ pub struct KvMeta { /// the compression type for the file. #[prost(enumeration = "super::backup::CompressionType", tag = "13")] pub compression_type: i32, + /// encryption information of the kv file, not set if encryption is not enabled. + #[prost(message, optional, tag = "14")] + pub file_encryption_info: ::core::option::Option< + super::encryptionpb::FileEncryptionInfo, + >, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -498,9 +518,12 @@ pub struct ApplyRequest { /// context represents region info and it used to build raft commands. #[prost(message, optional, tag = "4")] pub context: ::core::option::Option, - /// cipher_info is used to decrypt kv file when download file. + /// plaintext data key to decrypt kv file if configured during log backup. #[prost(message, optional, tag = "11")] pub cipher_info: ::core::option::Option, + /// master keys config used to decrypt data keys in restore if configured during log backup. + #[prost(message, repeated, tag = "14")] + pub master_keys: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -525,6 +548,31 @@ pub struct ClearResponse { #[prost(message, optional, tag = "1")] pub error: ::core::option::Option, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AddPartitionRangeRequest { + #[prost(message, optional, tag = "1")] + pub range: ::core::option::Option, + /// the number of seconds this range is valid. + /// After this duration, if this range is still not removed by + /// `RemoveForcePartitionRange`, tikv will automically remove it. So the client + /// should set a big enough value to avoid auto cleanup. But in general, 1h + /// should be a big enough value. If its value is 0, tikv will auto adjust it to 3600(1h). + #[prost(uint64, tag = "2")] + pub ttl_seconds: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AddPartitionRangeResponse {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemovePartitionRangeRequest { + #[prost(message, optional, tag = "1")] + pub range: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RemovePartitionRangeResponse {} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum SwitchMode { @@ -858,6 +906,62 @@ pub mod import_sst_client { .insert(GrpcMethod::new("import_sstpb.ImportSST", "Download")); self.inner.unary(req, path, codec).await } + /// Download SST files in batch from external storage, perform key rewrite, + /// and merge them into one SST after downloading. + pub async fn batch_download( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/BatchDownload", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("import_sstpb.ImportSST", "BatchDownload")); + self.inner.unary(req, path, codec).await + } + /// Download SST files in batch from external storage, perform key rewrite, + /// merge them into one SST, and keep only the latest MVCC version after downloading. + pub async fn batch_download_latest_mvcc( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/BatchDownloadLatestMVCC", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("import_sstpb.ImportSST", "BatchDownloadLatestMVCC"), + ); + self.inner.unary(req, path, codec).await + } /// Open a write stream to generate sst files pub async fn write( &mut self, @@ -1027,5 +1131,68 @@ pub mod import_sst_client { .insert(GrpcMethod::new("import_sstpb.ImportSST", "SuspendImportRPC")); self.inner.unary(req, path, codec).await } + /// AddForcePartitionRange marks a range in tikv that any compact overlaps with this range + /// should generates SST files partitioned at region boundaries as well as this range boundary. + /// TiKV will also try to do manual compact(if needed) after setting this range to eusure + /// any incoming SST under this range can be ingested into the bottom level if there is no real kv overlap. + pub async fn add_force_partition_range( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/AddForcePartitionRange", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("import_sstpb.ImportSST", "AddForcePartitionRange"), + ); + self.inner.unary(req, path, codec).await + } + /// Remove the force partition range after the task is finished. If this function is not called, + /// tikv will cleanup the range after TTL to ensure it can be cleaned eventually. + pub async fn remove_force_partition_range( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/import_sstpb.ImportSST/RemoveForcePartitionRange", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "import_sstpb.ImportSST", + "RemoveForcePartitionRange", + ), + ); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/keyspacepb.rs b/src/generated/keyspacepb.rs index 3cf798d9..53e3bb00 100644 --- a/src/generated/keyspacepb.rs +++ b/src/generated/keyspacepb.rs @@ -28,6 +28,14 @@ pub struct LoadKeyspaceRequest { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct LoadKeyspaceByIdRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(uint32, tag = "2")] + pub id: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct LoadKeyspaceResponse { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -227,6 +235,31 @@ pub mod keyspace_client { .insert(GrpcMethod::new("keyspacepb.Keyspace", "LoadKeyspace")); self.inner.unary(req, path, codec).await } + pub async fn load_keyspace_by_id( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/keyspacepb.Keyspace/LoadKeyspaceByID", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("keyspacepb.Keyspace", "LoadKeyspaceByID")); + self.inner.unary(req, path, codec).await + } /// WatchKeyspaces first return all current keyspaces' metadata as its first response. /// Then, it returns responses containing keyspaces that had their metadata changed. pub async fn watch_keyspaces( diff --git a/src/generated/kvrpcpb.rs b/src/generated/kvrpcpb.rs index 4bf66314..89f2d472 100644 --- a/src/generated/kvrpcpb.rs +++ b/src/generated/kvrpcpb.rs @@ -10,6 +10,9 @@ pub struct GetRequest { pub key: ::prost::alloc::vec::Vec, #[prost(uint64, tag = "3")] pub version: u64, + /// If true, the response will include the commit ts of the key. + #[prost(bool, tag = "4")] + pub need_commit_ts: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -30,6 +33,10 @@ pub struct GetResponse { /// Time and scan details when processing the request. #[prost(message, optional, tag = "6")] pub exec_details_v2: ::core::option::Option, + /// The commit timestamp of the key. + /// If it is zero, it means the commit timestamp is unknown. + #[prost(uint64, tag = "7")] + pub commit_ts: u64, } /// Scan fetches values for a range of keys; it is part of the transaction with /// starting timestamp = `version`. @@ -166,10 +173,13 @@ pub mod prewrite_request { #[repr(i32)] pub enum PessimisticAction { /// The key needn't be locked and no extra write conflict checks are needed. + /// Deprecated in next-gen (cloud-storage-engine). SkipPessimisticCheck = 0, - /// The key should have been locked at the time of prewrite. + /// The key should have been locked at the time of prewrite. If the lock is missing, + /// the lock will be amended. This is the normal case for pessimistic transactions. DoPessimisticCheck = 1, - /// The key doesn't need a pessimistic lock. But we need to do data constraint checks. + /// The key does not acquire a pessimistic lock for performance optimization. + /// Constraint checking (write conflicts + data constraints) is deferred to prewrite. DoConstraintCheck = 2, } impl PessimisticAction { @@ -366,6 +376,11 @@ pub struct TxnHeartBeatRequest { /// The new TTL the sender would like. #[prost(uint64, tag = "4")] pub advise_lock_ttl: u64, + /// Optionally update PK's min_commit_ts. + /// Only for non-async-commmit and non-1PC transactions. + /// If it is 0, ignore this field. + #[prost(uint64, tag = "5")] + pub min_commit_ts: u64, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -506,6 +521,17 @@ pub struct CommitRequest { /// Timestamp for the end of the transaction. Must be greater than `start_version`. #[prost(uint64, tag = "4")] pub commit_version: u64, + /// commit_role indicates the current commit request is a primary commit or a secondary commit. + /// It's value maybe `Unknown` when using a client with an old version. + #[prost(enumeration = "CommitRole", tag = "6")] + pub commit_role: i32, + /// primary_key indicates the primary key of the transaction. + /// Its value may be empty when using an old version client. + #[prost(bytes = "vec", tag = "7")] + pub primary_key: ::prost::alloc::vec::Vec, + /// Whether committing an `use_async_commit` (i.e. should be treated as committed) prewrite. + #[prost(bool, tag = "8")] + pub use_async_commit: bool, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -579,6 +605,9 @@ pub struct BatchGetRequest { pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, #[prost(uint64, tag = "3")] pub version: u64, + /// If true, the response will include the commit ts of the key. + #[prost(bool, tag = "4")] + pub need_commit_ts: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -676,6 +705,9 @@ pub struct ResolveLockRequest { /// Only resolve specified keys. #[prost(bytes = "vec", repeated, tag = "5")] pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// If true, the resolve lock action is performed asynchronously and the response may be returned before it's finished. + #[prost(bool, tag = "6")] + pub is_async: bool, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -1112,6 +1144,9 @@ pub struct SplitRegionResponse { /// include all result regions. #[prost(message, repeated, tag = "4")] pub regions: ::prost::alloc::vec::Vec, + /// Reserved for file based transaction. + #[prost(message, repeated, tag = "100")] + pub errors: ::prost::alloc::vec::Vec, } /// Sent from TiFlash to a TiKV node. #[allow(clippy::derive_partial_eq_without_eq)] @@ -1255,6 +1290,17 @@ pub struct Context { /// Some information used for resource control. #[prost(message, optional, tag = "28")] pub resource_control_context: ::core::option::Option, + /// The declared origin of the request. TiDB server requests set this to TIDB. + /// This field is client-provided metadata; consumers should rely on it only when + /// the caller is authenticated/authorized by the transport or deployment boundary. + /// UNKNOWN means the origin is unset or not recognized by this protocol version. + /// Other components should add dedicated enum variants when they need origin-specific behavior. + #[prost(enumeration = "RequestOrigin", tag = "29")] + pub request_origin: i32, + /// The keyspace that the request is sent to. + /// NOTE: This field is only meaningful while the api_version is V2. + #[prost(string, tag = "31")] + pub keyspace_name: ::prost::alloc::string::String, /// The keyspace that the request is sent to. /// NOTE: This field is only meaningful while the api_version is V2. #[prost(uint32, tag = "32")] @@ -1267,6 +1313,21 @@ pub struct Context { /// This is for tests only and thus can be safely changed/removed without affecting compatibility. #[prost(message, optional, tag = "34")] pub source_stmt: ::core::option::Option, + /// The cluster id of the request + #[prost(uint64, tag = "35")] + pub cluster_id: u64, + /// The trace id of the request, will be used for tracing the request's execution's inner steps. + #[prost(bytes = "vec", tag = "36")] + pub trace_id: ::prost::alloc::vec::Vec, + /// Control flags for trace logging behavior. + /// Bit 0: immediate_log - Force immediate logging without buffering + /// Bit 1: category_req_resp - Enable request/response tracing + /// Bit 2: category_write_details - Enable detailed write tracing + /// Bit 3: category_read_details - Enable detailed read tracing + /// Bits 4-63: Reserved for future use + /// This field is set by client-go based on an extractor function provided by TiDB. + #[prost(uint64, tag = "37")] + pub trace_control_flags: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1326,6 +1387,14 @@ pub struct LockInfo { /// It can be used to help the client decide whether to try resolving the lock. #[prost(uint64, tag = "11")] pub duration_to_last_update_ms: u64, + /// When lock_type is SharedLock, this describes transactions holding the shared lock. + /// Important: when lock_type is SharedLock, all shared locks must use shared_lock_infos; + /// DO NOT read from the wrapper LockInfo. + /// TODO(slock): tidb should send requests with a feature flag to indicate whether it + /// supports shared locks, so that tikv can fail the requests from old tidb versions + /// when needed. + #[prost(message, repeated, tag = "12")] + pub shared_lock_infos: ::prost::alloc::vec::Vec, /// Reserved for file based transaction. #[prost(bool, tag = "100")] pub is_txn_file: bool, @@ -1366,6 +1435,12 @@ pub struct KeyError { /// CheckTxnStatus is sent to a lock that's not the primary. #[prost(message, optional, tag = "11")] pub primary_mismatch: ::core::option::Option, + /// TxnLockNotFound indicates the txn lock is not found. + #[prost(message, optional, tag = "12")] + pub txn_lock_not_found: ::core::option::Option, + /// Extra information for error debugging + #[prost(message, optional, tag = "100")] + pub debug_info: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1407,8 +1482,10 @@ pub mod write_conflict { SelfRolledBack = 3, /// RcCheckTs failure by meeting a newer version, let TiDB retry. RcCheckTs = 4, - /// write conflict found in lazy uniqueness check in pessimistic transactions. + /// write conflict found when deferring constraint checks in pessimistic transactions. Deprecated in next-gen (cloud-storage-engine). LazyUniquenessCheck = 5, + /// write conflict found on keys that do not acquire pessimistic locks in pessimistic transactions. + NotLockedKeyConflict = 6, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -1423,6 +1500,7 @@ pub mod write_conflict { Reason::SelfRolledBack => "SelfRolledBack", Reason::RcCheckTs => "RcCheckTs", Reason::LazyUniquenessCheck => "LazyUniquenessCheck", + Reason::NotLockedKeyConflict => "NotLockedKeyConflict", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -1434,6 +1512,7 @@ pub mod write_conflict { "SelfRolledBack" => Some(Self::SelfRolledBack), "RcCheckTs" => Some(Self::RcCheckTs), "LazyUniquenessCheck" => Some(Self::LazyUniquenessCheck), + "NotLockedKeyConflict" => Some(Self::NotLockedKeyConflict), _ => None, } } @@ -1452,10 +1531,14 @@ pub struct Deadlock { pub lock_ts: u64, #[prost(bytes = "vec", tag = "2")] pub lock_key: ::prost::alloc::vec::Vec, + /// The hash of `deadlock_key` field. #[prost(uint64, tag = "3")] pub deadlock_key_hash: u64, #[prost(message, repeated, tag = "4")] pub wait_chain: ::prost::alloc::vec::Vec, + /// The key that the current transaction has already acquired and blocks another transaction to form the deadlock. + #[prost(bytes = "vec", tag = "5")] + pub deadlock_key: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1506,6 +1589,26 @@ pub struct PrimaryMismatch { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxnLockNotFound { + #[prost(bytes = "vec", tag = "1")] + pub key: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MvccDebugInfo { + #[prost(bytes = "vec", tag = "1")] + pub key: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "2")] + pub mvcc: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DebugInfo { + #[prost(message, repeated, tag = "1")] + pub mvcc_info: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct TimeDetail { /// Off-cpu wall time elapsed in TiKV side. Usually this includes queue waiting time and /// other kind of waitings in series. (Wait time in the raftstore is not included.) @@ -1548,6 +1651,12 @@ pub struct TimeDetailV2 { /// Total wall clock time spent on this RPC in TiKV . #[prost(uint64, tag = "5")] pub total_rpc_wall_time_ns: u64, + /// Time spent on the gRPC layer. + #[prost(uint64, tag = "6")] + pub kv_grpc_process_time_ns: u64, + /// Time spent on waiting for run again in grpc pool from other executor pool. + #[prost(uint64, tag = "7")] + pub kv_grpc_wait_time_ns: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1622,6 +1731,29 @@ pub struct ScanDetailV2 { /// Time used for read pool scheduling. #[prost(uint64, tag = "13")] pub read_pool_schedule_wait_nanos: u64, + /// The total MVCC key-value size meet during scanning. It includes + /// deleted versions, but does not include RocksDB tombstone keys. + /// + /// When this field is notably larger than `processed_versions_size`, it means + /// there are a lot of deleted MVCC keys. + #[prost(uint64, tag = "14")] + pub total_versions_size: u64, + /// Total number of IA segment cache hits for this request. + #[prost(uint64, tag = "15")] + pub ia_cache_hit_count: u64, + /// Total number of IA remote segment reads for this request. + /// + /// IA segments are only read on cache misses, so this is also the IA segment + /// cache miss count for the request. + #[prost(uint64, tag = "16")] + pub ia_remote_read_segment_count: u64, + /// Total number of logical bytes returned from IA remote segment reads for this + /// request. + #[prost(uint64, tag = "17")] + pub ia_remote_read_segment_bytes: u64, + /// Total time spent serving IA remote segment reads for this request. + #[prost(uint64, tag = "18")] + pub ia_remote_read_segment_nanos: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1649,6 +1781,51 @@ pub struct ExecDetailsV2 { /// Available when ctx.record_time_stat = true or meet slow query. #[prost(message, optional, tag = "4")] pub time_detail_v2: ::core::option::Option, + /// RU (Request Unit) consumption details. + #[prost(message, optional, tag = "5")] + pub ru_v2: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Ruv2 { + #[prost(uint64, tag = "1")] + pub kv_engine_cache_miss: u64, + #[prost(message, optional, tag = "2")] + pub executor_inputs: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub coprocessor_executor_iterations: u64, + #[prost(uint64, tag = "4")] + pub coprocessor_response_bytes: u64, + #[prost(uint64, tag = "5")] + pub raftstore_store_write_trigger_wb_bytes: u64, + #[prost(uint64, tag = "6")] + pub storage_processed_keys_batch_get: u64, + #[prost(uint64, tag = "7")] + pub storage_processed_keys_get: u64, + /// Filled by client-go. + #[prost(uint64, tag = "8")] + pub read_rpc_count: u64, + /// Filled by client-go. + #[prost(uint64, tag = "9")] + pub write_rpc_count: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExecutorInputs { + #[prost(uint64, tag = "1")] + pub tikv_coprocessor_executor_work_total_batch_index_scan: u64, + #[prost(uint64, tag = "2")] + pub tikv_coprocessor_executor_work_total_batch_table_scan: u64, + #[prost(uint64, tag = "3")] + pub tikv_coprocessor_executor_work_total_batch_selection: u64, + #[prost(uint64, tag = "4")] + pub tikv_coprocessor_executor_work_total_batch_top_n: u64, + #[prost(uint64, tag = "5")] + pub tikv_coprocessor_executor_work_total_batch_limit: u64, + #[prost(uint64, tag = "6")] + pub tikv_coprocessor_executor_work_total_batch_simple_aggr: u64, + #[prost(uint64, tag = "7")] + pub tikv_coprocessor_executor_work_total_batch_fast_hash_aggr: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1714,6 +1891,10 @@ pub struct KvPair { pub key: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "3")] pub value: ::prost::alloc::vec::Vec, + /// The commit timestamp of the key. + /// If it is zero, it means the commit timestamp is unknown. + #[prost(uint64, tag = "4")] + pub commit_ts: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1905,6 +2086,9 @@ pub struct RawCasRequest { pub cf: ::prost::alloc::string::String, #[prost(uint64, tag = "7")] pub ttl: u64, + /// If `delete` is true, the request is to delete the key when the comparison succeeds + #[prost(bool, tag = "8")] + pub delete: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -2097,6 +2281,129 @@ pub struct TiFlashSystemTableResponse { #[prost(bytes = "vec", tag = "1")] pub data: ::prost::alloc::vec::Vec, } +/// Flush is introduced from the pipelined DML protocol. +/// A Flush request writes some keys and values to TiKV, storing in LOCK and DEFAULT CF, just like a Prewrite request. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub mutations: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "3")] + pub primary_key: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "4")] + pub start_ts: u64, + #[prost(uint64, tag = "5")] + pub min_commit_ts: u64, + /// generation of the flush request. It is a monotonically increasing number in each transaction. + #[prost(uint64, tag = "6")] + pub generation: u64, + #[prost(uint64, tag = "7")] + pub lock_ttl: u64, + #[prost(enumeration = "AssertionLevel", tag = "8")] + pub assertion_level: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushResponse { + #[prost(message, optional, tag = "1")] + pub region_error: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub errors: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "3")] + pub exec_details_v2: ::core::option::Option, +} +/// BufferBatchGet is introduced from the pipelined DML protocol. +/// It is similar to a BatchGet request, except that it can only read the data that has been flushed by itself. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BufferBatchGetRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(bytes = "vec", repeated, tag = "2")] + pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(uint64, tag = "3")] + pub version: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BufferBatchGetResponse { + #[prost(message, optional, tag = "1")] + pub region_error: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub error: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub pairs: ::prost::alloc::vec::Vec, + /// Time and scan details when processing the request. + #[prost(message, optional, tag = "4")] + pub exec_details_v2: ::core::option::Option, +} +/// Actively request TiKV to report health feedback information. TiKV won't omit the health feedback information when sending the +/// `BatchCommandsResponse` that contains this response. +/// The health feedback information won't be replied in the response, but will be attached to `BatchCommandsResponse.health_feedback` field as usual. +/// Only works when batch RPC is enabled. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetHealthFeedbackRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetHealthFeedbackResponse { + /// The error field is added for keeping consistent. This request won't meet any region error as it's store level rather than region level. + #[prost(message, optional, tag = "1")] + pub region_error: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub health_feedback: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HealthFeedback { + #[prost(uint64, tag = "1")] + pub store_id: u64, + /// The sequence number of the feedback message. + /// It's defined as an incrementing integer, starting from the unix timestamp (milliseconds) at + /// the time that the TiKV node is started. + /// This can be useful for filtering out out-of-order feedback messages. + /// Note that considering the possibility of system clock changing, this field doesn't guarantee + /// uniqueness and monotonic if the TiKV node is restarted. + #[prost(uint64, tag = "2")] + pub feedback_seq_no: u64, + /// The slow_score calculated in raftstore module. Due to some limitations of slow score, this would + /// be replaced by `SlowTrend` in the future. + #[prost(int32, tag = "3")] + pub slow_score: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BroadcastTxnStatusRequest { + #[prost(message, optional, tag = "1")] + pub context: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub txn_status: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TxnStatus { + #[prost(uint64, tag = "1")] + pub start_ts: u64, + /// a non-zero min_commit_ts indicates the transaction is ongoing + #[prost(uint64, tag = "2")] + pub min_commit_ts: u64, + /// a non-zero commit_ts indicates the transaction is committed + #[prost(uint64, tag = "3")] + pub commit_ts: u64, + #[prost(bool, tag = "4")] + pub rolled_back: bool, + /// The txn has unlocked all keys, implying that it can be removed from txn_status_cache. + #[prost(bool, tag = "5")] + pub is_completed: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BroadcastTxnStatusResponse {} /// Used to specify the behavior when a pessimistic lock request is woken up after waiting for another /// lock. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] @@ -2163,6 +2470,65 @@ impl PessimisticLockKeyResultType { } } } +/// CommitRole indicates the current commit request is a primary commit or a secondary commit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CommitRole { + /// Unknown about the current request is a primary commit or a secondary commit; + Unknown = 0, + /// Current request is a primary commit; + Primary = 1, + /// Current request is a secondary commit; + Secondary = 2, +} +impl CommitRole { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + CommitRole::Unknown => "Unknown", + CommitRole::Primary => "Primary", + CommitRole::Secondary => "Secondary", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Unknown" => Some(Self::Unknown), + "Primary" => Some(Self::Primary), + "Secondary" => Some(Self::Secondary), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RequestOrigin { + Unknown = 0, + TiDb = 1, +} +impl RequestOrigin { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + RequestOrigin::Unknown => "RequestOriginUnknown", + RequestOrigin::TiDb => "RequestOriginTiDB", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "RequestOriginUnknown" => Some(Self::Unknown), + "RequestOriginTiDB" => Some(Self::TiDb), + _ => None, + } + } +} /// The API version the server and the client is using. /// See more details in #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] @@ -2344,8 +2710,13 @@ pub enum Op { Rollback = 3, /// insert operation has a constraint that key should not exist before. Insert = 4, + /// PessimisticLock is exclusive lock acquired in pessimistic transaction. PessimisticLock = 5, CheckNotExists = 6, + /// SharedLock likes Lock but in shared mode. + SharedLock = 7, + /// SharedPessimisticLock is shared lock acquired in pessimistic transaction. + SharedPessimisticLock = 8, } impl Op { /// String value of the enum field names used in the ProtoBuf definition. @@ -2361,6 +2732,8 @@ impl Op { Op::Insert => "Insert", Op::PessimisticLock => "PessimisticLock", Op::CheckNotExists => "CheckNotExists", + Op::SharedLock => "SharedLock", + Op::SharedPessimisticLock => "SharedPessimisticLock", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -2373,6 +2746,8 @@ impl Op { "Insert" => Some(Self::Insert), "PessimisticLock" => Some(Self::PessimisticLock), "CheckNotExists" => Some(Self::CheckNotExists), + "SharedLock" => Some(Self::SharedLock), + "SharedPessimisticLock" => Some(Self::SharedPessimisticLock), _ => None, } } diff --git a/src/generated/logbackup.rs b/src/generated/logbackup.rs index 51b3cf5f..ecc33476 100644 --- a/src/generated/logbackup.rs +++ b/src/generated/logbackup.rs @@ -54,6 +54,25 @@ pub struct FlushEvent { #[prost(uint64, tag = "3")] pub checkpoint: u64, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushNowRequest {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushNowResponse { + #[prost(message, repeated, tag = "1")] + pub results: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FlushResult { + #[prost(string, tag = "1")] + pub task_name: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub success: bool, + #[prost(string, tag = "3")] + pub error_message: ::prost::alloc::string::String, +} /// Generated client implementations. pub mod log_backup_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -195,5 +214,30 @@ pub mod log_backup_client { .insert(GrpcMethod::new("logbackup.LogBackup", "SubscribeFlushEvent")); self.inner.server_streaming(req, path, codec).await } + pub async fn flush_now( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/logbackup.LogBackup/FlushNow", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("logbackup.LogBackup", "FlushNow")); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/meta_storagepb.rs b/src/generated/meta_storagepb.rs index 1355ebd6..a0c67c24 100644 --- a/src/generated/meta_storagepb.rs +++ b/src/generated/meta_storagepb.rs @@ -109,6 +109,37 @@ pub struct PutResponse { /// copied from etcd #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(bytes = "vec", tag = "2")] + pub key: ::prost::alloc::vec::Vec, + /// range_end is the key following the last key to delete for the range \[key, range_end). + /// If range_end is not given, the range is defined to contain only the key argument. + /// If range_end is one bit larger than the given key, then the range is all the keys + /// with the prefix (the given key). + /// If range_end is '\0', the range is all keys greater than or equal to the key argument. + #[prost(bytes = "vec", tag = "3")] + pub range_end: ::prost::alloc::vec::Vec, + /// If prev_kv is set, etcd gets the previous key-value pairs before deleting it. + /// The previous key-value pairs will be returned in the delete response. + #[prost(bool, tag = "4")] + pub prev_kv: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// deleted is the number of keys deleted by the delete range request. + #[prost(int64, tag = "2")] + pub deleted: i64, + #[prost(message, repeated, tag = "3")] + pub prev_kvs: ::prost::alloc::vec::Vec, +} +/// copied from etcd +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct KeyValue { /// key is the key in bytes. An empty key is not allowed. #[prost(bytes = "vec", tag = "1")] @@ -379,5 +410,29 @@ pub mod meta_storage_client { .insert(GrpcMethod::new("meta_storagepb.MetaStorage", "Put")); self.inner.unary(req, path, codec).await } + /// Delete is the same as etcd Range which might be implemented in a more common way + /// so that we can use other storages to replace etcd in the future. + pub async fn delete( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/meta_storagepb.MetaStorage/Delete", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("meta_storagepb.MetaStorage", "Delete")); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/metapb.rs b/src/generated/metapb.rs index 8839fb0f..7ee897eb 100644 --- a/src/generated/metapb.rs +++ b/src/generated/metapb.rs @@ -110,6 +110,16 @@ pub struct Buckets { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct BucketMeta { + /// A hint indicate if keys have changed. + #[prost(uint64, tag = "1")] + pub version: u64, + /// keys of buckets, include start/end key of region + #[prost(bytes = "vec", repeated, tag = "2")] + pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct Region { #[prost(uint64, tag = "1")] pub id: u64, diff --git a/src/generated/mpp.rs b/src/generated/mpp.rs index e68ab94b..2ee84475 100644 --- a/src/generated/mpp.rs +++ b/src/generated/mpp.rs @@ -50,6 +50,10 @@ pub struct TaskMeta { /// This is the session alias between a client and tidb #[prost(string, tag = "19")] pub connection_alias: ::prost::alloc::string::String, + #[prost(string, tag = "20")] + pub sql_digest: ::prost::alloc::string::String, + #[prost(string, tag = "21")] + pub plan_digest: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -80,6 +84,9 @@ pub struct DispatchTaskRequest { /// Used for partition table scan #[prost(message, repeated, tag = "6")] pub table_regions: ::prost::alloc::vec::Vec, + /// Shard infos for TiCI/FTS routing in MPP dispatch path. + #[prost(message, repeated, tag = "7")] + pub table_shard_infos: ::prost::alloc::vec::Vec, } /// Get response of DispatchTaskRequest. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/src/generated/pdpb.rs b/src/generated/pdpb.rs index 548b333c..77187056 100644 --- a/src/generated/pdpb.rs +++ b/src/generated/pdpb.rs @@ -71,8 +71,17 @@ pub struct RequestHeader { #[prost(uint64, tag = "1")] pub cluster_id: u64, /// sender_id is the ID of the sender server, also member ID or etcd ID. + /// sender_id is used in PD internal communication. #[prost(uint64, tag = "2")] pub sender_id: u64, + /// caller_id is the ID of the client which sends the request, such as tikv, + /// tidb, cdc, etc. + #[prost(string, tag = "3")] + pub caller_id: ::prost::alloc::string::String, + /// caller_component is the component of the client which sends the request, + /// such as ddl, optimizer, etc. + #[prost(string, tag = "4")] + pub caller_component: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -162,6 +171,8 @@ pub struct IsBootstrappedResponse { pub struct AllocIdRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, + #[prost(uint32, tag = "2")] + pub count: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -170,6 +181,8 @@ pub struct AllocIdResponse { pub header: ::core::option::Option, #[prost(uint64, tag = "2")] pub id: u64, + #[prost(uint32, tag = "3")] + pub count: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -280,6 +293,57 @@ pub struct GetRegionByIdRequest { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct QueryRegionRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// Whether to include the buckets info within the response. + #[prost(bool, tag = "2")] + pub need_buckets: bool, + /// The region IDs to query. + #[prost(uint64, repeated, tag = "3")] + pub ids: ::prost::alloc::vec::Vec, + /// The region keys to query. + #[prost(bytes = "vec", repeated, tag = "4")] + pub keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// The previous region keys to query. + #[prost(bytes = "vec", repeated, tag = "5")] + pub prev_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct QueryRegionResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// This array functions as a map corresponding to the region IDs, + /// preserving the order of the input region keys, if they are present. + #[prost(uint64, repeated, tag = "2")] + pub key_id_map: ::prost::alloc::vec::Vec, + /// This array functions as a map corresponding to the previous region IDs, + /// preserving the order of the input previous region keys, if they are present. + #[prost(uint64, repeated, tag = "3")] + pub prev_key_id_map: ::prost::alloc::vec::Vec, + /// RegionID -> RegionResponse + #[prost(map = "uint64, message", tag = "4")] + pub regions_by_id: ::std::collections::HashMap, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionResponse { + #[prost(message, optional, tag = "1")] + pub region: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub leader: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub down_peers: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub pending_peers: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "5")] + pub buckets: ::core::option::Option, +} +/// Use GetRegionResponse as the response of GetRegionByIDRequest. +/// Deprecated: use BatchScanRegionsRequest instead. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ScanRegionsRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -306,6 +370,9 @@ pub struct Region { /// working followers. #[prost(message, repeated, tag = "4")] pub pending_peers: ::prost::alloc::vec::Vec, + /// buckets isn't nil only when need_buckets is true. + #[prost(message, optional, tag = "5")] + pub buckets: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -323,6 +390,44 @@ pub struct ScanRegionsResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyRange { + #[prost(bytes = "vec", tag = "1")] + pub start_key: ::prost::alloc::vec::Vec, + /// end_key is +inf when it is empty. + #[prost(bytes = "vec", tag = "2")] + pub end_key: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BatchScanRegionsRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(bool, tag = "2")] + pub need_buckets: bool, + /// the given ranges must be in order. + #[prost(message, repeated, tag = "3")] + pub ranges: ::prost::alloc::vec::Vec, + /// limit the total number of regions to scan. + #[prost(int32, tag = "4")] + pub limit: i32, + /// If contain_all_key_range is true, the output must contain all + /// key ranges in the request. + /// If the output does not contain all key ranges, the request is considered + /// failed and returns an error(REGIONS_NOT_CONTAIN_ALL_KEY_RANGE). + #[prost(bool, tag = "5")] + pub contain_all_key_range: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BatchScanRegionsResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + /// the returned regions are flattened into a list, because the given ranges can located in the same range, we do not return duplicated regions then. + #[prost(message, repeated, tag = "2")] + pub regions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetClusterConfigRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -468,14 +573,27 @@ pub struct RegionHeartbeatRequest { /// QueryStats reported write query stats, and there are read query stats in store heartbeat #[prost(message, optional, tag = "16")] pub query_stats: ::core::option::Option, - /// cpu_usage is the CPU time usage of the leader region since the last heartbeat, + /// cpu_usage is the total CPU time usage of the leader region since the last heartbeat, /// which is calculated by cpu_time_delta/heartbeat_reported_interval. + /// Deprecated: use cpu_stats instead. + #[deprecated] #[prost(uint64, tag = "17")] pub cpu_usage: u64, - /// (Serverless) Approximate size of key-value pairs for billing. + /// cpu_stats reports CPU usage breakdown for the leader region by kind + /// (e.g. unified read). + #[prost(message, optional, tag = "21")] + pub cpu_stats: ::core::option::Option, + /// Approximate size of row-based key-value pairs for billing. /// It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. #[prost(uint64, tag = "18")] pub approximate_kv_size: u64, + /// Approximate size of column-based key-value pairs for billing. + /// It's counted on size of user key & value (excluding metadata fields), before compression, and latest versions only. + #[prost(uint64, tag = "19")] + pub approximate_columnar_kv_size: u64, + /// BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + #[prost(message, optional, tag = "20")] + pub bucket_meta: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -530,6 +648,13 @@ pub struct BatchSwitchWitness { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChangeSplit { + /// auto_split_enabled configures whether the corresponding Region is allowed to be auto split by size or load. + #[prost(bool, tag = "1")] + pub auto_split_enabled: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RegionHeartbeatResponse { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -576,6 +701,8 @@ pub struct RegionHeartbeatResponse { pub change_peer_v2: ::core::option::Option, #[prost(message, optional, tag = "10")] pub switch_witnesses: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub change_split: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -624,6 +751,8 @@ pub struct AskBatchSplitRequest { pub region: ::core::option::Option, #[prost(uint32, tag = "3")] pub split_count: u32, + #[prost(enumeration = "SplitReason", tag = "4")] + pub reason: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -688,6 +817,10 @@ pub struct PeerStat { pub written_keys: u64, #[prost(uint64, tag = "6")] pub written_bytes: u64, + /// cpu_stats is the CPU usage of the region's unified read pool since the last heartbeat, + /// which is calculated by cpu_time_delta/heartbeat_reported_interval. + #[prost(message, optional, tag = "7")] + pub cpu_stats: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -777,6 +910,41 @@ pub struct StoreStats { /// Used memory of the store in bytes. #[prost(uint64, tag = "29")] pub used_memory: u64, + /// Network_slow_scores indicate the network status between TiKV nodes, ranging from 1 to 100 (lower is better). + /// StoreID -> score + #[prost(map = "uint64, uint64", tag = "30")] + pub network_slow_scores: ::std::collections::HashMap, + /// The statistics about DFS uploads. + #[prost(message, repeated, tag = "31")] + pub dfs: ::prost::alloc::vec::Vec, + /// True if the store is undergoing graceful shutdown. + #[prost(bool, tag = "32")] + pub is_stopping: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DfsStatScope { + /// When true, the statistic is not tied to any keyspace. + #[prost(bool, tag = "1")] + pub is_global: bool, + /// The keyspace of this statistic. Ignore when is_global is true. + #[prost(uint32, tag = "2")] + pub keyspace_id: u32, + /// The component that provides the statistic. + #[prost(string, tag = "3")] + pub component: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DfsStatItem { + #[prost(message, optional, tag = "1")] + pub scope: ::core::option::Option, + /// Number of bytes written to DFS. + #[prost(uint64, tag = "2")] + pub written_bytes: u64, + /// Number of write requests sent to DFS. + #[prost(uint64, tag = "3")] + pub write_requests: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -822,6 +990,9 @@ pub struct PeerReport { /// The peer has proposed but uncommitted commit merge. #[prost(bool, tag = "4")] pub has_commit_merge: bool, + /// raft applied index + #[prost(uint64, tag = "5")] + pub applied_index: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -928,6 +1099,9 @@ pub struct StoreHeartbeatResponse { /// Pd can return operations to let TiKV forcely PAUSE | RESUME grpc server. #[prost(message, optional, tag = "7")] pub control_grpc: ::core::option::Option, + /// NodeState is going to mark the state of the store. + #[prost(enumeration = "super::metapb::NodeState", tag = "8")] + pub state: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -962,6 +1136,8 @@ pub struct ScatterRegionResponse { pub header: ::core::option::Option, #[prost(uint64, tag = "2")] pub finished_percentage: u64, + #[prost(uint64, repeated, tag = "3")] + pub failed_regions_id: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1135,6 +1311,206 @@ pub struct GetAllGcSafePointV2Response { #[prost(int64, tag = "3")] pub revision: i64, } +/// A wrapper over keyspace_id. +/// When a field is not specified in proto3, its value will be regarded as 0; however, keyspace_id = 0 is regarded as a valid keyspace (which +/// is the "DEFAULT" keyspace). To distinguish unspecified keyspace (NullKeyspace, 0xffffffff) and the default keyspace in some APIs as well +/// as preventing potential misuse, we wrap the keyspace_id into a message type which is nullable. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyspaceScope { + #[prost(uint32, tag = "1")] + pub keyspace_id: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceGcSafePointRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub target: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceGcSafePointResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub old_gc_safe_point: u64, + #[prost(uint64, tag = "3")] + pub new_gc_safe_point: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceTxnSafePointRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub target: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdvanceTxnSafePointResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(uint64, tag = "2")] + pub old_txn_safe_point: u64, + #[prost(uint64, tag = "3")] + pub new_txn_safe_point: u64, + #[prost(string, tag = "4")] + pub blocker_description: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(string, tag = "3")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub barrier_ts: u64, + #[prost(int64, tag = "5")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GcBarrierInfo { + #[prost(string, tag = "1")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub barrier_ts: u64, + #[prost(int64, tag = "3")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub new_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(string, tag = "3")] + pub barrier_id: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub deleted_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGlobalGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(string, tag = "2")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "3")] + pub barrier_ts: u64, + #[prost(int64, tag = "4")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetGlobalGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub new_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGlobalGcBarrierRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(string, tag = "2")] + pub barrier_id: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteGlobalGcBarrierResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub deleted_barrier_info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GlobalGcBarrierInfo { + #[prost(string, tag = "1")] + pub barrier_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub barrier_ts: u64, + #[prost(int64, tag = "3")] + pub ttl_seconds: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetGcStateRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub keyspace_scope: ::core::option::Option, + #[prost(bool, tag = "3")] + pub exclude_gc_barriers: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GcState { + #[prost(message, optional, tag = "1")] + pub keyspace_scope: ::core::option::Option, + #[prost(bool, tag = "2")] + pub is_keyspace_level_gc: bool, + #[prost(uint64, tag = "3")] + pub txn_safe_point: u64, + #[prost(uint64, tag = "4")] + pub gc_safe_point: u64, + #[prost(message, repeated, tag = "5")] + pub gc_barriers: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetGcStateResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub gc_state: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetAllKeyspacesGcStatesRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(bool, tag = "2")] + pub exclude_gc_barriers: bool, + #[prost(bool, tag = "3")] + pub exclude_global_gc_barriers: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetAllKeyspacesGcStatesResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub gc_states: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub global_gc_barriers: ::prost::alloc::vec::Vec, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegionStat { @@ -1330,6 +1706,17 @@ pub struct QueryStats { #[prost(uint64, tag = "11")] pub rollback: u64, } +/// CPU usage breakdown by kind. New kinds may be added in the future. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CpuStats { + /// UnifiedRead is the CPU usage of the unified read pool. + #[prost(uint64, tag = "1")] + pub unified_read: u64, + /// Scheduler is the CPU usage of the scheduler pool, it contains `sched-pool`, `sched-high`, and `sched-pri`. + #[prost(uint64, tag = "2")] + pub scheduler: u64, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ReportBucketsRequest { @@ -1446,6 +1833,7 @@ pub enum ErrorType { InvalidValue = 10, /// required watch revision is smaller than current compact/min revision. DataCompacted = 11, + RegionsNotContainAllKeyRange = 12, } impl ErrorType { /// String value of the enum field names used in the ProtoBuf definition. @@ -1466,6 +1854,9 @@ impl ErrorType { ErrorType::EntryNotFound => "ENTRY_NOT_FOUND", ErrorType::InvalidValue => "INVALID_VALUE", ErrorType::DataCompacted => "DATA_COMPACTED", + ErrorType::RegionsNotContainAllKeyRange => { + "REGIONS_NOT_CONTAIN_ALL_KEY_RANGE" + } } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -1483,6 +1874,9 @@ impl ErrorType { "ENTRY_NOT_FOUND" => Some(Self::EntryNotFound), "INVALID_VALUE" => Some(Self::InvalidValue), "DATA_COMPACTED" => Some(Self::DataCompacted), + "REGIONS_NOT_CONTAIN_ALL_KEY_RANGE" => { + Some(Self::RegionsNotContainAllKeyRange) + } _ => None, } } @@ -1547,6 +1941,35 @@ impl CheckPolicy { } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] +pub enum SplitReason { + Admin = 0, + Size = 1, + Load = 2, +} +impl SplitReason { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + SplitReason::Admin => "ADMIN", + SplitReason::Size => "SIZE", + SplitReason::Load => "LOAD", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "ADMIN" => Some(Self::Admin), + "SIZE" => Some(Self::Size), + "LOAD" => Some(Self::Load), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] pub enum ControlGrpcEvent { /// Pause TiKV grpc server. Pause = 0, @@ -2088,6 +2511,31 @@ pub mod pd_client { req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "GetRegionByID")); self.inner.unary(req, path, codec).await } + pub async fn query_region( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::QueryRegionRequest, + >, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/QueryRegion"); + let mut req = request.into_streaming_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "QueryRegion")); + self.inner.streaming(req, path, codec).await + } + /// Deprecated: use BatchScanRegions instead. pub async fn scan_regions( &mut self, request: impl tonic::IntoRequest, @@ -2110,6 +2558,28 @@ pub mod pd_client { req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "ScanRegions")); self.inner.unary(req, path, codec).await } + pub async fn batch_scan_regions( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/BatchScanRegions"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "BatchScanRegions")); + self.inner.unary(req, path, codec).await + } pub async fn ask_split( &mut self, request: impl tonic::IntoRequest, @@ -2457,6 +2927,197 @@ pub mod pd_client { .insert(GrpcMethod::new("pdpb.PD", "GetAllGCSafePointV2")); self.inner.unary(req, path, codec).await } + pub async fn advance_gc_safe_point( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/AdvanceGCSafePoint", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "AdvanceGCSafePoint")); + self.inner.unary(req, path, codec).await + } + pub async fn advance_txn_safe_point( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/AdvanceTxnSafePoint", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "AdvanceTxnSafePoint")); + self.inner.unary(req, path, codec).await + } + pub async fn set_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/SetGCBarrier"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "SetGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/DeleteGCBarrier"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "DeleteGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn set_global_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/SetGlobalGCBarrier", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "SetGlobalGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_global_gc_barrier( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/DeleteGlobalGCBarrier", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "DeleteGlobalGCBarrier")); + self.inner.unary(req, path, codec).await + } + pub async fn get_gc_state( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/pdpb.PD/GetGCState"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("pdpb.PD", "GetGCState")); + self.inner.unary(req, path, codec).await + } + pub async fn get_all_keyspaces_gc_states( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/pdpb.PD/GetAllKeyspacesGCStates", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("pdpb.PD", "GetAllKeyspacesGCStates")); + self.inner.unary(req, path, codec).await + } pub async fn sync_regions( &mut self, request: impl tonic::IntoStreamingRequest, diff --git a/src/generated/raft_cmdpb.rs b/src/generated/raft_cmdpb.rs index eb178070..1fc1e66f 100644 --- a/src/generated/raft_cmdpb.rs +++ b/src/generated/raft_cmdpb.rs @@ -312,6 +312,8 @@ pub struct CommitMergeRequest { /// Used in v2. When it's present, `source` and `commit` will not be set. #[prost(message, optional, tag = "4")] pub source_state: ::core::option::Option, + #[prost(bytes = "vec", tag = "100")] + pub source_meta: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -515,6 +517,12 @@ pub struct RaftResponseHeader { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct CustomRequest { + #[prost(bytes = "vec", tag = "1")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RaftCmdRequest { #[prost(message, optional, tag = "1")] pub header: ::core::option::Option, @@ -526,6 +534,8 @@ pub struct RaftCmdRequest { pub admin_request: ::core::option::Option, #[prost(message, optional, tag = "4")] pub status_request: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub custom_request: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/src/generated/raft_serverpb.rs b/src/generated/raft_serverpb.rs index f6b1f942..c215bcc4 100644 --- a/src/generated/raft_serverpb.rs +++ b/src/generated/raft_serverpb.rs @@ -392,6 +392,10 @@ pub struct ExtraMessage { /// notice the peer to refresh buckets version #[prost(message, optional, tag = "9")] pub refresh_buckets: ::core::option::Option, + /// snap_gen_precheck_passed is used to indicate the precheck result when + /// a follower responds to a leader's snapshot gen precheck request. + #[prost(bool, tag = "10")] + pub snap_gen_precheck_passed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -453,6 +457,15 @@ pub enum ExtraMessageType { MsgGcPeerResponse = 12, MsgFlushMemtable = 13, MsgRefreshBuckets = 14, + /// Messages for the snapshot gen precheck process. + MsgSnapGenPrecheckRequest = 15, + MsgSnapGenPrecheckResponse = 16, + /// Used in transfer leader process for leader to inform follower to load the + /// region into in-memory engine if the relevant region is cached. + MsgPreLoadRegionRequest = 17, + /// Used in transfer leader process for follower to inform leader the completes + /// of the region cache + MsgPreLoadRegionResponse = 18, } impl ExtraMessageType { /// String value of the enum field names used in the ProtoBuf definition. @@ -482,6 +495,10 @@ impl ExtraMessageType { ExtraMessageType::MsgGcPeerResponse => "MsgGcPeerResponse", ExtraMessageType::MsgFlushMemtable => "MsgFlushMemtable", ExtraMessageType::MsgRefreshBuckets => "MsgRefreshBuckets", + ExtraMessageType::MsgSnapGenPrecheckRequest => "MsgSnapGenPrecheckRequest", + ExtraMessageType::MsgSnapGenPrecheckResponse => "MsgSnapGenPrecheckResponse", + ExtraMessageType::MsgPreLoadRegionRequest => "MsgPreLoadRegionRequest", + ExtraMessageType::MsgPreLoadRegionResponse => "MsgPreLoadRegionResponse", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -508,6 +525,10 @@ impl ExtraMessageType { "MsgGcPeerResponse" => Some(Self::MsgGcPeerResponse), "MsgFlushMemtable" => Some(Self::MsgFlushMemtable), "MsgRefreshBuckets" => Some(Self::MsgRefreshBuckets), + "MsgSnapGenPrecheckRequest" => Some(Self::MsgSnapGenPrecheckRequest), + "MsgSnapGenPrecheckResponse" => Some(Self::MsgSnapGenPrecheckResponse), + "MsgPreLoadRegionRequest" => Some(Self::MsgPreLoadRegionRequest), + "MsgPreLoadRegionResponse" => Some(Self::MsgPreLoadRegionResponse), _ => None, } } diff --git a/src/generated/resource_manager.rs b/src/generated/resource_manager.rs index caf4b116..e093c6f0 100644 --- a/src/generated/resource_manager.rs +++ b/src/generated/resource_manager.rs @@ -1,7 +1,26 @@ // This file is @generated by prost-build. +/// KeyspaceIDValue is a wrapper for the value of keyspace ID. +/// Because the 0 value is a valid keyspace ID, we need to use a wrapper to distinguish it from the null keyspace ID. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ListResourceGroupsRequest {} +pub struct KeyspaceIdValue { + #[prost(uint32, tag = "1")] + pub value: u32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListResourceGroupsRequest { + #[prost(bool, tag = "1")] + pub with_ru_stats: bool, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, the listed resource groups will be filtered + /// by the given keyspace ID. + #[prost(message, optional, tag = "2")] + pub keyspace_id: ::core::option::Option, +} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ListResourceGroupsResponse { @@ -15,6 +34,16 @@ pub struct ListResourceGroupsResponse { pub struct GetResourceGroupRequest { #[prost(string, tag = "1")] pub resource_group_name: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub with_ru_stats: bool, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will try to get the resource group within + /// the given keyspace ID. + #[prost(message, optional, tag = "3")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -29,6 +58,14 @@ pub struct GetResourceGroupResponse { pub struct DeleteResourceGroupRequest { #[prost(string, tag = "1")] pub resource_group_name: ::prost::alloc::string::String, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will try to delete the resource group within + /// the given keyspace ID. + #[prost(message, optional, tag = "2")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -75,6 +112,14 @@ pub struct TokenBucketRequest { pub is_background: bool, #[prost(bool, tag = "6")] pub is_tiflash: bool, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will try to request the token bucket from + /// the resource group within the given keyspace ID. + #[prost(message, optional, tag = "7")] + pub keyspace_id: ::core::option::Option, #[prost(oneof = "token_bucket_request::Request", tags = "2, 3")] pub request: ::core::option::Option, } @@ -122,6 +167,14 @@ pub struct TokenBucketResponse { /// Raw mode #[prost(message, repeated, tag = "3")] pub granted_resource_tokens: ::prost::alloc::vec::Vec, + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version, + /// which can be safely ignored to keep compatibility. + /// * If the keyspace ID is set to a valid value, it means this response is from the resource + /// group within this keyspace ID. + #[prost(message, optional, tag = "4")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -162,6 +215,19 @@ pub struct Consumption { pub kv_read_rpc_count: f64, #[prost(double, tag = "8")] pub kv_write_rpc_count: f64, + #[prost(uint64, tag = "9")] + pub read_cross_az_traffic_bytes: u64, + #[prost(uint64, tag = "10")] + pub write_cross_az_traffic_bytes: u64, + /// RUv2 is an experimental v2 RU calculation. + /// For now it only records the consumption without actual token deduction. + #[prost(double, tag = "11")] + pub tikv_r_u_v2: f64, + #[prost(double, tag = "12")] + pub tidb_r_u_v2: f64, + /// tiflash_r_u_v2 equals tiflash RRU + tiflash WRU. + #[prost(double, tag = "13")] + pub tiflash_r_u_v2: f64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -199,8 +265,20 @@ pub struct ResourceGroup { /// Runaway queries settings #[prost(message, optional, tag = "6")] pub runaway_settings: ::core::option::Option, + /// Background task control settings. #[prost(message, optional, tag = "7")] pub background_settings: ::core::option::Option, + /// RU consumption statistics. + #[prost(message, optional, tag = "8")] + pub ru_stats: ::core::option::Option, + /// The keyspace ID that the resource group belongs to. + /// There're two cases for this field: + /// + /// * If the keyspace ID is not set, it means this may be a message from an older version. + /// To maintain compatibility, we will treat it as a null keyspace ID, which is uint32.Max. + /// * If the keyspace ID is set to a valid value, it will directly be used. + #[prost(message, optional, tag = "9")] + pub keyspace_id: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -248,6 +326,10 @@ pub struct Error { pub struct RunawayRule { #[prost(uint64, tag = "1")] pub exec_elapsed_time_ms: u64, + #[prost(int64, tag = "2")] + pub processed_keys: i64, + #[prost(int64, tag = "3")] + pub request_unit: i64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -267,12 +349,20 @@ pub struct RunawaySettings { pub action: i32, #[prost(message, optional, tag = "3")] pub watch: ::core::option::Option, + /// When the runaway action is `SwitchGroup`, + /// this field will be used to indicate which group to switch. + #[prost(string, tag = "4")] + pub switch_group_name: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BackgroundSettings { + /// background task types. #[prost(string, repeated, tag = "1")] pub job_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// the percentage limit of total resource(cpu/io) that background tasks can use. + #[prost(uint64, tag = "2")] + pub utilization_limit: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -379,6 +469,8 @@ pub enum RunawayAction { CoolDown = 2, /// kill the task Kill = 3, + /// switch the task to another group + SwitchGroup = 4, } impl RunawayAction { /// String value of the enum field names used in the ProtoBuf definition. @@ -391,6 +483,7 @@ impl RunawayAction { RunawayAction::DryRun => "DryRun", RunawayAction::CoolDown => "CoolDown", RunawayAction::Kill => "Kill", + RunawayAction::SwitchGroup => "SwitchGroup", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -400,6 +493,7 @@ impl RunawayAction { "DryRun" => Some(Self::DryRun), "CoolDown" => Some(Self::CoolDown), "Kill" => Some(Self::Kill), + "SwitchGroup" => Some(Self::SwitchGroup), _ => None, } } diff --git a/src/generated/resource_usage_agent.rs b/src/generated/resource_usage_agent.rs index a8f7f5fd..26f04ac6 100644 --- a/src/generated/resource_usage_agent.rs +++ b/src/generated/resource_usage_agent.rs @@ -8,7 +8,7 @@ pub struct EmptyResponse {} #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ResourceUsageRecord { - #[prost(oneof = "resource_usage_record::RecordOneof", tags = "1")] + #[prost(oneof = "resource_usage_record::RecordOneof", tags = "1, 2")] pub record_oneof: ::core::option::Option, } /// Nested message and enum types in `ResourceUsageRecord`. @@ -18,6 +18,8 @@ pub mod resource_usage_record { pub enum RecordOneof { #[prost(message, tag = "1")] Record(super::GroupTagRecord), + #[prost(message, tag = "2")] + RegionRecord(super::RegionRecord), } } /// GroupTagRecord is a set of resource usage data grouped by resource_group_tag. @@ -40,6 +42,23 @@ pub struct GroupTagRecordItem { pub read_keys: u32, #[prost(uint32, tag = "4")] pub write_keys: u32, + #[prost(uint64, tag = "5")] + pub network_in_bytes: u64, + #[prost(uint64, tag = "6")] + pub network_out_bytes: u64, + #[prost(uint64, tag = "7")] + pub logical_read_bytes: u64, + #[prost(uint64, tag = "8")] + pub logical_write_bytes: u64, +} +/// RegionRecord is a set of resource usage data grouped by region. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionRecord { + #[prost(uint64, tag = "1")] + pub region_id: u64, + #[prost(message, repeated, tag = "2")] + pub items: ::prost::alloc::vec::Vec, } /// Generated client implementations. pub mod resource_usage_agent_client { diff --git a/src/generated/schedulingpb.rs b/src/generated/schedulingpb.rs index 5c6d5de3..e4efe144 100644 --- a/src/generated/schedulingpb.rs +++ b/src/generated/schedulingpb.rs @@ -98,6 +98,9 @@ pub struct RegionHeartbeatRequest { /// Actually reported time interval #[prost(message, optional, tag = "14")] pub interval: ::core::option::Option, + /// BucketMeta is the bucket version and keys of this region if TiKV enabled the bucket feature + #[prost(message, optional, tag = "15")] + pub bucket_meta: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -147,6 +150,8 @@ pub struct RegionHeartbeatResponse { pub change_peer_v2: ::core::option::Option, #[prost(message, optional, tag = "10")] pub switch_witnesses: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub change_split: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -172,6 +177,8 @@ pub struct ScatterRegionsResponse { pub header: ::core::option::Option, #[prost(uint64, tag = "2")] pub finished_percentage: u64, + #[prost(uint64, repeated, tag = "3")] + pub failed_regions_id: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -224,6 +231,8 @@ pub struct AskBatchSplitRequest { pub region: ::core::option::Option, #[prost(uint32, tag = "3")] pub split_count: u32, + #[prost(enumeration = "super::pdpb::SplitReason", tag = "4")] + pub reason: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -233,6 +242,22 @@ pub struct AskBatchSplitResponse { #[prost(message, repeated, tag = "2")] pub ids: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionBucketsRequest { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub region_epoch: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub buckets: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegionBucketsResponse { + #[prost(message, optional, tag = "1")] + pub header: ::core::option::Option, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ErrorType { @@ -508,5 +533,32 @@ pub mod scheduling_client { .insert(GrpcMethod::new("schedulingpb.Scheduling", "AskBatchSplit")); self.inner.unary(req, path, codec).await } + pub async fn region_buckets( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::RegionBucketsRequest, + >, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/schedulingpb.Scheduling/RegionBuckets", + ); + let mut req = request.into_streaming_request(); + req.extensions_mut() + .insert(GrpcMethod::new("schedulingpb.Scheduling", "RegionBuckets")); + self.inner.streaming(req, path, codec).await + } } } diff --git a/src/generated/tikvpb.rs b/src/generated/tikvpb.rs index e27b1cbf..f85e37d4 100644 --- a/src/generated/tikvpb.rs +++ b/src/generated/tikvpb.rs @@ -6,6 +6,10 @@ pub struct BatchCommandsRequest { pub requests: ::prost::alloc::vec::Vec, #[prost(uint64, repeated, tag = "2")] pub request_ids: ::prost::alloc::vec::Vec, + /// Unix epoch timestamp in nanoseconds recorded by the client when this + /// batch request is sent to TiKV. + #[prost(uint64, tag = "3")] + pub client_send_time_ns: u64, } /// Nested message and enum types in `BatchCommandsRequest`. pub mod batch_commands_request { @@ -14,7 +18,7 @@ pub mod batch_commands_request { pub struct Request { #[prost( oneof = "request::Cmd", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 255" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 37, 38, 39, 40, 255" )] pub cmd: ::core::option::Option, } @@ -89,6 +93,14 @@ pub mod batch_commands_request { PrepareFlashbackToVersion( super::super::super::kvrpcpb::PrepareFlashbackToVersionRequest, ), + #[prost(message, tag = "37")] + Flush(super::super::super::kvrpcpb::FlushRequest), + #[prost(message, tag = "38")] + BufferBatchGet(super::super::super::kvrpcpb::BufferBatchGetRequest), + #[prost(message, tag = "39")] + GetHealthFeedback(super::super::super::kvrpcpb::GetHealthFeedbackRequest), + #[prost(message, tag = "40")] + BroadcastTxnStatus(super::super::super::kvrpcpb::BroadcastTxnStatusRequest), /// For some test cases. #[prost(message, tag = "255")] Empty(super::super::BatchCommandsEmptyRequest), @@ -105,6 +117,12 @@ pub struct BatchCommandsResponse { /// 280 means TiKV gRPC cpu usage is 280%. #[prost(uint64, tag = "3")] pub transport_layer_load: u64, + #[prost(message, optional, tag = "4")] + pub health_feedback: ::core::option::Option, + /// Unix epoch timestamp in nanoseconds recorded by TiKV when this batch + /// response is ready to be sent back to the client. + #[prost(uint64, tag = "5")] + pub tikv_send_time_ns: u64, } /// Nested message and enum types in `BatchCommandsResponse`. pub mod batch_commands_response { @@ -113,7 +131,7 @@ pub mod batch_commands_response { pub struct Response { #[prost( oneof = "response::Cmd", - tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 255" + tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 33, 34, 35, 36, 37, 38, 39, 40, 255" )] pub cmd: ::core::option::Option, } @@ -188,6 +206,14 @@ pub mod batch_commands_response { PrepareFlashbackToVersion( super::super::super::kvrpcpb::PrepareFlashbackToVersionResponse, ), + #[prost(message, tag = "37")] + Flush(super::super::super::kvrpcpb::FlushResponse), + #[prost(message, tag = "38")] + BufferBatchGet(super::super::super::kvrpcpb::BufferBatchGetResponse), + #[prost(message, tag = "39")] + GetHealthFeedback(super::super::super::kvrpcpb::GetHealthFeedbackResponse), + #[prost(message, tag = "40")] + BroadcastTxnStatus(super::super::super::kvrpcpb::BroadcastTxnStatusResponse), /// For some test cases. #[prost(message, tag = "255")] Empty(super::super::BatchCommandsEmptyResponse), @@ -199,6 +225,9 @@ pub mod batch_commands_response { pub struct BatchRaftMessage { #[prost(message, repeated, tag = "1")] pub msgs: ::prost::alloc::vec::Vec, + /// Used for measure the send duration. + #[prost(uint64, tag = "13")] + pub last_observed_time: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -762,6 +791,55 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "KvFlashbackToVersion")); self.inner.unary(req, path, codec).await } + pub async fn kv_flush( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/tikvpb.Tikv/KvFlush"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new("tikvpb.Tikv", "KvFlush")); + self.inner.unary(req, path, codec).await + } + pub async fn kv_buffer_batch_get( + &mut self, + request: impl tonic::IntoRequest< + super::super::kvrpcpb::BufferBatchGetRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/KvBufferBatchGet", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "KvBufferBatchGet")); + self.inner.unary(req, path, codec).await + } /// Raw commands; no transaction support. pub async fn raw_get( &mut self, @@ -1256,6 +1334,32 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "BatchCoprocessor")); self.inner.server_streaming(req, path, codec).await } + /// Command send by remote coprocessor to TiKV for executing coprocessor request. + pub async fn delegate_coprocessor( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/DelegateCoprocessor", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "DelegateCoprocessor")); + self.inner.unary(req, path, codec).await + } /// Command for executing custom user requests in TiKV coprocessor_v2. pub async fn raw_coprocessor( &mut self, @@ -1796,6 +1900,34 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "GetTiFlashSystemTable")); self.inner.unary(req, path, codec).await } + /// / Get estimate tici count from TiFlash + pub async fn get_estimate_ti_ci_count( + &mut self, + request: impl tonic::IntoRequest< + super::super::coprocessor::TiCiEstimateCountRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/GetEstimateTiCICount", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "GetEstimateTiCICount")); + self.inner.unary(req, path, codec).await + } /// These are for TiFlash disaggregated architecture /// / Try to lock a S3 object, atomically pub async fn try_add_lock( @@ -1963,5 +2095,179 @@ pub mod tikv_client { .insert(GrpcMethod::new("tikvpb.Tikv", "GetDisaggConfig")); self.inner.unary(req, path, codec).await } + /// / Get health feedback info from the TiKV node. + pub async fn get_health_feedback( + &mut self, + request: impl tonic::IntoRequest< + super::super::kvrpcpb::GetHealthFeedbackRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/GetHealthFeedback", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "GetHealthFeedback")); + self.inner.unary(req, path, codec).await + } + /// / Broadcast the transaction status to all TiKV nodes + pub async fn broadcast_txn_status( + &mut self, + request: impl tonic::IntoRequest< + super::super::kvrpcpb::BroadcastTxnStatusRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.Tikv/BroadcastTxnStatus", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.Tikv", "BroadcastTxnStatus")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated client implementations. +pub mod versioned_kv_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + /// VersionedKv provides versioned coprocessor APIs for TiCI lookup. + /// + /// Invariants: + /// + /// * For `VersionedCoprocessor`, callers should fill `coprocessor.Request.versioned_ranges` + /// (each `VersionedKeyRange.range` must be a point range) and keep `coprocessor.Request.ranges` empty. + #[derive(Debug, Clone)] + pub struct VersionedKvClient { + inner: tonic::client::Grpc, + } + impl VersionedKvClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl VersionedKvClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> VersionedKvClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + VersionedKvClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn versioned_coprocessor( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/tikvpb.VersionedKv/VersionedCoprocessor", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("tikvpb.VersionedKv", "VersionedCoprocessor")); + self.inner.unary(req, path, codec).await + } } } diff --git a/src/generated/tsopb.rs b/src/generated/tsopb.rs index 61a3c6f5..fd84ab11 100644 --- a/src/generated/tsopb.rs +++ b/src/generated/tsopb.rs @@ -14,6 +14,12 @@ pub struct RequestHeader { /// keyspace_group_id is the unique id of the keyspace group to which the tenant/keyspace belongs. #[prost(uint32, tag = "4")] pub keyspace_group_id: u32, + /// callee_id is the ID of the server which the client expects to receive the request. + /// such as tso-0, tso-1, pd-0, pd-1 etc. + /// This field is used to check if the request is sent to the expected server. + /// If it is not matched, the server will return an error. + #[prost(string, tag = "5")] + pub callee_id: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -105,6 +111,8 @@ pub struct FindGroupByKeyspaceIdRequest { pub header: ::core::option::Option, #[prost(uint32, tag = "2")] pub keyspace_id: u32, + #[prost(uint64, tag = "3")] + pub mod_revision: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -113,6 +121,8 @@ pub struct FindGroupByKeyspaceIdResponse { pub header: ::core::option::Option, #[prost(message, optional, tag = "2")] pub keyspace_group: ::core::option::Option, + #[prost(uint64, tag = "3")] + pub mod_revision: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/src/pd/timestamp.rs b/src/pd/timestamp.rs index a1cc7fbd..30cfd8ee 100644 --- a/src/pd/timestamp.rs +++ b/src/pd/timestamp.rs @@ -156,7 +156,7 @@ impl Stream for TsoRequestStream { let req = TsoRequest { header: Some(RequestHeader { cluster_id: *this.cluster_id, - sender_id: 0, + ..Default::default() }), count: requests.len() as u32, dc_location: String::new(),