update import_sstpb and fix go & rust docker image build (#1224)

Signed-off-by: lance6716 <[email protected]>
Signed-off-by: hillium <[email protected]>
Co-authored-by: hillium <[email protected]>
Co-authored-by: disksing <[email protected]>

Author: 3 people

Dockerfile

@@ -39,7 +39,7 @@ RUN apt-get update --yes \
  && apt-get install --yes curl unzip \
  && rm -rf /var/lib/apt/lists/*
 
-ENV GO_VERSION "1.18.2"
+ENV GO_VERSION "1.21.7"
 RUN rm -rf /usr/local/go \
  && curl -O https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz \
  && tar -C /usr/local -xzf go$GO_VERSION.linux-amd64.tar.gz \

pkg/brpb/brpb.pb.go

@@ -644,6 +644,12 @@ enum FileType {
     Put = 1;
 }
 
+enum FileFormat {
+    Unknown = 0;
+    KvStream = 1;
+    Sst = 2;
+}
+
 // DataFileGroup is the merged file info in log-backup
 message DataFileGroup {
     // Path of the file.
@@ -669,6 +675,7 @@ message DataFileInfo {
     bytes sha256 = 1;
     // Path of the file.
     string path = 2;
+    // The number of KV entries recoreded in the file.
     int64 number_of_entries = 3;
 
     /// Below are extra information of the file, for better filtering files.
@@ -694,7 +701,7 @@ message DataFileInfo {
     // The table ID of the file contains, when `is_meta` is true, would be ignored.
     int64 table_id = 13;
 
-    // The file length.
+    // The length of concateing all kv entries in this file.
     uint64 length = 14;
 
     // The minimal begin ts in default cf if this file is write cf.
@@ -704,11 +711,16 @@ message DataFileInfo {
     uint64 range_offset = 16;
 
     // The range length of the merged file, if it exists.
+    // Diff as `length`, it presents the 'used space' of the file.
     uint64 range_length = 17;
 
     // The compression type for the file.
     CompressionType compression_type = 18;
 
+    // The format of the file's content.
+    FileFormat file_format = 19;
+
+
     // It may support encrypting at future.
     reserved "iv";
 }

pkg/import_sstpb/import_sstpb.pb.go

@@ -128,6 +128,10 @@ message SSTMeta {
     kvrpcpb.APIVersion api_version = 11;
     // cipher_iv is used to encrypt/decrypt sst
     bytes cipher_iv = 12;
+    // in order to be compatible with features like PiTR, import service should use
+    // this optional fields to compose data in write CF. When it's not set, import
+    // service can use WriteBatch.commit_ts instead.
+    uint64 start_ts = 13;
 }
 
 // A rewrite rule is applied on the *encoded* keys (the internal storage

proto/brpb.proto

@@ -20,8 +20,7 @@ check-protos-compatible() {
     export PATH=$GOPATH/bin:$PATH
 
     if [ ! -f "$GOPATH/bin/protolock" ]; then
-        GO111MODULE=off go get github.com/nilslice/protolock/cmd/protolock
-        GO111MODULE=off go install github.com/nilslice/protolock/cmd/protolock
+        GO111MODULE=off go install github.com/nilslice/protolock/cmd/[email protected]
 	fi
 
     if protolock status -lockdir=scripts -protoroot=proto; then
@@ -31,6 +30,8 @@ check-protos-compatible() {
         # In order not to block local branch development, when meet break compatibility will force to update `proto.lock`.
         protolock commit --force -lockdir=scripts -protoroot=proto
     fi
+    # git report error like "fatal: detected dubious ownership in repository at" when reading the host's git folder
+    git config --global --add safe.directory $(pwd)
     # If the output message is encountered, please add proto.lock to git as well.
     git diff scripts/proto.lock | cat
     git diff --quiet scripts/proto.lock

proto/import_sstpb.proto

@@ -244,6 +244,22 @@
                 "integer": 1
               }
             ]
+          },
+          {
+            "name": "FileFormat",
+            "enum_fields": [
+              {
+                "name": "Unknown"
+              },
+              {
+                "name": "KvStream",
+                "integer": 1
+              },
+              {
+                "name": "Sst",
+                "integer": 2
+              }
+            ]
           }
         ],
         "messages": [
@@ -1607,6 +1623,11 @@
                 "id": 18,
                 "name": "compression_type",
                 "type": "CompressionType"
+              },
+              {
+                "id": 19,
+                "name": "file_format",
+                "type": "FileFormat"
               }
             ],
             "reserved_names": [
@@ -6723,6 +6744,11 @@
                 "id": 12,
                 "name": "cipher_iv",
                 "type": "bytes"
+              },
+              {
+                "id": 13,
+                "name": "start_ts",
+                "type": "uint64"
               }
             ]
           },