Merge pull request #384 from pingyu/support-rust-protobuf

*: support rust-protobuf

Author: pingyu

.github/workflows/ci.yml

@@ -10,69 +10,37 @@ jobs:
   check:
     name: check
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: [ 'RUST_PROTOBUF=0', 'RUST_PROTOBUF=1' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: nightly
           override: true
+      - run: rustup component add rustfmt clippy
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
           version: '3.x'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v1.4.0
-      - uses: actions-rs/cargo@v1
         with:
-          command: check
-          args: --all-targets --all-features
-
-  fmt:
-    name: rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  clippy:
-    name: clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          components: clippy
-          override: true
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v1
-        with:
-          version: '3.x'
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v1.4.0
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-targets --all-features -- -D clippy::all
-          name: Clippy Output
+          key: ${{ matrix.env }}
+      - name: make check
+        run: ${{ matrix.env }} make check
 
   unit-test:
     name: unit test
     env:
       CARGO_INCREMENTAL: 0
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: [ 'RUST_PROTOBUF=0', 'RUST_PROTOBUF=1' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -87,14 +55,19 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v1.4.0
+        with:
+          key: ${{ matrix.env }}
       - name: unit test
-        run: make unit-test
+        run: ${{ matrix.env }} make unit-test
 
   integration-test:
     name: integration test
     env:
       CARGO_INCREMENTAL: 0
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: [ 'RUST_PROTOBUF=0', 'RUST_PROTOBUF=1' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -109,6 +82,8 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Rust Cache
         uses: Swatinem/rust-cache@v1.4.0
+        with:
+          key: ${{ matrix.env }}
       - name: install tiup
         run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
       - name: start tiup playground
@@ -121,4 +96,4 @@ jobs:
             sleep 1
           done
       - name: integration test
-        run: MULTI_REGION=1 make integration-test
+        run: ${{ matrix.env }} MULTI_REGION=1 make integration-test

Cargo.toml

@@ -9,10 +9,26 @@ description = "The Rust language implementation of TiKV client."
 edition = "2018"
 
 [features]
-default = []
+default = [ "prost-codec", "prometheus/process" ]
 # Enable integration tests with a running TiKV and PD instance.
 # Use $PD_ADDRS, comma separated, to set the addresses the tests use.
 integration-tests = []
+protobuf-codec = [
+    "grpcio/protobuf-codec",
+    "tikv-client-proto/protobuf-codec",
+    "tikv-client-common/protobuf-codec",
+    "tikv-client-pd/protobuf-codec",
+    "tikv-client-store/protobuf-codec",
+    "mock-tikv/protobuf-codec",
+]
+prost-codec = [
+    "grpcio/prost-codec",
+    "tikv-client-proto/prost-codec",
+    "tikv-client-common/prost-codec",
+    "tikv-client-pd/prost-codec",
+    "tikv-client-store/prost-codec",
+    "mock-tikv/prost-codec",
+]
 
 [lib]
 name = "tikv_client"
@@ -24,10 +40,10 @@ either = "1.6"
 fail = "0.4"
 futures = { version = "0.3", features = ["async-await", "thread-pool"] }
 futures-timer = "3.0"
-grpcio = { version = "0.10", features = [ "prost-codec", "openssl-vendored" ], default-features = false }
+grpcio = { version = "0.10", features = [ "openssl-vendored" ], default-features = false }
 lazy_static = "1"
 log = "0.4"
-prometheus = { version = "0.12", features = [ "push", "process" ], default-features = false } 
+prometheus = { version = "0.13", features = [ "push" ], default-features = false }
 rand = "0.8"
 regex = "1"
 semver = "1.0"
@@ -70,3 +86,8 @@ members = [
 name = "failpoint_tests"
 path = "tests/failpoint_tests.rs"
 required-features = ["fail/failpoints"]
+
+[patch.crates-io]
+raft-proto = { git = "https://github.com/tikv/raft-rs", rev="95c532612ee6a83591fce9a8b51d6afe87b58835"}
+protobuf-codegen = { git = "https://github.com/pingcap/rust-protobuf", rev="82b49fea7e696fd647b5aca0a6c6ec944eab3189" }
+protobuf = { git = "https://github.com/pingcap/rust-protobuf", rev="82b49fea7e696fd647b5aca0a6c6ec944eab3189" }

Makefile

@@ -2,21 +2,35 @@ export RUSTFLAGS=-Dwarnings
 
 .PHONY: default check unit-test integration-tests test doc docker-pd docker-kv docker all
 
+ENABLE_FEATURES ?=
+PD_ADDRS ?= "127.0.0.1:2379"
+MULTI_REGION ?= 1
+
+# Use Rust-protobuf instead of Prost to encode and decode protocol buffers.
+ifeq ($(RUST_PROTOBUF),1)
+ENABLE_FEATURES += protobuf-codec
+else
+ENABLE_FEATURES += prost-codec
+endif
+
+ALL_FEATURES := ${ENABLE_FEATURES} integration-tests
+
+INTEGRATION_TEST_ARGS := --no-default-features --features "${ENABLE_FEATURES} integration-tests"
+
 default: check
 
 check:
-	cargo check --all --all-targets --all-features
+	cargo check --all --all-targets --no-default-features --features "${ALL_FEATURES}"
 	cargo fmt -- --check
-	cargo clippy --all-targets --all-features -- -D clippy::all
+	cargo clippy --all-targets --no-default-features --features "${ALL_FEATURES}" -- -D clippy::all
 
 unit-test:
-	cargo test --all
+	cargo test --all --no-default-features --features "${ENABLE_FEATURES}"
 
 integration-test:
-# MULTI_REGION shall be set manually if needed
-	PD_ADDRS="127.0.0.1:2379" cargo test txn_ --all --features integration-tests -- --nocapture
-	PD_ADDRS="127.0.0.1:2379" cargo test raw_ --all --features integration-tests -- --nocapture
-	PD_ADDRS="127.0.0.1:2379" cargo test misc_ --all --features integration-tests -- --nocapture
+	cargo test txn_ --all ${INTEGRATION_TEST_ARGS} -- --nocapture
+	cargo test raw_ --all ${INTEGRATION_TEST_ARGS} -- --nocapture
+	cargo test misc_ --all ${INTEGRATION_TEST_ARGS} -- --nocapture
 
 test: unit-test integration-test
 
@@ -36,3 +50,7 @@ tiup:
 	tiup playground nightly --mode tikv-slim --kv 3 --without-monitor --kv.config $(shell pwd)/config/tikv.toml --pd.config $(shell pwd)/config/pd.toml &
 
 all: check doc test
+
+clean:
+	cargo clean
+	rm -rf target

mock-tikv/Cargo.toml

@@ -3,9 +3,13 @@ name = "mock-tikv"
 version = "0.0.0"
 edition = "2018"
 
+[features]
+protobuf-codec = ["grpcio/protobuf-codec"]
+prost-codec = ["grpcio/prost-codec"]
+
 [dependencies]
 derive-new = "0.5"
 futures = "0.3"
-grpcio = { version = "0.10", features = [ "prost-codec" ], default-features = false }
+grpcio = { version = "0.10", default-features = false }
 log = "0.4"
 tikv-client-proto = { path = "../tikv-client-proto"}

mock-tikv/src/lib.rs

@@ -1,4 +1,8 @@
 // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+
+// To support both prost & rust-protobuf.
+#![cfg_attr(feature = "prost-codec", allow(clippy::useless_conversion))]
+
 mod pd;
 mod server;
 mod store;

mock-tikv/src/pd.rs

@@ -20,7 +20,7 @@ impl MockPd {
         tikv_client_proto::metapb::Region {
             start_key: vec![],
             end_key: vec![],
-            peers: vec![Self::leader()],
+            peers: vec![Self::leader()].into(),
             ..Default::default()
         }
     }
@@ -58,12 +58,12 @@ impl Pd for MockPd {
     ) {
         let member = Member {
             name: "mock tikv".to_owned(),
-            client_urls: vec![format!("localhost:{MOCK_PD_PORT}")],
+            client_urls: vec![format!("localhost:{MOCK_PD_PORT}")].into(),
             ..Default::default()
         };
         let resp = GetMembersResponse {
-            members: vec![member.clone()],
-            leader: Some(member),
+            members: vec![member.clone()].into(),
+            leader: Some(member).into(),
             ..Default::default()
         };
         spawn_unary_success!(ctx, req, resp, sink);
@@ -120,7 +120,7 @@ impl Pd for MockPd {
         sink: ::grpcio::UnarySink<GetStoreResponse>,
     ) {
         let resp = GetStoreResponse {
-            store: Some(Self::store()),
+            store: Some(Self::store()).into(),
             ..Default::default()
         };
         spawn_unary_success!(ctx, req, resp, sink);
@@ -169,8 +169,8 @@ impl Pd for MockPd {
         sink: ::grpcio::UnarySink<GetRegionResponse>,
     ) {
         let resp = GetRegionResponse {
-            region: Some(Self::region()),
-            leader: Some(Self::leader()),
+            region: Some(Self::region()).into(),
+            leader: Some(Self::leader()).into(),
             ..Default::default()
         };
         spawn_unary_success!(ctx, req, resp, sink);
@@ -192,8 +192,8 @@ impl Pd for MockPd {
         sink: ::grpcio::UnarySink<GetRegionResponse>,
     ) {
         let resp = GetRegionResponse {
-            region: Some(Self::region()),
-            leader: Some(Self::leader()),
+            region: Some(Self::region()).into(),
+            leader: Some(Self::leader()).into(),
             ..Default::default()
         };
         spawn_unary_success!(ctx, req, resp, sink);

mock-tikv/src/server.rs

@@ -1,5 +1,7 @@
 // Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
 
+#![allow(clippy::useless_conversion)] // To support both prost & rust-protobuf.
+
 use crate::{spawn_unary_success, KvStore};
 use derive_new::new;
 use futures::{FutureExt, TryFutureExt};
@@ -192,7 +194,7 @@ impl Tikv for MockTikv {
         sink: grpcio::UnarySink<tikv_client_proto::kvrpcpb::RawBatchGetResponse>,
     ) {
         let mut resp = tikv_client_proto::kvrpcpb::RawBatchGetResponse::default();
-        resp.set_pairs(self.inner.raw_batch_get(req.take_keys()));
+        resp.set_pairs(self.inner.raw_batch_get(req.take_keys().into()).into());
         spawn_unary_success!(ctx, req, resp, sink);
     }
 
@@ -214,7 +216,7 @@ impl Tikv for MockTikv {
         mut req: tikv_client_proto::kvrpcpb::RawBatchPutRequest,
         sink: grpcio::UnarySink<tikv_client_proto::kvrpcpb::RawBatchPutResponse>,
     ) {
-        let pairs = req.take_pairs();
+        let pairs = req.take_pairs().into();
         self.inner.raw_batch_put(pairs);
         let resp = RawBatchPutResponse::default();
         spawn_unary_success!(ctx, req, resp, sink);
@@ -238,7 +240,7 @@ impl Tikv for MockTikv {
         mut req: tikv_client_proto::kvrpcpb::RawBatchDeleteRequest,
         sink: grpcio::UnarySink<tikv_client_proto::kvrpcpb::RawBatchDeleteResponse>,
     ) {
-        let keys = req.take_keys();
+        let keys = req.take_keys().into();
         self.inner.raw_batch_delete(keys);
         let resp = RawBatchDeleteResponse::default();
         spawn_unary_success!(ctx, req, resp, sink);

src/lib.rs

@@ -90,6 +90,9 @@
 //! # })}
 //! ```
 
+// To support both prost & rust-protobuf.
+#![cfg_attr(feature = "prost-codec", allow(clippy::useless_conversion))]
+
 #[macro_use]
 pub mod request;
 #[macro_use]