Refine CI (#22)

* Refine CI

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Debug

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* debug

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Update windows path

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Correct switchdaemon

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Nocapture

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Merge before and script

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Try removing the travis cargo build cache

Signed-off-by: Brian Anderson <andersrb@gmail.com>
Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Install protoc

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Add some dependencies

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Try to get windows to detect environment properly

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Sudo is disabled on linux

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Add updated ubuntu package

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Note why windows is disabled

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Correct pkg name

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Add mac dependencies

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Resolve lint

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Allow unused code in select places

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* fmt

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Use raft 0.5.0 branch

Signed-off-by: Ana Hobden <operator@hoverbear.org>

* Remove AppVeyor, don't need it.

Signed-off-by: Ana Hobden <operator@hoverbear.org>

Author: Hoverbear

.travis.yml

@@ -1,44 +1,47 @@
-branches:
-  only:
-    # This is where pull requests from "bors r+" are built.
-    - staging
-    # This is where pull requests from "bors try" are built.
-    - trying
-    # Enable building pull requests.
-    - master
+sudo: false
 
 language: rust
-sudo: false
+os:
+  - linux
+  # - windows # TODO: https://github.com/pingcap/kvproto/issues/355
+  - osx
+rust:
+  - stable
+  - nightly
 env:
   global:
     - RUST_BACKTRACE=1
     - RUSTFLAGS="-D warnings"
-cache: cargo
-
-rust:
-os:
-  - linux
-  - windows
-  - osx
 
-matrix:
-  include:
-  # This build uses stable and checks rustfmt and clippy.
-  # We don't check them on nightly since their lints and formatting may differ.
-  - rust: stable
-    install:
-      - rustup component add rustfmt-preview
-      - rustup component add clippy-preview
-    before_script:
-      - cargo fmt --all -- --check
-      - cargo clippy --all -- -D clippy
-  # Since many users will use nightlies, also test that.
-  - rust: nightly
+addons:
+  apt:
+    update: true
+    sources:
+      - sourceline: 'ppa:maarten-fonville/protobuf'
+    packages:
+      - protobuf-compiler
+  homebrew:
+    packages:
+      - protobuf
+      - cmake
+      - go
 
+install:
+  - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then rustup component add rustfmt; fi
+  - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then rustup component add clippy; fi
+  - if [[ $TRAVIS_OS_NAME == "windows" ]]; then choco install golang cmake strawberryperl protoc; fi
+  - if [[ $TRAVIS_OS_NAME == "windows" ]]; then export PATH="$PATH:/c/Go/bin/:/c/Program Files/CMake/bin"; fi
 
 script:
-  - docker run -d --net=host --name pd --rm pingcap/pd --name "pd" --data-dir "pd" --client-urls "http://127.0.0.1:2379" --advertise-client-urls "http://127.0.0.1:2379"
-  - docker run -d --net=host --name kv --rm pingcap/tikv --pd-endpoints "127.0.0.1:2379" --addr "127.0.0.1:2378" --data-dir "kv"
+  - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then cargo fmt -- --check; fi
+  - if [[ $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_OS_NAME == "linux" ]]; then cargo clippy -- -D clippy::all; fi
   - cargo test --all -- --nocapture
-  # Validate benches still work.
-  - cargo bench --all -- --test
+  # For now we only run full integration tests on Linux. Here's why:
+  # * Docker on OS X is not supported by Travis.
+  # * Docker on Windows seems to not have the correct binary at `"/c/Program Files/Docker/Docker/DockerCli.exe" to switch it to Linux containers.
+  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then docker run -d --net=host --name pd --rm pingcap/pd --name "pd" --data-dir "pd" --client-urls "http://127.0.0.1:2379" --advertise-client-urls "http://127.0.0.1:2379"; fi
+  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then docker run -d --net=host --name kv --rm --ulimit nofile=90000:90000 pingcap/tikv --pd-endpoints "127.0.0.1:2379" --addr "127.0.0.1:2378" --data-dir "kv"; fi
+  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then docker ps; fi
+  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then docker logs pd; fi
+  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then docker logs kv; fi
+  - if [[ $TRAVIS_OS_NAME == "linux" ]]; then PD_ADDR="127.0.0.1:2379" cargo test --all --features integration-tests -- --nocapture; fi

Cargo.toml

@@ -8,6 +8,12 @@ repository = "https://github.com/tikv/client-rust"
 description = "The rust language implementation of TiKV client."
 edition = "2018"
 
+[features]
+default = []
+# Enable integration tests with a running TiKV and PD instance.
+# Use $PD_ADDRS, comma separated, to set the addresses the tests use. 
+integration-tests = []
+
 [lib]
 name = "tikv_client"
 
@@ -26,7 +32,7 @@ failure = "0.1"
 
 [dependencies.kvproto]
 git = "https://github.com/pingcap/kvproto.git"
-rev  = "f9b9e7d362c7cc2c90202fc7c300b2e466cbfbf2"
+branch  = "raft-0.5.0"
 
 [dependencies.prometheus]
 version = "0.4.2"

appveyor.yml

@@ -62,10 +62,7 @@ fn main() -> Result<()> {
 
     // You can also set the `ColumnFamily` used by the request.
     // This is *advanced usage* and should have some special considerations.
-    client
-        .delete(KEY)
-        .wait()
-        .expect("Could not delete value");
+    client.delete(KEY).wait().expect("Could not delete value");
     println!("Key: {:?} deleted", Key::from(KEY));
 
     // Here we check if the key has been deleted from the key-value store.
@@ -82,10 +79,7 @@ fn main() -> Result<()> {
         KvPair::from(("k2", "v2")),
         KvPair::from(("k3", "v3")),
     ];
-    client
-        .batch_put(pairs)
-        .wait()
-        .expect("Could not put pairs");
+    client.batch_put(pairs).wait().expect("Could not put pairs");
 
     // Same thing when you want to retrieve multiple values.
     let keys = vec![Key::from("k1"), Key::from("k2")];

examples/raw.rs

@@ -360,7 +360,6 @@ impl fmt::Debug for Value {
             Ok(s) => write!(f, "Value({:?})", s),
             Err(_) => write!(f, "Value({})", HexRepr(&self.0)),
         }
-
     }
 }
 

src/lib.rs

@@ -11,6 +11,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// TODO: Remove this when txn is done.
+#![allow(dead_code)]
+
 use std::{
     collections::HashMap,
     fmt,

src/rpc/client.rs

@@ -11,6 +11,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// TODO: Remove this when txn is done.
+#![allow(dead_code)]
+
 use std::ops::{Deref, DerefMut};
 
 use kvproto::{kvrpcpb, metapb};

src/rpc/pd/mod.rs

@@ -11,6 +11,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// TODO: Remove this when txn is done.
+#![allow(dead_code)]
+
 use std::{fmt, sync::Arc, time::Duration};
 
 use futures::Future;
@@ -46,7 +49,7 @@ impl From<errorpb::Error> for Error {
         } else if e.has_region_not_found() {
             Error::region_not_found(e.get_region_not_found().get_region_id(), Some(message))
         } else if e.has_key_not_in_region() {
-            let mut e = e.take_key_not_in_region();
+            let e = e.take_key_not_in_region();
             Error::key_not_in_region(e)
         } else if e.has_epoch_not_match() {
             Error::stale_epoch(Some(format!(
@@ -133,7 +136,7 @@ macro_rules! has_str_error {
                 if self.get_error().is_empty() {
                     None
                 } else {
-                    Some(Error::kv_error(self.take_error())                    )
+                    Some(Error::kv_error(self.take_error()))
                 }
             }
         }

src/rpc/tikv/client.rs

@@ -0,0 +1,25 @@
+// Copyright 2019 The TiKV Project Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod raw;
+
+use std::env::var;
+const ENV_PD_ADDR: &str = "PD_ADDR";
+
+pub fn pd_addr() -> Vec<String> {
+    var(ENV_PD_ADDR)
+        .expect(&format!("Expected {}:", ENV_PD_ADDR))
+        .split(",")
+        .map(From::from)
+        .collect()
+}

tests/integration_tests/mod.rs

@@ -1,4 +1,4 @@
-// Copyright 2018 The TiKV Project Authors
+// Copyright 2019 The TiKV Project Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 use futures::future::Future;
 
 const NUM_TEST_KEYS: u32 = 100;
+use crate::integration_tests::pd_addr;
 use tikv_client::{raw::Client, Config, Key, KvPair, Value};
 
 fn generate_key(id: i32) -> Key {
@@ -34,7 +35,7 @@ fn wipe_all(client: &Client) {
 }
 
 fn connect() -> Client {
-    let client = Client::new(&Config::new(vec!["127.0.0.1:2379"]))
+    let client = Client::new(&Config::new(pd_addr()))
         .wait()
         .expect("Could not connect to tikv");
     wipe_all(&client);