Merge branch 'main' into in-improve

Author: mergify[bot]

.github/actions/test_stateful_cluster_linux/action.yml

@@ -0,0 +1,58 @@
+name: "Test Stateful Cluster Linux"
+description: "Running stateful tests in cluster mode"
+inputs:
+  profile:
+    description: "The profile for this test"
+    required: true
+    default: "debug"
+  target:
+    description: ""
+    required: true
+    default: "x86_64-unknown-linux-gnu"
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        profile: ${{ inputs.profile }}
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d -p 9900:9000 --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/admin/data  --recursive
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.csv s3://testbucket/admin/data/ontime_200_v1.csv
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data/ontime_200.parquet s3://testbucket/admin/data/ontime_200_v1.parquet
+
+    - name: Run Stateful Tests with Cluster mode (ubuntu-latest only)
+      shell: bash
+      run: |
+        ./scripts/ci/ci-run-stateful-tests-cluster-s3.sh
+
+    - name: Upload .stdout files
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: test-stateful-cluster-linux-stdouts
+        path: |
+           ./tests/suites/1_stateful/*/*.stdout
+
+    - name: Upload failure
+      if: failure()
+      uses: ./.github/actions/artifact_failure
+      with:
+        name: test-stateful-cluster-linux

.github/mergify.yml

@@ -10,11 +10,12 @@ queue_rules:
       - check-success~=^build_(aarch64|x86_64)_musl$
       - check-success=test_unit
       - check-success=test_metactl
-      - check-success=test_stateless_standalone_linux
-      - check-success=test_stateless_cluster_linux
-      - check-success=test_stateful_standalone_linux
+      # state tests
+      - check-success~=^test_(stateless|stateful)_(standalone|cluster)_linux$
+      # sqllogic tests
       - check-success=test_sqllogic_base_standalone_linux
       - check-success=test_sqllogic_ydb_standalone_linux
+      - check-success=test_sqllogic_base_cluster_linux
 
   - name: docs_queue
     conditions:

.github/workflows/dev-linux.yml

@@ -112,22 +112,6 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_meta_cluster
 
-  test_stateless_standalone_linux:
-    timeout-minutes: 10
-    runs-on: [self-hosted, X64, Linux, development]
-    needs: build_gnu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_standalone_linux
-
-  test_stateless_cluster_linux:
-    timeout-minutes: 15
-    runs-on: [self-hosted, X64, Linux, development]
-    needs: build_gnu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_cluster_linux
-
   test_sqllogic_standalone_linux:
     timeout-minutes: 30
     name: test_sqllogic_${{ matrix.dirs }}_standalone_linux
@@ -161,14 +145,40 @@ jobs:
         with:
           dirs: ${{ matrix.dirs }}
 
-  test_stateful_standalone_linux:
-    timeout-minutes: 10
+  test_stateless_standalone_linux:
+    timeout-minutes: 20
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_standalone_linux
+
+  test_stateless_cluster_linux:
+    timeout-minutes: 20
     runs-on: [self-hosted, X64, Linux, development]
     needs: build_gnu
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_cluster_linux
+
+  test_stateful_standalone_linux:
+    timeout-minutes: 20
+    runs-on: [ self-hosted, X64, Linux, development ]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateful_standalone_linux
 
+  # Wait for issue https://github.com/datafuselabs/databend/issues/7684
+  #
+  # test_stateful_cluster_linux:
+  #  timeout-minutes: 20
+  #  runs-on: [ self-hosted, X64, Linux, development ]
+  #  needs: build_gnu
+  #  steps:
+  #    - uses: actions/checkout@v3
+  #    - uses: ./.github/actions/test_stateful_cluster_linux
+
   test_fuzz_standalone_linux:
     timeout-minutes: 10
     runs-on: [self-hosted, X64, Linux, development]

docs/doc/30-reference/20-functions/111-system-functions/fuse_snapshot.md

@@ -19,12 +19,12 @@ INSERT INTO mytable VALUES(1,1),(3,3);
 INSERT INTO mytable VALUES(2,2),(5,5);
 INSERT INTO mytable VALUES(4,4);
 
-SELECT * FROM FUSE_SNAPSHOT(‘default‘,’mytable‘);
+SELECT * FROM FUSE_SNAPSHOT('default','mytable');
 
 ---
 | snapshot_id                      | snapshot_location                                          | format_version | previous_snapshot_id             | segment_count | block_count | row_count | bytes_uncompressed | bytes_compressed | timestamp                  |
 |----------------------------------|------------------------------------------------------------|----------------|----------------------------------|---------------|-------------|-----------|--------------------|------------------|----------------------------|
 | dd98266f968f4817b470255592d04fda | 670655/670675/_ss/dd98266f968f4817b470255592d04fda_v1.json | 1              | \N                               | 1             | 1           | 2         | 16                 | 290              | 2022-09-07 01:58:55.204997 |
 | 2f2d004ff6f842cdb25f5631b2bbb703 | 670655/670675/_ss/2f2d004ff6f842cdb25f5631b2bbb703_v1.json | 1              | dd98266f968f4817b470255592d04fda | 2             | 2           | 4         | 32                 | 580              | 2022-09-07 01:59:09.742999 |
 | 0aa6dfd5d5364bde80f21161ba48c96e | 670655/670675/_ss/0aa6dfd5d5364bde80f21161ba48c96e_v1.json | 1              | 2f2d004ff6f842cdb25f5631b2bbb703 | 3             | 3           | 5         | 40                 | 862              | 2022-09-07 01:59:16.858454 |
-```
+```

docs/doc/30-reference/30-sql/10-dml/dml-copy-into-location.md

@@ -5,14 +5,15 @@ description:
   'Unload Data using COPY INTO <location>'
 ---
 
-`COPY` moves data between Databend and object storage systems (such as Amazon S3).
+`COPY` loads data into Databend or unloads data from Databend.
 
-Unloads data from a table (or query) into one or more files in one of the following locations:
+This command unloads data from a table (or query) into one or more files in one of the following locations:
 
 * Named internal stage: The files can be downloaded from the stage using the GET command.
 * Named external stage: An external location (including Amazon S3).
 * External location: An object storage system (including Amazon S3).
 
+See Also: [COPY INTO table](dml-copy-into-table.md)
 
 ## Syntax
 
@@ -102,5 +103,4 @@ copy into @s2 from test_table FILE_FORMAT = (type = 'CSV');
 
 -- Unload the data from a query into the stage as a Parquet file
 copy into @s2 from (select name, age, id from test_table limit 100) FILE_FORMAT = (type = 'PARQUET');
-```
-
+```

docs/doc/30-reference/30-sql/10-dml/dml-copy-into-table.md

@@ -3,13 +3,15 @@ title: 'COPY INTO <table>'
 sidebar_label: 'COPY INTO <table>'
 ---
 
-`COPY` moves data between Databend tables and object storage systems (AWS S3 compatible object storage services and Azure Blob storage).
+`COPY` loads data into Databend or unloads data from Databend.
 
-This command loads data into a table from files staged in one of the following locations:
+This command loads data into a table from files in one of the following locations:
 
-* Named internal stage, files can be staged using the [PUT to Stage](../../00-api/10-put-to-stage.md).
-* Named external stage that references an external location (AWS S3 compatible object storage services and Azure Blob storage).
-* External location. This includes AWS S3 compatible object storage services, Azure Blob storage, Google Cloud Storage, Huawei OBS.
+* Named internal stage: Databend internal named stages. Files can be staged using the [PUT to Stage](../../00-api/10-put-to-stage.md) API.
+* Named external stage: Stages created in AWS S3 compatible object storage services and Azure Blob storage.
+* External location: This can be a bucket in AWS S3 compatible object storage services, Azure Blob storage, Google Cloud Storage, or Huawei OBS. The exteranl location can be also just a remote server from where you can access the file by a URL (starting with "https://..."). 
+
+See Also: [COPY INTO location](dml-copy-into-location.md)
 
 ## Syntax
 
@@ -123,10 +125,10 @@ externalLocation ::=
   'https://<url>'
 ```
 
-Especially, HTTP supports glob patterns. For example, use
+Please note that, HTTP supports glob patterns. For example, use
 
-- `ontime_200{6,7,8}.csv` to represents `ontime_2006.csv`,`ontime_2007.csv`,`ontime_20080.csv`.
-- `ontime_200[6-8].csv` to represents `ontime_2006.csv`,`ontime_2007.csv`,`ontime_20080.csv`.
+- `ontime_200{6,7,8}.csv` to represents `ontime_2006.csv`,`ontime_2007.csv`,`ontime_2008.csv`.
+- `ontime_200[6-8].csv` to represents `ontime_2006.csv`,`ontime_2007.csv`,`ontime_2008.csv`.
 
 ### FILES = ( 'file_name' [ , 'file_name' ... ] )
 
@@ -261,14 +263,16 @@ COPY INTO mytable
   FILE_FORMAT = (type = 'CSV' field_delimiter = ',' record_delimiter = '\n' skip_header = 1 compression = GZIP) size_limit=10;
 ```
 
-This example moves data from a CSV file without specifying the endpoint URL:
+This example loads data from a CSV file without specifying the endpoint URL:
+
 ```sql
 COPY INTO mytable
   FROM 's3://mybucket/data.csv'
   FILE_FORMAT = (type = 'CSV' field_delimiter = ','  record_delimiter = '\n' skip_header = 1) size_limit=10;
 ```
-  
-`Parquet` file example:
+
+This is an example loading data from a `Parquet` file:
+
 ```sql
 COPY INTO mytable
   FROM 's3://mybucket/data.parquet'
@@ -280,7 +284,8 @@ COPY INTO mytable
 
 **Azure Blob storage**
 
-This example reads data from a CSV file and inserts them into a table:
+This example reads data from a CSV file and inserts it into a table:
+
 ```sql
 COPY INTO mytable
     FROM 'azblob://mybucket/data.csv'
@@ -294,10 +299,10 @@ COPY INTO mytable
 
 **HTTP**
 
-This example reads data from a CSV file and inserts them into a table:
+This example reads data from three CSV files and inserts it into a table:
 
 ```sql
 COPY INTO mytable
     FROM 'https://repo.databend.rs/dataset/stateful/ontime_200{6,7,8}_200.csv'
     FILE_FORMAT = (type = 'CSV');
-```
+```

scripts/ci/ci-run-stateful-tests-cluster-s3.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+echo "*************************************"
+echo "* Setting STORAGE_TYPE to S3.       *"
+echo "*                                   *"
+echo "* Please make sure that S3 backend  *"
+echo "* is ready, and configured properly.*"
+echo "*************************************"
+export STORAGE_TYPE=s3
+export STORAGE_S3_BUCKET=testbucket
+export STORAGE_S3_ROOT=admin
+export STORAGE_S3_ENDPOINT_URL=http://127.0.0.1:9900
+export STORAGE_S3_ACCESS_KEY_ID=minioadmin
+export STORAGE_S3_SECRET_ACCESS_KEY=minioadmin
+export STORAGE_ALLOW_INSECURE=true
+
+echo "Install dependence"
+python3 -m pip install --quiet mysql-connector-python
+
+echo "calling test suite"
+echo "Starting Cluster databend-query"
+./scripts/ci/deploy/databend-query-cluster-3-nodes.sh
+
+SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
+cd "$SCRIPT_PATH/../../tests" || exit
+
+echo "Starting databend-test"
+./databend-test --mode 'cluster' --run-dir 1_stateful

src/meta/api/src/lib.rs

@@ -20,7 +20,6 @@ mod id_generator;
 mod kv_api;
 mod kv_api_key;
 mod kv_api_test_suite;
-mod kv_api_utils;
 mod schema_api;
 mod schema_api_impl;
 mod schema_api_keys;
@@ -29,6 +28,8 @@ mod share_api;
 mod share_api_impl;
 mod share_api_keys;
 mod share_api_test_suite;
+pub(crate) mod testing;
+pub(crate) mod util;
 
 pub use id::Id;
 pub(crate) use id_generator::IdGenerator;
@@ -40,34 +41,32 @@ pub use kv_api::KVApi;
 pub use kv_api_key::KVApiKey;
 pub use kv_api_key::KVApiKeyError;
 pub use kv_api_test_suite::KVApiTestSuite;
-pub use kv_api_utils::db_has_to_exist;
-pub use kv_api_utils::deserialize_struct;
-pub use kv_api_utils::deserialize_u64;
-pub use kv_api_utils::fetch_id;
-pub use kv_api_utils::get_kv_data;
-pub use kv_api_utils::get_object_shared_by_share_ids;
-pub use kv_api_utils::get_share_account_meta_or_err;
-pub use kv_api_utils::get_share_database_id_and_privilege;
-pub use kv_api_utils::get_share_id_to_name_or_err;
-pub use kv_api_utils::get_share_meta_by_id_or_err;
-pub use kv_api_utils::get_share_or_err;
-pub use kv_api_utils::get_struct_value;
-pub use kv_api_utils::get_u64_value;
-pub use kv_api_utils::is_all_db_data_removed;
-pub use kv_api_utils::is_db_need_to_be_remove;
-pub use kv_api_utils::list_keys;
-pub use kv_api_utils::list_u64_value;
-pub use kv_api_utils::send_txn;
-pub use kv_api_utils::serialize_struct;
-pub use kv_api_utils::serialize_u64;
-pub use kv_api_utils::table_has_to_exist;
-pub use kv_api_utils::txn_cond_seq;
-pub use kv_api_utils::txn_op_del;
-pub use kv_api_utils::txn_op_put;
-pub use kv_api_utils::txn_op_put_with_expire;
-pub use kv_api_utils::TXN_MAX_RETRY_TIMES;
 pub use schema_api::SchemaApi;
 pub(crate) use schema_api_impl::get_db_or_err;
 pub use schema_api_test_suite::SchemaApiTestSuite;
 pub use share_api::ShareApi;
 pub use share_api_test_suite::ShareApiTestSuite;
+pub use util::db_has_to_exist;
+pub use util::deserialize_struct;
+pub use util::fetch_id;
+pub use util::get_object_shared_by_share_ids;
+pub use util::get_share_account_meta_or_err;
+pub use util::get_share_database_id_and_privilege;
+pub use util::get_share_id_to_name_or_err;
+pub use util::get_share_meta_by_id_or_err;
+pub use util::get_share_or_err;
+pub use util::get_struct_value;
+pub use util::get_u64_value;
+pub use util::is_all_db_data_removed;
+pub use util::is_db_need_to_be_remove;
+pub use util::list_keys;
+pub use util::list_u64_value;
+pub use util::send_txn;
+pub use util::serialize_struct;
+pub use util::serialize_u64;
+pub use util::table_has_to_exist;
+pub use util::txn_cond_seq;
+pub use util::txn_op_del;
+pub use util::txn_op_put;
+pub use util::txn_op_put_with_expire;
+pub use util::TXN_MAX_RETRY_TIMES;

src/meta/api/src/schema_api_test_suite.rs

@@ -74,9 +74,9 @@ use common_meta_types::UpsertKVReq;
 use tracing::debug;
 use tracing::info;
 
-use crate::get_kv_data;
 use crate::is_all_db_data_removed;
 use crate::serialize_struct;
+use crate::testing::get_kv_data;
 use crate::ApiBuilder;
 use crate::AsKVApi;
 use crate::KVApi;

src/meta/api/src/share_api_test_suite.rs

@@ -25,13 +25,13 @@ use common_meta_types::MetaError;
 use enumflags2::BitFlags;
 use tracing::info;
 
-use crate::get_kv_data;
 use crate::get_object_shared_by_share_ids;
 use crate::get_share_account_meta_or_err;
 use crate::get_share_id_to_name_or_err;
 use crate::get_share_meta_by_id_or_err;
 use crate::get_share_or_err;
 use crate::is_all_db_data_removed;
+use crate::testing::get_kv_data;
 use crate::ApiBuilder;
 use crate::AsKVApi;
 use crate::KVApi;