Merge branch 'main' into feat-insert

Author: mergify[bot]

.github/actions/artifact_upload/action.yml

@@ -39,6 +39,13 @@ runs:
         name: ${{ inputs.profile }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
         path: ./target/${{ inputs.target }}/${{ inputs.profile }}/databend-*
 
+    - name: Upload open-sharing to github
+      if: steps.info.outputs.src == 'github'
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.profile }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
+        path: ./target/${{ inputs.target }}/${{ inputs.profile }}/open-sharing
+
     - name: Upload artifacts to s3
       if: steps.info.outputs.src == 's3'
       shell: bash

Cargo.lock

@@ -160,34 +160,78 @@ formatTypeOptions ::=
   RECORD_DELIMITER = '<character>'
   FIELD_DELIMITER = '<character>'
   SKIP_HEADER = <integer>
+  QUOTE = '<character>'
+  ESCAPE = '<character>'
+  NAN_DISPLAY = '<string>'
+  ROW_TAG = '<string>'
   COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE
 ```
 
-#### `RECORD_DELIMITER = '<character>'`
+#### `TYPE = 'CSV'`
 
-Description: One character that separate records in an input file.
+Comma Separated Values format ([RFC](https://www.rfc-editor.org/rfc/rfc4180)).
 
-Default: `'\n'`
+some notice:
 
-#### `FIELD_DELIMITER = '<character>'`
+1. a string field contains `Quote`|`Escape`|`RECORD_DELIMITER`|`RECORD_DELIMITER` must be quoted.
+2. no character is escaped except `Quote` in quoted string.
+3. no space between `FIELD_DELIMITER` and `Quote`.
+4. no trailing `FIELD_DELIMITER` for a record.
+5. Array/Struct field is serialized to a string as in SQL, and then the resulting string is output to CSV in quotes.
+6. if you are generating CSV via programing, we highly recommend you to use the CSV lib of the programing language.
+7. for text file unloaded from [MySQL](https://dev.mysql.com/doc/refman/8.0/en/load-data.html), the default format is
+   TSV in databend. it is valid CSV only if  `ESCAPED BY` is empty and `ENCLOSED BY` is not empty.
 
-Description: One character that separate fields in an input file.
+##### `RECORD_DELIMITER = '<character>'`
 
-Default: `','` (comma)
+**Description**: One character that separate records in an input file.
+**Supported Values**: `\r\n` or One character including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+**Default**: `\n`
 
-#### `SKIP_HEADER = '<integer>'`
+##### `FIELD_DELIMITER = '<character>'`
 
-Description: Number of lines at the start of the file to skip.
+**Description**: One character that separate fields in an input file.
+**Supported Values**: One character only, including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+**Default**: `,` (comma)
 
-Default: `0`
+##### `Quote = '<character>'`
 
-#### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+**Description**: One character to quote strings in CSV file.
 
-Description: String that represents the compression algorithm.
+for data loading, quote is not necessary unless a string contains `Quote`|`Escape`|`RECORD_DELIMITER`|`RECORD_DELIMITER`
 
-Default: `NONE`
+**Supported Values**: `\'` or `\"`.
 
-Values:
+**Default**: `\"`
+
+##### `ESCAPE = '<character>'`
+
+**Description**: One character to escape quote in quoted strings.
+
+**Supported Values**: `\'` or `\"` or `\\`.
+
+**Default**: `\"`
+
+##### `SKIP_HEADER = '<integer>'`
+
+**Use**: Data loading only.
+
+**Description**: Number of lines at the start of the file to skip.
+
+**Default**: `0`
+
+##### `NAN_DISPLAY = '<string>'`
+
+**Supported Values**: must be literal `'nan'` or `'null'` (case-insensitive)
+**Default**: `'NaN'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+**Description**: String that represents the compression algorithm.
+
+**Default**: `NONE`
+
+**Supported Values**:
 
 | Values        | Notes                                                           |
 | ------------- | --------------------------------------------------------------- |
@@ -201,6 +245,55 @@ Values:
 | `XZ`          |                                                                 |
 | `NONE`        | Indicates that the files have not been compressed.              |
 
+#### `TYPE = 'TSV'`
+
+1. these characters are escaped: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\\`, `\'`, `RECORD_DELIMITER`,`FIELD_DELIMITER`.
+2. quoting/enclosing is not support now.
+3. Array/Struct field is serialized to a string as in SQL, and then the resulting string is output to CSV in quotes.
+4. Null is serialized as `\N`
+
+##### `RECORD_DELIMITER = '<character>'`
+
+**Description**: One character that separate records in an input file.
+
+**Supported Values**: `\r\n` or One character including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+
+**Default**: `\n`
+
+##### `FIELD_DELIMITER = '<character>'`
+
+**Description**: One character that separate fields in an input file.
+
+**Supported Values**: One character only, including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+
+**Default**: `\t` (TAB)
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+#### `TYPE = 'NDJSON'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+#### `TYPE = 'XML'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+##### `ROW_TAG` = `<string>`
+
+**Description**: used to select XML elements to be decoded as a record.
+
+**Default**: `'row'`
+
+#### `TYPE = 'Parquet'`
+
+No options available now.
+
 ### copyOptions
 
 ```

docs/doc/14-sql-commands/10-dml/dml-copy-into-table.md

@@ -56,7 +56,7 @@ impl SingletonType {
                 let guard = LOCAL.wait().read();
                 let v: &T = guard
                     .get(&thread_name)
-                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated"))
+                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated, don't worry if we are in dropping"))
                     .get();
                 v.clone()
             }
@@ -75,7 +75,7 @@ impl SingletonType {
                 let guard = LOCAL.wait().read();
                 guard
                     .get(&thread_name)
-                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated"))
+                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated, don't worry if we are in dropping"))
                     .set(value)
             }
         }

src/common/base/src/base/singleton_instance.rs

@@ -51,14 +51,14 @@ pub mod linux {
         #[inline(always)]
         fn mmap_alloc(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
             debug_assert!(layout.align() <= page_size());
+            ThreadTracker::alloc(layout.size() as i64);
             const PROT: i32 = libc::PROT_READ | libc::PROT_WRITE;
             const FLAGS: i32 = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_POPULATE;
             let addr = unsafe { libc::mmap(null_mut(), layout.size(), PROT, FLAGS, -1, 0) };
             if addr == libc::MAP_FAILED {
                 return Err(AllocError);
             }
             let addr = NonNull::new(addr as *mut ()).ok_or(AllocError)?;
-            ThreadTracker::alloc(layout.size() as i64);
             Ok(NonNull::<[u8]>::from_raw_parts(addr, layout.size()))
         }
 

src/common/base/src/mem_allocator/mmap_allocator.rs

@@ -451,6 +451,7 @@ pub struct StorageMokaConfig {
 }
 
 impl Default for StorageMokaConfig {
+    #[no_sanitize(address)]
     fn default() -> Self {
         Self {
             // Use 1G as default.

src/common/storage/src/config.rs

@@ -28,6 +28,8 @@
 //! - Table snapshots, segments cache must be stored accessed via cache operator.
 //! - Intermediate data generated by query could be stored by temporary operator.
 
+#![feature(no_sanitize)]
+
 mod config;
 pub use config::CacheConfig;
 pub use config::ShareTableConfig;
@@ -55,10 +57,6 @@ pub use operator::init_operator;
 pub use operator::CacheOperator;
 pub use operator::DataOperator;
 
-mod location;
-pub use location::parse_uri_location;
-pub use location::UriLocation;
-
 mod metrics;
 pub use metrics::StorageMetrics;
 pub use metrics::StorageMetricsLayer;

src/common/storage/src/lib.rs

@@ -33,6 +33,7 @@ use common_meta_app::schema::DatabaseIdent;
 use common_meta_app::schema::DatabaseInfo;
 use common_meta_app::schema::DatabaseMeta;
 use common_meta_app::schema::DatabaseNameIdent;
+use common_meta_app::schema::DatabaseType;
 use common_meta_app::schema::DbIdList;
 use common_meta_app::schema::DbIdListKey;
 use common_meta_app::schema::DropDatabaseReply;
@@ -1602,6 +1603,11 @@ impl<KV: KVApi> SchemaApi for KV {
             "get_table"
         );
 
+        let db_type = match db_meta.from_share {
+            Some(share_ident) => DatabaseType::ShareDB(share_ident),
+            None => DatabaseType::NormalDB,
+        };
+
         let tb_info = TableInfo {
             ident: TableIdent {
                 table_id: tbid.table_id,
@@ -1612,7 +1618,7 @@ impl<KV: KVApi> SchemaApi for KV {
             // Safe unwrap() because: tb_meta_seq > 0
             meta: tb_meta.unwrap(),
             tenant: req.tenant.clone(),
-            from_share: db_meta.from_share.clone(),
+            db_type,
         };
 
         return Ok(Arc::new(tb_info));
@@ -1698,6 +1704,11 @@ impl<KV: KVApi> SchemaApi for KV {
                     table_name: table_id_list_key.table_name.clone(),
                 };
 
+                let db_type = match db_meta.from_share.clone() {
+                    Some(share_ident) => DatabaseType::ShareDB(share_ident),
+                    None => DatabaseType::NormalDB,
+                };
+
                 let tb_info = TableInfo {
                     ident: TableIdent {
                         table_id: *table_id,
@@ -1707,7 +1718,7 @@ impl<KV: KVApi> SchemaApi for KV {
                     name: table_id_list_key.table_name.clone(),
                     meta: tb_meta,
                     tenant: tenant_dbname.tenant.clone(),
-                    from_share: db_meta.from_share.clone(),
+                    db_type,
                 };
 
                 tb_info_list.push(Arc::new(tb_info));
@@ -2633,7 +2644,7 @@ async fn get_tableinfos_by_ids(
     ids: &[u64],
     tenant_dbname: &DatabaseNameIdent,
     dbid_tbnames_opt: Option<Vec<DBIdTableName>>,
-    share_name: Option<ShareNameIdent>,
+    db_type: DatabaseType,
 ) -> Result<Vec<Arc<TableInfo>>, KVAppError> {
     let mut tb_meta_keys = Vec::with_capacity(ids.len());
     for id in ids.iter() {
@@ -2671,7 +2682,7 @@ async fn get_tableinfos_by_ids(
                 meta: tb_meta,
                 name: tbnames[i].clone(),
                 tenant: tenant_dbname.tenant.clone(),
-                from_share: share_name.clone(),
+                db_type: db_type.clone(),
             };
             tb_infos.push(Arc::new(tb_info));
         } else {
@@ -2700,7 +2711,14 @@ async fn list_tables_from_unshare_db(
 
     let (dbid_tbnames, ids) = list_u64_value(kv_api, &dbid_tbname).await?;
 
-    get_tableinfos_by_ids(kv_api, &ids, tenant_dbname, Some(dbid_tbnames), None).await
+    get_tableinfos_by_ids(
+        kv_api,
+        &ids,
+        tenant_dbname,
+        Some(dbid_tbnames),
+        DatabaseType::NormalDB,
+    )
+    .await
 }
 
 async fn list_tables_from_share_db(
@@ -2739,5 +2757,12 @@ async fn list_tables_from_share_db(
             ids.push(table_id);
         }
     }
-    get_tableinfos_by_ids(kv_api, &ids, tenant_dbname, None, Some(share)).await
+    get_tableinfos_by_ids(
+        kv_api,
+        &ids,
+        tenant_dbname,
+        None,
+        DatabaseType::ShareDB(share),
+    )
+    .await
 }

src/meta/api/src/schema_api_impl.rs

@@ -30,6 +30,7 @@ use common_meta_app::schema::DatabaseIdToName;
 use common_meta_app::schema::DatabaseInfo;
 use common_meta_app::schema::DatabaseMeta;
 use common_meta_app::schema::DatabaseNameIdent;
+use common_meta_app::schema::DatabaseType;
 use common_meta_app::schema::DbIdList;
 use common_meta_app::schema::DbIdListKey;
 use common_meta_app::schema::DropDatabaseReq;
@@ -3533,7 +3534,7 @@ impl SchemaApiTestSuite {
             assert_eq!(table_info.name, tb1.to_string());
             assert_eq!(table_info.ident.table_id, share_table_id);
             assert_eq!(table_info.tenant, tenant2.to_string());
-            assert_eq!(table_info.from_share, Some(share_name.clone()));
+            assert_eq!(table_info.db_type, DatabaseType::ShareDB(share_name));
         }
 
         info!("--- get tables from share db");

src/meta/api/src/schema_api_test_suite.rs

@@ -47,6 +47,7 @@ pub use table::CountTablesReq;
 pub use table::CreateTableReply;
 pub use table::CreateTableReq;
 pub use table::DBIdTableName;
+pub use table::DatabaseType;
 pub use table::DropTableReply;
 pub use table::DropTableReq;
 pub use table::GetTableCopiedFileReply;