databendlabs
diff --git a/‎.github/actions/build_bindings_python/action.yml‎
Lines changed: 67 additions & 5 deletions b/‎.github/actions/build_bindings_python/action.yml‎
Lines changed: 67 additions & 5 deletions
diff --git a/‎.github/workflows/bindings.python.yml‎
Lines changed: 26 additions & 0 deletions b/‎.github/workflows/bindings.python.yml‎
Lines changed: 26 additions & 0 deletions
diff --git a/‎Cargo.lock‎
Lines changed: 2 additions & 2 deletions b/‎Cargo.lock‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎src/bendpy/Cargo.toml‎
Lines changed: 2 additions & 2 deletions b/‎src/bendpy/Cargo.toml‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎src/bendpy/README.md‎
Lines changed: 60 additions & 101 deletions b/‎src/bendpy/README.md‎
Lines changed: 60 additions & 101 deletions
@@ -15,8 +15,12 @@ runs:
       if: inputs.version
       shell: bash
       run: |
-        VERSION=`echo ${{ inputs.version }} | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+'`
-        echo "building tag and version: git tag: $GIT_TAG version: $VERSION"
+        raw_version="${{ inputs.version }}"
+        # Remove v prefix: v1.2.3-nightly -> 1.2.3-nightly
+        version_no_v=${raw_version/v/}
+        # Convert to Python-compatible version: 1.2.3-nightly -> 1.2.3.dev0, 1.2.3-p1 -> 1.2.3.post1
+        VERSION=$(echo "$version_no_v" | sed 's/-nightly/.dev0/' | sed 's/-p\([0-9]*\)/.post\1/')
+        echo "building version: $raw_version -> $VERSION"
         sed "s#version = \"0.1.0\"#version = \"$VERSION\"#g" Cargo.toml > Cargo.toml.bak
         sed "s#version = \"0.1.0\"#version = \"$VERSION\"#g" pyproject.toml > pyproject.toml.bak
 
@@ -57,19 +61,77 @@ runs:
         enable-cache: true
 
     - name: Build wheels
-      if: inputs.tag
+      if: inputs.version
       uses: PyO3/maturin-action@v1
       with:
         rust-toolchain: ${{ steps.toolchain.outputs.RUST_TOOLCHAIN }}
         working-directory: src/bendpy
         target: ${{ inputs.target }}
         manylinux: "2_28"
         # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
-        rustup-components: rust-std rustfmt
+        rustup-components: rustfmt
         args: ${{ steps.opts.outputs.BUILD_ARGS }}
-        docker-options: --env RUSTC_WRAPPER=sccache --env SCCACHE_GCS_RW_MODE=READ_WRITE --env SCCACHE_GCS_BUCKET=databend-ci --env SCCACHE_GCS_KEY_PREFIX=cache/sccache/
+        docker-options: --env RUSTC_WRAPPER=sccache --env SCCACHE_GCS_RW_MODE=READ_WRITE --env SCCACHE_GCS_BUCKET=databend-ci --env SCCACHE_GCS_KEY_PREFIX=cache/sccache/ --env MATURIN_NO_AUTO_INSTALL=1
+        container-options: --platform linux/amd64
         before-script-linux: |
           unset RUSTC_WRAPPER
+          # Add the target for the specified architecture
+          rustup target add ${{ inputs.target }}
+          # Install cargo-zigbuild manually to avoid musl target issues
+          cargo install cargo-zigbuild --target ${{ inputs.target }}
           ../../scripts/setup/dev_setup.sh -yb
           uv venv --python=python3.12
           uv sync --all-groups --all-extras
+
+    - name: Setup Rust for development builds
+      if: inputs.version == ''
+      uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ steps.toolchain.outputs.RUST_TOOLCHAIN }}
+        components: rustfmt
+
+    - name: Build development wheel and run tests
+      if: inputs.version == ''
+      shell: bash
+      working-directory: src/bendpy
+      run: |
+        echo "Building development wheel for testing..."
+
+        # Create and activate virtual environment
+        uv venv --python python3.12
+        source .venv/bin/activate
+
+        # Install development dependencies
+        uv pip install maturin pytest pandas polars pyarrow
+
+        # Ensure we have a clean environment
+        export PATH="$HOME/.local/bin:$PATH"
+        export PATH="$HOME/.cargo/bin:$PATH"
+
+        echo "Running development build tests..."
+        maturin develop -E test --quiet
+
+        # Run pytest tests
+        echo "Executing pytest tests..."
+        python -m pytest tests/ -v --tb=short
+
+        echo "All Python binding tests passed!"
+
+    - name: Test built wheels
+      if: inputs.version != ''
+      shell: bash
+      working-directory: src/bendpy
+      run: |
+        echo "Testing built wheels..."
+
+        # Install from built wheel for release testing
+        uv venv --python python3.12
+        source .venv/bin/activate
+        uv pip install dist/*.whl
+        uv pip install pytest pandas polars pyarrow
+
+        # Run pytest tests
+        echo "Executing pytest tests..."
+        python -m pytest tests/ -v --tb=short
+
+        echo "All Python binding tests passed!"
@@ -3,6 +3,12 @@ name: Bindings Python
 on:
   ## uncomment it when bendpy is enabled
   workflow_dispatch:
+    inputs:
+      version:
+        description: Version to release (optional for testing)
+        required: false
+        type: string
+        default: "test-build"
   pull_request:
     branches:
       - main
@@ -27,7 +33,27 @@ permissions:
   packages: write
 
 jobs:
+  test:
+    # Run tests on all PRs and workflow dispatches
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+    runs-on:
+      - self-hosted
+      - X64
+      - Linux
+      - 4c16g
+      - aws
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/build_bindings_python
+        with:
+          target: x86_64-unknown-linux-gnu
+          # No version means development build with tests
+
   linux:
+    # Only run for version builds (releases)
+    if: inputs.version
     runs-on:
       - self-hosted
       - "${{ matrix.runner }}"
 
@@ -18,20 +18,20 @@ crate-type = ["cdylib"]
 arrow = { workspace = true, features = ["pyarrow"] }
 arrow-schema = { workspace = true }
 ctor = { workspace = true }
+databend-common-base = { workspace = true }
 databend-common-catalog = { workspace = true }
 databend-common-config = { workspace = true }
 databend-common-exception = { workspace = true }
 databend-common-expression = { workspace = true }
 databend-common-license = { workspace = true }
 databend-common-meta-app = { workspace = true }
-databend-common-users = { workspace = true }
+databend-common-tracing = { workspace = true }
 databend-common-version = { workspace = true }
 databend-query = { workspace = true, features = [
     "simd",
     "disable_initial_exec_tls",
 ] }
 pyo3 = { version = "0.24", features = ["generate-import-lib", "abi3-py312"] }
-tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync"] }
 tokio-stream = { workspace = true }
 
 
@@ -1,141 +1,100 @@
 # Databend Python Binding
 
-This crate intends to build a native python binding.
+Official Python binding for [Databend](https://databend.com) - The AI-Native Data Warehouse.
+
+Databend is the open-source alternative to Snowflake with near 100% SQL compatibility and native AI capabilities. Built in Rust with MPP architecture and S3-native storage, Databend unifies structured tables, JSON documents, and vector embeddings in a single platform.
 
 ## Installation
 
 ```bash
 pip install databend
 ```
 
-## Usage
+## Quick Start
 
-### Basic:
 ```python
 import databend
-databend.init_service(local_dir = ".databend")
-# or use config
-# databend.init_service( config = "config.toml.sample" )
-
-from databend import SessionContext
-ctx = SessionContext()
 
-df = ctx.sql("select number, number + 1, number::String as number_p_1 from numbers(8)")
+# Create session (automatically initializes embedded mode)
+ctx = databend.SessionContext()
 
+# Execute SQL
+df = ctx.sql("SELECT number, number + 1 FROM numbers(5)")
 df.show()
-# convert to pyarrow
-import pyarrow
-df.to_py_arrow()
 
-# convert to pandas
-import pandas
-df.to_pandas()
+# Convert to pandas/polars
+pandas_df = df.to_pandas()
+polars_df = df.to_polars()
 ```
 
-### Register external table:
+## Examples
 
-***supported functions:***
-- register_parquet
-- register_ndjson
-- register_csv
-- register_tsv
+### 1. Register External Files and Query
 
 ```python
+import databend
 
-ctx.register_parquet("pa", "/home/sundy/dataset/hits_p/", pattern = ".*.parquet")
-ctx.sql("select * from pa limit 10").collect()
-```
-
-### Tenant separation:
+ctx = databend.SessionContext()
 
-Tenant has it's own catalog and tables
+# Register external files
+ctx.register_parquet("pa", "/home/dataset/hits_p/", pattern=".*.parquet")
+ctx.register_csv("users", "/path/to/users.csv")
+ctx.register_ndjson("logs", "/path/to/logs/", pattern=".*.jsonl")
 
-```python
-ctx = SessionContext(tenant = "your_tenant_name")
+# Query external data
+result = ctx.sql("SELECT * FROM pa LIMIT 10").collect()
+print(result)
 ```
 
-## Development
+### 2. Create Table, Insert and Select
 
-Setup virtualenv:
+```python
+import databend
 
-```shell
-uv sync
-```
+ctx = databend.SessionContext()
 
-Activate venv:
+# Create table
+ctx.sql("CREATE TABLE aa (a INT, b STRING, c BOOL, d DOUBLE)").collect()
 
-```shell
-source .venv/bin/activate
-````
+# Insert data
+ctx.sql("INSERT INTO aa SELECT number, number, true, number FROM numbers(10)").collect()
+ctx.sql("INSERT INTO aa SELECT number, number, true, number FROM numbers(10)").collect()
 
-Install `maturin`:
+# Query and convert to pandas
+df = ctx.sql("SELECT sum(a) x, max(b) y, max(d) z FROM aa WHERE c").to_pandas()
+print(df.values.tolist())  # [[90.0, "9", 9.0]]
 
-```shell
-pip install "maturin[patchelf]"
+# Query and convert to polars  
+df_polars = ctx.sql("SELECT sum(a) x, max(b) y, max(d) z FROM aa WHERE c").to_polars()
+print(df_polars.to_pandas().values.tolist())  # [[90.0, "9", 9.0]]
 ```
 
-Build bindings:
+## API Reference
+
+| Method                                           | Description                             | Example                                        |
+|--------------------------------------------------|-----------------------------------------|------------------------------------------------|
+| `SessionContext(tenant=None, data_path=".databend")` | Create session context (auto-initializes) | `ctx = databend.SessionContext()`              |
+| `ctx.sql(sql)`                                   | Execute SQL and return DataFrame        | `df = ctx.sql("SELECT * FROM table")`          |
+| `df.show(num=20)`                                | Display DataFrame results               | `df.show()`                                    |
+| `df.collect()`                                   | Collect DataFrame as DataBlocks         | `blocks = df.collect()`                        |
+| `df.to_pandas()`                                 | Convert to Pandas DataFrame             | `pdf = df.to_pandas()`                         |
+| `df.to_polars()`                                 | Convert to Polars DataFrame             | `pldf = df.to_polars()`                        |
+| `df.to_py_arrow()`                               | Convert to PyArrow batches              | `batches = df.to_py_arrow()`                   |
+| `df.to_arrow_table()`                            | Convert to PyArrow Table                | `table = df.to_arrow_table()`                  |
+| `ctx.register_parquet(name, path, pattern=None)` | Register Parquet files                  | `ctx.register_parquet("data", "/path/")`       |
+| `ctx.register_csv(name, path, pattern=None)`     | Register CSV files                      | `ctx.register_csv("users", "/users.csv")`      |
+| `ctx.register_ndjson(name, path, pattern=None)`  | Register NDJSON files                   | `ctx.register_ndjson("logs", "/logs/")`        |
+| `ctx.register_tsv(name, path, pattern=None)`     | Register TSV files                      | `ctx.register_tsv("data", "/data.tsv")`        |
 
-```shell
-uvx maturin develop
-```
+## Development
 
-Run tests:
+```bash
+# Setup environment
+uv sync
+source .venv/bin/activate
 
-```shell
+# Run tests
 uvx maturin develop -E test
+pytest tests/
 ```
 
-Build API docs:
-
-```shell
-uvx maturin develop -E docs
-uvx pdoc databend
-```
-
-## Service configuration
-
-> Note:
-
-**`databend.init_service`  must be initialized before `SessionContext`**
-
-**`databend.init_service`  must be called only once**
-
-
--  By default, you can init the service by a local directory, then data & catalogs will be stored inside the directory.
-```
-import databend
-
-databend.init_service(local_dir = ".databend")
-```
-
--  You can also init by file
-
-```
-import databend
-databend.init_service( config = "config.toml.sample" )
-```
-
-- And by config str
-```
-import databend
-
-databend.init_service(config = """
-[meta]
-embedded_dir = "./.databend/"
-
-# Storage config.
-[storage]
-# fs | s3 | azblob | obs | oss
-type = "fs"
-allow_insecure = true
-
-[storage.fs]
-data_path = "./.databend/"
-""")
-```
-
-Read more about configs of databend in [docs](https://docs.databend.com/guides/deploy/deploy/production/metasrv-deploy)
-
-## More
-Databend python api is inspired by [arrow-datafusion-python](https://github.com/apache/arrow-datafusion-python), thanks for their great work.