Add ci

.github/workflows/ci.yml

@@ -0,0 +1,69 @@
+name: Auto-sqlancer CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build-images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Set up Docker
+        run: docker --version
+
+      - name: Build sqlancer image
+        run: python3 start.py build --sqlancer
+
+      - name: Build DBMS images
+        run: python3 start.py build --dbms all 
+  test-dbs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Set up Docker
+        run: docker --version
+
+      - name: Test mysql
+        run: python3 start.py test --dbms mysql --config mysql/config.json
+
+      - name: Test postgres
+        run: python3 start.py test --dbms postgres --config postgres/config.json
+
+      - name: Test sqlite
+        run: python3 start.py test --dbms sqlite --config sqlite/config.json
+
+      - name: Test tidb
+        run: python3 start.py test --dbms tidb --config tidb/config.json 
+
+      - name: Test cockroachdb
+        run: python3 start.py test --dbms cockroachdb --config cockroachdb/config.json
+
+      - name: Test duckdb
+        run: python3 start.py test --dbms duckdb --config duckdb/config.json

build.py

@@ -4,7 +4,7 @@
 import logging
 from utils import run_command
 
-def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=False):
+def build_sqlancer_image(script_log, docker_log, embedded, dbms='mysql', force_rebuild=False):
     context_dir = "./sqlancer"
     if embedded == "yes":
         dockerfile_path = f"./{dbms}/Dockerfile"
@@ -23,7 +23,7 @@ def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=F
 
     if force_rebuild:
         script_log.info("Rebuilding SQLancer image: sqlancer:latest ...")
-         run_command(
+        run_command(
             ["docker", "build", "--no-cache", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir],
             docker_log
         )
@@ -68,27 +68,28 @@ def build_network(script_log, docker_log, network_name="sqlancer-net"):
         sys.exit(1)
 
 def build_db_image(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""):
-    image = f"{cfg['image_name']}:{cfg['tag']}"
+    if cfg["embedded"] == "no":
+        image = f"{cfg['image_name']}:{cfg['tag']}"
 
-    if not use_cache and not custom:
-        script_log.info("Pulling db image: %s ...", image)
-        run_command(["docker", "pull", image], docker_log)
-        script_log.info("DB image pulled: %s", image)
-    elif custom:
-        build_cmd = ["docker", "build", "-t", image, os.path.dirname(dockerfile_path)]
-        if not use_cache:
-            build_cmd.insert(2, "--no-cache")
-        script_log.info("Building db image: %s ...", image)
-        run_command(build_cmd, docker_log)
-        script_log.info("DB image built: %s ...", image)
-    else:
-        script_log.info("DB image already exists: %s", image)
+        if not use_cache and not custom:
+            script_log.info("Pulling db image: %s ...", image)
+            run_command(["docker", "pull", image], docker_log)
+            script_log.info("DB image pulled: %s", image)
+        elif custom:
+            build_cmd = ["docker", "build", "-t", image, os.path.dirname(dockerfile_path)]
+            if not use_cache:
+                build_cmd.insert(2, "--no-cache")
+            script_log.info("Building db image: %s ...", image)
+            run_command(build_cmd, docker_log)
+            script_log.info("DB image built: %s ...", image)
+        else:
+            script_log.info("DB image already exists: %s", image)
 
 
-def build_environment(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""):
+def build_environment(cfg, use_cache, script_log, docker_log, dbms='mysql', custom=False, dockerfile_path=""):
     script_log.info("==============================Building environment==============================")
     build_network(script_log, docker_log)
-    build_sqlancer_image(script_log, docker_log, cfg["embedded"], cfg["dbms"], force_rebuild=False)
+    build_sqlancer_image(script_log, docker_log, cfg["embedded"], dbms, force_rebuild=False)
     if cfg["embedded"] == "no":
         build_db_image(cfg, use_cache, script_log, docker_log, custom, dockerfile_path)
     script_log.info("==============================Building environment==============================")

sqlancer/entrypoint.sh

@@ -19,19 +19,22 @@ for i in {1..60}; do
 done
 
 
-CMD="java -jar sqlancer-*.jar --num-threads \"$SQLANCER_THREADS\" --timeout-seconds \"$SQLANCER_TIMEOUT\" --host \"$SQLANCER_HOST\" \"$SQLANCER_DBMS\" --oracle \"$SQLANCER_ORACLE\""
-
+CMD="java -jar sqlancer-*.jar \
+  --num-threads \"$SQLANCER_THREADS\" \
+  --timeout-seconds \"$SQLANCER_TIMEOUT\""
 
 if [ "$(printf '%s' "${SQLANCER_USERNAME:-}" | tr '[:lower:]' '[:upper:]')" != "N/A" ]; then
   CMD="$CMD --username \"$SQLANCER_USERNAME\""
 fi
 
-
 if [ "$(printf '%s' "${SQLANCER_PASSWORD:-}" | tr '[:lower:]' '[:upper:]')" != "N/A" ]; then
   CMD="$CMD --password \"$SQLANCER_PASSWORD\""
 fi
 
 
+CMD="$CMD --host \"$SQLANCER_HOST\" \"$SQLANCER_DBMS\" --oracle \"$SQLANCER_ORACLE\""
+
+
 echo "Running: $CMD" | tee -a "$LOG_FILE"
 
 eval "$CMD" 2>&1 | tee -a "$LOG_FILE" | grep --line-buffered "Executed"

sqlite/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:20.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y openjdk-17-jdk maven git jq netcat sqlite3 && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /root/sqlancer
+
+ARG SQLITE_JDBC_VERSION=3.45.1.0
+
+RUN git clone https://github.com/sqlancer/sqlancer.git . && \
+    mvn -q versions:use-dep-version \
+        -Dincludes=org.xerial:sqlite-jdbc \
+        -DdepVersion=${SQLITE_JDBC_VERSION} \
+        -DforceVersion=true && \
+    mvn -q clean package -DskipTests
+
+COPY entrypoint.sh /root/entrypoint.sh
+RUN chmod +x /root/entrypoint.sh
+
+CMD ["/root/entrypoint.sh"]

start.py

@@ -52,14 +52,14 @@ def main():
                     # print(f"[WARNING] Skipping {dbms}, missing config file.")
                     continue
                 cfg = load_json(config_path)
-                build_environment(cfg, use_cache, script_log, docker_log)
+                build_environment(cfg, use_cache, script_log, docker_log, dbms)
                 test_single(cfg, script_log, docker_log, sqlancer_log, run_dir, use_cache)
 
         elif args.dbms:
             if not args.config:
                 parser.error("Single DBMS test requires --config")
             cfg = load_json(args.config)
-            build_environment(cfg, use_cache, script_log, docker_log)
+            build_environment(cfg, use_cache, script_log, docker_log, args.dbms)
             test_single(cfg, script_log, docker_log, sqlancer_log, run_dir, use_cache)
         else:
             parser.error("Must specify either --dbms or --dockerfile")
@@ -73,7 +73,7 @@ def main():
                 if not os.path.exists(config_path):
                     # print(f"[WARNING] Skipping {dbms}, missing config file.")
                     continue
-                cfg = load_json(config_path)
+                cfg = load_json(config_path)    
                 build_db_image(cfg, use_cache, script_log, docker_log)
         elif args.dbms:
             config_path = os.path.join(args.dbms, "config.json")