diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d10442d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +name: Auto-sqlancer CI + +on: + push: + pull_request: + +jobs: + build-images: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Set up Docker + run: docker --version + + - name: Build sqlancer image + run: python3 start.py build --sqlancer + + - name: Build DBMS images + run: python3 start.py build --dbms all + test-dbs: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Set up Docker + run: docker --version + + - name: Test mysql + run: python3 start.py test --dbms mysql --config mysql/config.json + + - name: Test postgres + run: python3 start.py test --dbms postgres --config postgres/config.json + + - name: Test sqlite + run: python3 start.py test --dbms sqlite --config sqlite/config.json + + - name: Test tidb + run: python3 start.py test --dbms tidb --config tidb/config.json + + - name: Test cockroachdb + run: python3 start.py test --dbms cockroachdb --config cockroachdb/config.json + + - name: Test duckdb + run: python3 start.py test --dbms duckdb --config duckdb/config.json diff --git a/build.py b/build.py index 43733e8..0b677cd 100644 --- a/build.py +++ b/build.py @@ -4,7 +4,7 @@ import logging from utils import run_command -def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=False): +def build_sqlancer_image(script_log, docker_log, embedded, dbms='mysql', force_rebuild=False): context_dir = "./sqlancer" if embedded == "yes": dockerfile_path = f"./{dbms}/Dockerfile" @@ -23,7 +23,7 @@ def build_sqlancer_image(script_log, docker_log, embedded, dbms, force_rebuild=F if force_rebuild: script_log.info("Rebuilding SQLancer image: sqlancer:latest ...") - run_command( + run_command( ["docker", "build", "--no-cache", "-f", dockerfile_path, "-t", "sqlancer:latest", context_dir], docker_log ) @@ -68,27 +68,28 @@ def build_network(script_log, docker_log, network_name="sqlancer-net"): sys.exit(1) def build_db_image(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""): - image = f"{cfg['image_name']}:{cfg['tag']}" + if cfg["embedded"] == "no": + image = f"{cfg['image_name']}:{cfg['tag']}" - if not use_cache and not custom: - script_log.info("Pulling db image: %s ...", image) - run_command(["docker", "pull", image], docker_log) - script_log.info("DB image pulled: %s", image) - elif custom: - build_cmd = ["docker", "build", "-t", image, os.path.dirname(dockerfile_path)] - if not use_cache: - build_cmd.insert(2, "--no-cache") - script_log.info("Building db image: %s ...", image) - run_command(build_cmd, docker_log) - script_log.info("DB image built: %s ...", image) - else: - script_log.info("DB image already exists: %s", image) + if not use_cache and not custom: + script_log.info("Pulling db image: %s ...", image) + run_command(["docker", "pull", image], docker_log) + script_log.info("DB image pulled: %s", image) + elif custom: + build_cmd = ["docker", "build", "-t", image, os.path.dirname(dockerfile_path)] + if not use_cache: + build_cmd.insert(2, "--no-cache") + script_log.info("Building db image: %s ...", image) + run_command(build_cmd, docker_log) + script_log.info("DB image built: %s ...", image) + else: + script_log.info("DB image already exists: %s", image) -def build_environment(cfg, use_cache, script_log, docker_log, custom=False, dockerfile_path=""): +def build_environment(cfg, use_cache, script_log, docker_log, dbms='mysql', custom=False, dockerfile_path=""): script_log.info("==============================Building environment==============================") build_network(script_log, docker_log) - build_sqlancer_image(script_log, docker_log, cfg["embedded"], cfg["dbms"], force_rebuild=False) + build_sqlancer_image(script_log, docker_log, cfg["embedded"], dbms, force_rebuild=False) if cfg["embedded"] == "no": build_db_image(cfg, use_cache, script_log, docker_log, custom, dockerfile_path) script_log.info("==============================Building environment==============================") diff --git a/sqlancer/entrypoint.sh b/sqlancer/entrypoint.sh index 185b6d2..4bd9b7e 100644 --- a/sqlancer/entrypoint.sh +++ b/sqlancer/entrypoint.sh @@ -19,19 +19,22 @@ for i in {1..60}; do done -CMD="java -jar sqlancer-*.jar --num-threads \"$SQLANCER_THREADS\" --timeout-seconds \"$SQLANCER_TIMEOUT\" --host \"$SQLANCER_HOST\" \"$SQLANCER_DBMS\" --oracle \"$SQLANCER_ORACLE\"" - +CMD="java -jar sqlancer-*.jar \ + --num-threads \"$SQLANCER_THREADS\" \ + --timeout-seconds \"$SQLANCER_TIMEOUT\"" if [ "$(printf '%s' "${SQLANCER_USERNAME:-}" | tr '[:lower:]' '[:upper:]')" != "N/A" ]; then CMD="$CMD --username \"$SQLANCER_USERNAME\"" fi - if [ "$(printf '%s' "${SQLANCER_PASSWORD:-}" | tr '[:lower:]' '[:upper:]')" != "N/A" ]; then CMD="$CMD --password \"$SQLANCER_PASSWORD\"" fi +CMD="$CMD --host \"$SQLANCER_HOST\" \"$SQLANCER_DBMS\" --oracle \"$SQLANCER_ORACLE\"" + + echo "Running: $CMD" | tee -a "$LOG_FILE" eval "$CMD" 2>&1 | tee -a "$LOG_FILE" | grep --line-buffered "Executed" diff --git a/sqlite/Dockerfile b/sqlite/Dockerfile new file mode 100644 index 0000000..4513a10 --- /dev/null +++ b/sqlite/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y openjdk-17-jdk maven git jq netcat sqlite3 && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /root/sqlancer + +ARG SQLITE_JDBC_VERSION=3.45.1.0 + +RUN git clone https://github.com/sqlancer/sqlancer.git . && \ + mvn -q versions:use-dep-version \ + -Dincludes=org.xerial:sqlite-jdbc \ + -DdepVersion=${SQLITE_JDBC_VERSION} \ + -DforceVersion=true && \ + mvn -q clean package -DskipTests + +COPY entrypoint.sh /root/entrypoint.sh +RUN chmod +x /root/entrypoint.sh + +CMD ["/root/entrypoint.sh"] diff --git a/start.py b/start.py index 4f51b1f..f6c9ef8 100644 --- a/start.py +++ b/start.py @@ -52,14 +52,14 @@ def main(): # print(f"[WARNING] Skipping {dbms}, missing config file.") continue cfg = load_json(config_path) - build_environment(cfg, use_cache, script_log, docker_log) + build_environment(cfg, use_cache, script_log, docker_log, dbms) test_single(cfg, script_log, docker_log, sqlancer_log, run_dir, use_cache) elif args.dbms: if not args.config: parser.error("Single DBMS test requires --config") cfg = load_json(args.config) - build_environment(cfg, use_cache, script_log, docker_log) + build_environment(cfg, use_cache, script_log, docker_log, args.dbms) test_single(cfg, script_log, docker_log, sqlancer_log, run_dir, use_cache) else: parser.error("Must specify either --dbms or --dockerfile") @@ -73,7 +73,7 @@ def main(): if not os.path.exists(config_path): # print(f"[WARNING] Skipping {dbms}, missing config file.") continue - cfg = load_json(config_path) + cfg = load_json(config_path) build_db_image(cfg, use_cache, script_log, docker_log) elif args.dbms: config_path = os.path.join(args.dbms, "config.json")