From ec56e390635900aeec79cdd6106819a6a4700d3c Mon Sep 17 00:00:00 2001 From: Wojtek Majewski Date: Tue, 4 Nov 2025 11:15:57 +0100 Subject: [PATCH] Optimize CI: separate fast tests from live infrastructure tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split test targets into two categories: - test: Fast tests without infrastructure (vitest, types) - test:live: Tests requiring live infrastructure (pgtap, integration) Enable Nx Cloud caching for verification tasks: - Remove local: true from verify-* targets - Add db:verify meta-target for verification pipeline - Verification results now cached across CI jobs Restructure CI workflow: - Job 1: db-verification (runs once, caches to Nx Cloud) - Job 2: fast-tests (restores cache, runs all packages in parallel) - Job 3-5: *-live-tests (separate jobs per package infrastructure) - Job 6: edge-worker-e2e (unchanged) Benefits: - verify-migrations runs once instead of multiple times - Fast tests complete in ~2-3 min with full parallelization - Simple commands: nx affected -t test (fast), nx affected -t test:live (infra) - Nx handles dependency resolution and caching automatically ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/actions/setup/action.yml | 19 +++ .github/workflows/ci.yml | 175 ++++++++++++++++----- nx.json | 25 ++- pkgs/cli/package.json | 1 + pkgs/cli/project.json | 19 +-- pkgs/cli/scripts/test-async-hang-issue-123 | 20 +-- pkgs/cli/scripts/test-compile | 31 ++++ pkgs/cli/scripts/test-install | 20 +-- pkgs/cli/scripts/test-install-duplicates | 22 +-- pkgs/client/project.json | 33 ++-- pkgs/core/project.json | 67 +++++--- pkgs/edge-worker/project.json | 40 +++-- pnpm-lock.yaml | 3 + scripts/ensure-supabase.sh | 26 +++ 14 files changed, 370 insertions(+), 131 deletions(-) create mode 100755 pkgs/cli/scripts/test-compile create mode 100755 scripts/ensure-supabase.sh diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index cc21ac772..779405ca9 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,6 +1,11 @@ name: 'Setup pgflow workspace' description: 'Common setup steps for pgflow CI workflow (run after checkout)' +inputs: + atlas-cloud-token: + description: 'Atlas Cloud token for schema verification' + required: false + runs: using: 'composite' steps: @@ -16,6 +21,20 @@ runs: cache-dependency-path: | **/pnpm-lock.yaml + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: '1.45.2' + + - name: Install sqruff + uses: quarylabs/install-sqruff-cli-action@main + + - name: Setup Atlas + if: inputs.atlas-cloud-token != '' + uses: ariga/setup-atlas@master + with: + cloud-token: ${{ inputs.atlas-cloud-token }} + - name: Install dependencies shell: bash run: pnpm install --frozen-lockfile --prefer-offline \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e75d0d01..b3aa51f1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,8 @@ permissions: deployments: write # Netlify action needs it jobs: -# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 1. BUILD & TEST โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - build-and-test: +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 1. DB VERIFICATION โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + db-verification: runs-on: ubuntu-latest env: NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} @@ -27,19 +27,33 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup - - - name: Setup Deno - uses: denoland/setup-deno@v2 with: - deno-version: '1.45.2' + atlas-cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} + + - name: Set Nx SHAs for affected commands + uses: nrwl/nx-set-shas@v4 + + - name: Verify NX_BASE and NX_HEAD are set + run: echo "BASE=$NX_BASE HEAD=$NX_HEAD" + + - name: Run database verification + run: pnpm nx run core:db:verify - - name: Install sqruff - uses: quarylabs/install-sqruff-cli-action@main - - name: Setup Atlas - uses: ariga/setup-atlas@master +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 2. FAST TESTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + fast-tests: + needs: [db-verification] + runs-on: ubuntu-latest + env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup with: - cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} + atlas-cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} - name: Set Nx SHAs for affected commands uses: nrwl/nx-set-shas@v4 @@ -47,15 +61,16 @@ jobs: - name: Verify NX_BASE and NX_HEAD are set run: echo "BASE=$NX_BASE HEAD=$NX_HEAD" - - name: Quality gate (lint + typecheck + test) + - name: Lint, typecheck, build, and fast tests run: pnpm nx affected -t lint typecheck test --parallel --configuration=production --base="$NX_BASE" --head="$NX_HEAD" - name: Build all affected projects (except playground) run: pnpm nx affected -t build --configuration=production --parallel --exclude=playground --base="$NX_BASE" --head="$NX_HEAD" -# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 2. EDGE-WORKER E2E โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - edge-worker-e2e: +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 3. CLI LIVE TESTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + cli-live-tests: + needs: [db-verification] runs-on: ubuntu-latest env: NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} @@ -65,45 +80,133 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup + with: + atlas-cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} - - name: Setup Deno - uses: denoland/setup-deno@v2 + - name: Set Nx SHAs for affected commands + uses: nrwl/nx-set-shas@v4 + + - name: Check if cli is affected + id: check-affected + run: | + if pnpm nx show projects --affected --base="$NX_BASE" --head="$NX_HEAD" | grep -q "^cli$"; then + echo "affected=true" >> $GITHUB_OUTPUT + echo "CLI is affected by changes" + else + echo "affected=false" >> $GITHUB_OUTPUT + echo "CLI is not affected by changes - skipping" + fi + + - name: Run cli live tests + if: steps.check-affected.outputs.affected == 'true' + run: pnpm nx run cli:test:live + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 4. CORE LIVE TESTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + core-live-tests: + needs: [db-verification] + runs-on: ubuntu-latest + env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 with: - deno-version: '1.45.2' + fetch-depth: 0 - - name: Install sqruff - uses: quarylabs/install-sqruff-cli-action@main + - uses: ./.github/actions/setup + with: + atlas-cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} - - name: Setup Atlas - uses: ariga/setup-atlas@master + - name: Set Nx SHAs for affected commands + uses: nrwl/nx-set-shas@v4 + + - name: Check if core is affected + id: check-affected + run: | + if pnpm nx show projects --affected --base="$NX_BASE" --head="$NX_HEAD" | grep -q "^core$"; then + echo "affected=true" >> $GITHUB_OUTPUT + echo "Core is affected by changes" + else + echo "affected=false" >> $GITHUB_OUTPUT + echo "Core is not affected by changes - skipping" + fi + + - name: Run core live tests + if: steps.check-affected.outputs.affected == 'true' + run: pnpm nx run core:test:live + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 5. CLIENT LIVE TESTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + client-live-tests: + needs: [db-verification] + runs-on: ubuntu-latest + env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup with: - cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} + atlas-cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} - name: Set Nx SHAs for affected commands uses: nrwl/nx-set-shas@v4 - - name: Verify NX_BASE and NX_HEAD are set - run: echo "BASE=$NX_BASE HEAD=$NX_HEAD" + - name: Check if client is affected + id: check-affected + run: | + if pnpm nx show projects --affected --base="$NX_BASE" --head="$NX_HEAD" | grep -q "^client$"; then + echo "affected=true" >> $GITHUB_OUTPUT + echo "Client is affected by changes" + else + echo "affected=false" >> $GITHUB_OUTPUT + echo "Client is not affected by changes - skipping" + fi + + - name: Run client live tests + if: steps.check-affected.outputs.affected == 'true' + run: pnpm nx run client:test:live + + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 6. EDGE-WORKER LIVE TESTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + edge-worker-live-tests: + needs: [db-verification] + runs-on: ubuntu-latest + env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup + with: + atlas-cloud-token: ${{ secrets.ATLAS_CLOUD_TOKEN }} - - name: Check if edge-worker e2e tests are affected + - name: Set Nx SHAs for affected commands + uses: nrwl/nx-set-shas@v4 + + - name: Check if edge-worker is affected id: check-affected run: | - if pnpm nx show projects --affected -t test:e2e --base="$NX_BASE" --head="$NX_HEAD" | grep -q "^edge-worker$"; then + if pnpm nx show projects --affected --base="$NX_BASE" --head="$NX_HEAD" | grep -q "^edge-worker$"; then echo "affected=true" >> $GITHUB_OUTPUT - echo "Edge-worker e2e tests are affected by changes" + echo "Edge-worker is affected by changes" else echo "affected=false" >> $GITHUB_OUTPUT - echo "Edge-worker e2e tests are not affected by changes - skipping" + echo "Edge-worker is not affected by changes - skipping" fi - - name: Run edge-worker e2e tests + - name: Run edge-worker live tests if: steps.check-affected.outputs.affected == 'true' - run: pnpm nx affected -t test:e2e --parallel --base="$NX_BASE" --head="$NX_HEAD" + run: pnpm nx run edge-worker:test:live -# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 3. DEPLOY PLAYGROUND โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 7. DEPLOY PLAYGROUND โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ deploy-playground: - needs: [build-and-test, edge-worker-e2e] + needs: [fast-tests] if: false # Disabled # if: >- # ${{ @@ -143,9 +246,10 @@ jobs: preview-url: https://pr-${{ github.event.pull_request.number }}--pgflow-demo.netlify.app production-url: https://playground.pgflow.dev -# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 4. DEPLOY WEBSITE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 8. DEPLOY WEBSITE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ deploy-website: - needs: [build-and-test, edge-worker-e2e] + needs: [fast-tests, cli-live-tests, core-live-tests, client-live-tests, edge-worker-live-tests] runs-on: ubuntu-latest environment: ${{ github.event_name == 'pull_request' && 'preview' || 'production' }} env: @@ -177,7 +281,7 @@ jobs: echo "affected=false" >> $GITHUB_OUTPUT echo "Website is not affected by changes - skipping deployment" fi - + - name: Deploy website id: deploy-website if: steps.check-affected.outputs.affected == 'true' @@ -197,4 +301,3 @@ jobs: project-name: Website preview-url: https://pr-${{ github.event.pull_request.number }}.pgflow.pages.dev production-url: https://pgflow.dev - diff --git a/nx.json b/nx.json index 0cb5f2b39..010ee9f4d 100644 --- a/nx.json +++ b/nx.json @@ -80,7 +80,23 @@ "preview": { "dependsOn": ["^build"] }, - "supabase:*": { + "supabase:start": { + "local": true, + "cache": false + }, + "supabase:stop": { + "local": true, + "cache": false + }, + "supabase:status": { + "local": true, + "cache": false + }, + "supabase:reset": { + "local": true, + "cache": false + }, + "supabase:restart": { "local": true, "cache": false }, @@ -94,9 +110,14 @@ "local": true }, "verify-*": { - "local": true, "cache": true }, + "db:verify": { + "cache": true + }, + "test:live": { + "local": true + }, "dump-realtime-schema": { "local": true, "cache": true diff --git a/pkgs/cli/package.json b/pkgs/cli/package.json index a6ebefd87..b3e28fe29 100644 --- a/pkgs/cli/package.json +++ b/pkgs/cli/package.json @@ -16,6 +16,7 @@ "module": "./dist/index.js", "devDependencies": { "@types/node": "^22.14.1", + "supabase": "^2.34.3", "tsx": "^4.19.3" }, "dependencies": { diff --git a/pkgs/cli/project.json b/pkgs/cli/project.json index e4299e71f..01e0e8f93 100644 --- a/pkgs/cli/project.json +++ b/pkgs/cli/project.json @@ -35,16 +35,23 @@ } }, "test": { + "executor": "nx:noop", + "inputs": ["default", "^production"], + "dependsOn": ["test:vitest"] + }, + "test:live": { "executor": "nx:noop", "inputs": ["default", "^production"], "dependsOn": [ - "test:vitest", "test:e2e:install", "test:e2e:compile", "test:e2e:async-hang-issue-123" ], "options": { "parallel": false + }, + "metadata": { + "description": "Tests requiring external tools (Supabase CLI)" } }, "test:vitest": { @@ -75,14 +82,8 @@ "dependsOn": ["test:e2e:install", "build"], "inputs": ["default", "^production"], "options": { - "commands": [ - "rm -rf supabase/", - "npx -y supabase@latest init --with-vscode-settings --with-intellij-settings", - "node dist/index.js compile examples/analyze_website.ts --deno-json examples/deno.json --supabase-path supabase", - "./scripts/assert-flow-compiled" - ], - "cwd": "{projectRoot}", - "parallel": false + "command": "./scripts/test-compile", + "cwd": "{projectRoot}" } }, "test:e2e:async-hang-issue-123": { diff --git a/pkgs/cli/scripts/test-async-hang-issue-123 b/pkgs/cli/scripts/test-async-hang-issue-123 index d6d8c318b..ecc09e8cb 100755 --- a/pkgs/cli/scripts/test-async-hang-issue-123 +++ b/pkgs/cli/scripts/test-async-hang-issue-123 @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -euo pipefail # Script to test if pgflow CLI compile hangs on flows with async functions # This reproduces issue #123: https://github.com/pgflow-dev/pgflow/issues/123 @@ -12,25 +12,27 @@ set -e cd "$(dirname "$0")/.." ROOT_DIR=$(pwd) +# Create unique temp directory for this test +TEST_DIR=$(mktemp -d) +trap "rm -rf $TEST_DIR" EXIT + echo "๐Ÿงช Testing async function compilation" echo " Issue #123: https://github.com/pgflow-dev/pgflow/issues/123" +echo "๐Ÿ“ Test directory: $TEST_DIR" echo "" -# Clean up any existing output -echo "๐Ÿงน Cleaning up old test directory" -rm -rf supabase/ - -# Initialize a fresh Supabase project +# Initialize a fresh Supabase project in temp dir +cd "$TEST_DIR" echo "๐Ÿ—๏ธ Creating new Supabase project" -npx -y supabase@latest init --force --with-vscode-settings --with-intellij-settings +pnpm --dir="$ROOT_DIR" supabase init --workdir "$TEST_DIR" --with-vscode-settings --with-intellij-settings # Install pgflow with our CLI echo "๐Ÿ“ฆ Installing pgflow with CLI" -node dist/index.js install --supabase-path supabase/ --yes +node "$ROOT_DIR/dist/index.js" install --supabase-path supabase/ --yes # Try to compile the flow with async functions echo "๐Ÿ”ง Compiling flow with async functions" -timeout 30s node dist/index.js compile examples/async-function-hang.ts --deno-json examples/deno.json --supabase-path supabase || { +timeout 30s node "$ROOT_DIR/dist/index.js" compile "$ROOT_DIR/examples/async-function-hang.ts" --deno-json "$ROOT_DIR/examples/deno.json" --supabase-path supabase || { EXIT_CODE=$? if [ $EXIT_CODE -eq 124 ]; then echo "โŒ FAILURE: Compilation hung and was killed by timeout (30s)" diff --git a/pkgs/cli/scripts/test-compile b/pkgs/cli/scripts/test-compile new file mode 100755 index 000000000..c05723881 --- /dev/null +++ b/pkgs/cli/scripts/test-compile @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Script to test pgflow CLI compile functionality +# This simulates the test:e2e:compile target from project.json + +cd "$(dirname "$0")/.." +ROOT_DIR=$(pwd) + +# Create unique temp directory for this test +TEST_DIR=$(mktemp -d) +trap "rm -rf $TEST_DIR" EXIT + +echo "๐Ÿงช Testing pgflow compile functionality" +echo "๐Ÿ“ Test directory: $TEST_DIR" + +# Initialize a fresh Supabase project in temp dir +cd "$TEST_DIR" +echo "๐Ÿ—๏ธ Creating new Supabase project" +pnpm --dir="$ROOT_DIR" supabase init --with-vscode-settings --with-intellij-settings + +# Compile the flow +echo "๐Ÿ”ง Compiling analyze_website flow" +node "$ROOT_DIR/dist/index.js" compile "$ROOT_DIR/examples/analyze_website.ts" --deno-json "$ROOT_DIR/examples/deno.json" --supabase-path supabase + +# Verify compilation succeeded +echo "โœ… Verifying flow compilation" +"$ROOT_DIR/scripts/assert-flow-compiled" + +# Show success message +echo "โœจ Compile test complete" diff --git a/pkgs/cli/scripts/test-install b/pkgs/cli/scripts/test-install index 10402f7c0..757a92279 100755 --- a/pkgs/cli/scripts/test-install +++ b/pkgs/cli/scripts/test-install @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -euo pipefail # Script to test pgflow CLI install functionality # This simulates the test:e2e:install target from project.json @@ -7,19 +7,21 @@ set -e cd "$(dirname "$0")/.." ROOT_DIR=$(pwd) -echo "๐Ÿงช Testing pgflow install functionality" +# Create unique temp directory for this test +TEST_DIR=$(mktemp -d) +trap "rm -rf $TEST_DIR" EXIT -# Clean up any existing supabase directory -echo "๐Ÿงน Cleaning up old test directory" -rm -rf supabase/ +echo "๐Ÿงช Testing pgflow install functionality" +echo "๐Ÿ“ Test directory: $TEST_DIR" -# Initialize a fresh Supabase project +# Initialize a fresh Supabase project in temp dir +cd "$TEST_DIR" echo "๐Ÿ—๏ธ Creating new Supabase project" -npx -y supabase@latest init --force --with-vscode-settings --with-intellij-settings +pnpm --dir="$ROOT_DIR" supabase init --workdir "$TEST_DIR" --with-vscode-settings --with-intellij-settings # Install pgflow with our CLI echo "๐Ÿ“ฆ Installing pgflow with CLI" -node dist/index.js install --supabase-path supabase/ --yes +node "$ROOT_DIR/dist/index.js" install --supabase-path supabase/ --yes # Verify installation succeeded echo "โœ… Verifying pgflow installation" @@ -31,5 +33,5 @@ echo "โœจ Installation test complete" # Optional: Test for duplicates by running install again if [ "$1" == "--test-duplicates" ]; then echo "๐Ÿ”„ Testing duplicate installation prevention" - node dist/index.js install --supabase-path supabase/ --yes + node "$ROOT_DIR/dist/index.js" install --supabase-path supabase/ --yes fi diff --git a/pkgs/cli/scripts/test-install-duplicates b/pkgs/cli/scripts/test-install-duplicates index 3895afba8..7c18fa403 100755 --- a/pkgs/cli/scripts/test-install-duplicates +++ b/pkgs/cli/scripts/test-install-duplicates @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -euo pipefail # Script to test pgflow CLI install duplicate prevention # This verifies that we don't create duplicate migrations @@ -7,19 +7,21 @@ set -e cd "$(dirname "$0")/.." ROOT_DIR=$(pwd) -echo "๐Ÿงช Testing pgflow migration duplicate prevention" +# Create unique temp directory for this test +TEST_DIR=$(mktemp -d) +trap "rm -rf $TEST_DIR" EXIT -# Clean up any existing supabase directory -echo "๐Ÿงน Cleaning up old test directory" -rm -rf supabase/ +echo "๐Ÿงช Testing pgflow migration duplicate prevention" +echo "๐Ÿ“ Test directory: $TEST_DIR" -# Initialize a fresh Supabase project +# Initialize a fresh Supabase project in temp dir +cd "$TEST_DIR" echo "๐Ÿ—๏ธ Creating new Supabase project" -npx -y supabase@latest init --force --with-vscode-settings --with-intellij-settings +pnpm --dir="$ROOT_DIR" supabase init --with-vscode-settings --with-intellij-settings # First installation with pgflow CLI echo "๐Ÿ“ฆ First pgflow installation" -node dist/index.js install --supabase-path supabase/ --yes +node "$ROOT_DIR/dist/index.js" install --supabase-path supabase/ --yes # Count number of migrations after first install FIRST_COUNT=$(find supabase/migrations -name "*.sql" | wc -l) @@ -27,7 +29,7 @@ echo "๐Ÿ”ข Found $FIRST_COUNT migrations after first install" # Second installation with pgflow CLI echo "๐Ÿ”„ Running second pgflow installation" -node dist/index.js install --supabase-path supabase/ --yes +node "$ROOT_DIR/dist/index.js" install --supabase-path supabase/ --yes # Count number of migrations after second install SECOND_COUNT=$(find supabase/migrations -name "*.sql" | wc -l) @@ -51,7 +53,7 @@ if [ "$1" == "--test-third-install" ]; then mv "$RANDOM_MIGRATION" "$NEW_NAME" echo "๐Ÿ”„ Running third pgflow installation" - node dist/index.js install --supabase-path supabase/ --yes + node "$ROOT_DIR/dist/index.js" install --supabase-path supabase/ --yes # Count number of migrations after third install THIRD_COUNT=$(find supabase/migrations -name "*.sql" | wc -l) diff --git a/pkgs/client/project.json b/pkgs/client/project.json index 2914dd121..2865b6348 100644 --- a/pkgs/client/project.json +++ b/pkgs/client/project.json @@ -58,18 +58,6 @@ "executor": "@nx/eslint:lint", "inputs": ["default", "^production"] }, - "supabase:ensure-started": { - "executor": "nx:run-commands", - "local": true, - "cache": false, - "options": { - "cwd": "{projectRoot}", - "commands": [ - "supabase status || (echo \"Starting Supabase...\" && supabase start)" - ], - "parallel": false - } - }, "supabase:start": { "executor": "nx:run-commands", "local": true, @@ -137,10 +125,13 @@ "db:ensure": { "executor": "nx:run-commands", "local": true, - "dependsOn": ["supabase:ensure-started", "supabase:prepare"], + "dependsOn": ["supabase:prepare"], "options": { "cwd": "{projectRoot}", - "commands": ["./scripts/ensure-db"], + "commands": [ + "../../scripts/ensure-supabase.sh .", + "./scripts/ensure-db" + ], "parallel": false }, "inputs": [ @@ -154,7 +145,7 @@ "outputs": [ "{projectRoot}/.nx-inputs/db-ready.txt" ], - "cache": true + "cache": false }, "test:integration": { "executor": "nx:run-commands", @@ -173,7 +164,7 @@ "inputs": ["default", "^production"], "options": { "cwd": "{projectRoot}", - "commands": ["vitest run __tests__/ --exclude __tests__/integration/"], + "commands": ["vitest run __tests__ --exclude='**/integration/**'"], "parallel": false } }, @@ -191,7 +182,15 @@ "test": { "executor": "nx:noop", "inputs": ["default", "^production"], - "dependsOn": ["test:vitest", "test:types"] + "dependsOn": ["test:unit", "test:types"] + }, + "test:live": { + "executor": "nx:noop", + "inputs": ["default", "^production"], + "dependsOn": ["test:integration"], + "metadata": { + "description": "Tests requiring live infrastructure (Supabase)" + } }, "benchmark": { "executor": "nx:run-commands", diff --git a/pkgs/core/project.json b/pkgs/core/project.json index 94fc16d10..d9d274c76 100644 --- a/pkgs/core/project.json +++ b/pkgs/core/project.json @@ -22,18 +22,20 @@ "dump-realtime-schema": { "executor": "nx:run-commands", "local": true, - "dependsOn": ["supabase:ensure-started"], "outputs": ["{projectRoot}/atlas/.supabase-baseline-schema.sql"], "options": { "cwd": "{projectRoot}", - "commands": ["scripts/atlas-dump-realtime-schema"], + "commands": [ + "../../scripts/ensure-supabase.sh .", + "scripts/atlas-dump-realtime-schema" + ], "parallel": false }, "cache": true }, "verify-schemas-synced": { "executor": "nx:run-commands", - "dependsOn": ["dump-realtime-schema", "supabase:ensure-started"], + "dependsOn": ["dump-realtime-schema"], "inputs": [ "schemas", "migrations", @@ -43,6 +45,7 @@ "options": { "cwd": "{projectRoot}", "commands": [ + "../../scripts/ensure-supabase.sh .", "mkdir -p .nx-inputs", "scripts/atlas-verify-schemas-synced > .nx-inputs/verify-schemas-synced.txt 2>&1 || (cat .nx-inputs/verify-schemas-synced.txt && exit 1)" ], @@ -52,12 +55,13 @@ }, "verify-migrations": { "executor": "nx:run-commands", - "dependsOn": ["verify-schemas-synced", "supabase:ensure-started"], + "dependsOn": ["verify-schemas-synced"], "inputs": ["migrations"], "outputs": ["{projectRoot}/.nx-inputs/verify-migrations.txt"], "options": { "cwd": "{projectRoot}", "commands": [ + "../../scripts/ensure-supabase.sh .", "mkdir -p .nx-inputs", "supabase db reset > .nx-inputs/verify-migrations.txt 2>&1 || (cat .nx-inputs/verify-migrations.txt && exit 1)" ], @@ -65,6 +69,14 @@ }, "cache": true }, + "db:verify": { + "executor": "nx:noop", + "dependsOn": ["verify-schemas-synced", "verify-migrations", "verify-gen-types"], + "cache": true, + "metadata": { + "description": "Run all database verification tasks" + } + }, "build": { "executor": "@nx/js:tsc", "inputs": ["production", "databaseTypes", "^production"], @@ -128,18 +140,6 @@ "parallel": false } }, - "supabase:ensure-started": { - "executor": "nx:run-commands", - "local": true, - "cache": false, - "options": { - "cwd": "{projectRoot}", - "commands": [ - "supabase status || (echo \"Starting Supabase...\" && supabase start)" - ], - "parallel": false - } - }, "supabase:start": { "executor": "nx:run-commands", "local": true, @@ -184,27 +184,40 @@ "executor": "nx:run-commands", "local": true, "cache": false, - "dependsOn": ["supabase:ensure-started"], "options": { "cwd": "{projectRoot}", - "commands": ["supabase db reset"], + "commands": [ + "../../scripts/ensure-supabase.sh .", + "supabase db reset" + ], "parallel": false } }, "test": { "executor": "nx:noop", - "inputs": ["default", "^production", "schemas", "migrations", "pgtapTests", "databaseTypes"], - "dependsOn": ["test:pgtap", "test:vitest", "test:types"] + "inputs": ["default", "^production"], + "dependsOn": ["test:vitest", "test:types"] + }, + "test:live": { + "executor": "nx:noop", + "inputs": ["schemas", "migrations", "pgtapTests"], + "dependsOn": ["test:pgtap"], + "metadata": { + "description": "Tests requiring live infrastructure (Supabase)" + } }, "test:pgtap": { "executor": "nx:run-commands", "local": true, - "dependsOn": ["verify-migrations", "supabase:ensure-started"], + "dependsOn": ["verify-migrations"], "inputs": ["schemas", "migrations", "pgtapTests"], "cache": true, "options": { "cwd": "{projectRoot}", - "commands": ["scripts/run-test-with-colors"], + "commands": [ + "../../scripts/ensure-supabase.sh .", + "scripts/run-test-with-colors" + ], "parallel": false } }, @@ -221,12 +234,16 @@ "test:pgtap:watch": { "executor": "nx:run-commands", "local": true, - "dependsOn": ["verify-migrations", "supabase:ensure-started"], + "dependsOn": ["verify-migrations"], "inputs": ["schemas", "migrations", "pgtapTests"], "cache": false, "options": { "cwd": "{projectRoot}", - "command": "scripts/watch-test" + "commands": [ + "../../scripts/ensure-supabase.sh .", + "scripts/watch-test" + ], + "parallel": false } }, "gen-types": { @@ -236,6 +253,7 @@ "outputs": ["{projectRoot}/src/database-types.ts"], "options": { "commands": [ + "../../scripts/ensure-supabase.sh .", "echo 'Generating database types...'", "supabase gen types --local --schema pgflow --schema pgmq > src/database-types.ts", "echo 'Verifying generated types...'", @@ -255,6 +273,7 @@ "options": { "cwd": "{projectRoot}", "commands": [ + "../../scripts/ensure-supabase.sh .", "mkdir -p .nx-inputs", "echo 'Verifying database types are up-to-date...'", "cp src/database-types.ts .nx-inputs/database-types.ts.backup", diff --git a/pkgs/edge-worker/project.json b/pkgs/edge-worker/project.json index b4af3401e..45dbd8e3f 100644 --- a/pkgs/edge-worker/project.json +++ b/pkgs/edge-worker/project.json @@ -45,7 +45,7 @@ "local": true, "inputs": ["{projectRoot}/src/**/*.ts"], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "command": "npx eslint 'src/**/*.ts' --rule 'no-restricted-syntax: [\"error\", {\"selector\": \"TSModuleDeclaration[global=true]\", \"message\": \"JSR forbids global type augmentation (declare global). This will cause JSR publish to fail.\"}]'" } }, @@ -58,7 +58,7 @@ "local": true, "cache": false, "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": ["supabase start"], "parallel": false } @@ -68,7 +68,7 @@ "local": true, "cache": false, "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": ["supabase stop --no-backup"], "parallel": false } @@ -78,7 +78,7 @@ "local": true, "cache": false, "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": ["supabase status"], "parallel": false } @@ -88,7 +88,7 @@ "local": true, "cache": false, "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": ["supabase stop --no-backup", "supabase start"], "parallel": false } @@ -103,8 +103,9 @@ "^production" ], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": [ + "../../scripts/ensure-supabase.sh .", "mkdir -p supabase/migrations/", "rm -f supabase/migrations/*.sql", "cp ../core/supabase/migrations/*.sql supabase/migrations/", @@ -119,7 +120,7 @@ "local": true, "cache": false, "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": [ "supabase functions serve --env-file supabase/functions/.env --no-verify-jwt" ], @@ -132,7 +133,7 @@ "dependsOn": ["^verify-migrations", "^dump-realtime-schema"], "inputs": ["default", "^production", "{projectRoot}/scripts/ensure-db"], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": ["./scripts/ensure-db"], "parallel": false } @@ -143,7 +144,7 @@ "local": true, "inputs": ["default", "^production"], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": [ "deno test --config deno.test.json --allow-all --env=supabase/functions/.env tests/unit/" ], @@ -156,7 +157,7 @@ "local": true, "inputs": ["default", "^production"], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": [ "deno test --config deno.test.json --allow-all --env=supabase/functions/.env tests/integration/" ], @@ -169,7 +170,7 @@ "local": true, "inputs": ["^production"], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": ["./scripts/sync-e2e-deps.sh"], "parallel": false } @@ -181,7 +182,7 @@ "local": true, "cache": false, "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": [ "supabase functions serve --env-file supabase/functions/.env --import-map supabase/functions/deno.json --no-verify-jwt" ], @@ -190,12 +191,13 @@ }, "test:e2e": { "executor": "nx:run-commands", - "dependsOn": ["supabase:reset", "serve:functions:e2e"], + "dependsOn": ["supabase:reset", "sync-e2e-deps"], "local": true, "inputs": ["default", "^production"], "options": { - "cwd": "pkgs/edge-worker", + "cwd": "{projectRoot}", "commands": [ + "../../scripts/ensure-supabase.sh .", "deno test --config deno.test.json --allow-all --env=supabase/functions/.env tests/e2e/" ], "parallel": false @@ -203,7 +205,15 @@ }, "test": { "inputs": ["default", "^production"], - "dependsOn": ["test:types", "test:unit", "test:integration"] + "dependsOn": ["test:types"] + }, + "test:live": { + "executor": "nx:noop", + "inputs": ["default", "^production"], + "dependsOn": ["test:unit", "test:integration", "test:e2e"], + "metadata": { + "description": "Tests requiring live infrastructure (Docker Compose + Supabase Edge Functions)" + } }, "test:types": { "executor": "nx:run-commands", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 75a7c5dc7..04490401c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -238,6 +238,9 @@ importers: '@types/node': specifier: ^22.14.1 version: 22.16.5 + supabase: + specifier: ^2.34.3 + version: 2.34.3 tsx: specifier: ^4.19.3 version: 4.20.3 diff --git a/scripts/ensure-supabase.sh b/scripts/ensure-supabase.sh new file mode 100755 index 000000000..d5252bf65 --- /dev/null +++ b/scripts/ensure-supabase.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e + +# Usage: ensure-supabase.sh +if [ -z "$1" ]; then + echo "ERROR: Supabase directory path required" + echo "Usage: ensure-supabase.sh " + exit 1 +fi + +SUPABASE_DIR="$1" + +if [ ! -d "$SUPABASE_DIR/supabase" ]; then + echo "ERROR: No supabase/ directory found in $SUPABASE_DIR" + exit 1 +fi + +cd "$SUPABASE_DIR" + +if pnpm supabase status &>/dev/null; then + echo "โœ“ Supabase already running in $SUPABASE_DIR" + exit 0 +fi + +echo "Starting Supabase in $SUPABASE_DIR..." +pnpm supabase start