From 7cf44a6edee171766a8c847137a70706b62d3e1f Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 4 Nov 2025 10:03:49 +0100 Subject: [PATCH 1/5] add cli to to ci for pr --- .github/workflows/pr-new-env.yml | 133 +++++++++++++++++++++++++------ 1 file changed, 108 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr-new-env.yml b/.github/workflows/pr-new-env.yml index 1374abf7..e7c49c53 100644 --- a/.github/workflows/pr-new-env.yml +++ b/.github/workflows/pr-new-env.yml @@ -103,6 +103,7 @@ jobs: environment: ${{ fromJSON(needs.detect-new-envs.outputs.new_envs_json) }} env: HF_TOKEN: ${{ secrets.HF_PR_TOKEN }} + HF_PR_TOKEN: ${{ secrets.HF_PR_TOKEN }} HF_NAMESPACE: ${{ vars.HF_PR_NAMESPACE }} SPACE_SUFFIX: -pr-${{ github.event.number }} steps: @@ -127,18 +128,31 @@ jobs: exit 1 fi - - name: Install Hugging Face CLI + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install OpenEnv package shell: bash run: | - curl -LsSf https://hf.co/cli/install.sh | bash - echo "$HOME/.local/bin" >> "$GITHUB_PATH" + set -euo pipefail + python -m pip install --upgrade pip + pip install . - - name: Deploy environment to Hugging Face + - name: Deploy environment with OpenEnv CLI shell: bash run: | set -euo pipefail - chmod +x scripts/deploy_to_hf.sh - ./scripts/deploy_to_hf.sh --env "${{ matrix.environment }}" --space-suffix "${SPACE_SUFFIX}" --hub-tag "openenv-pr" + repo_id="${HF_NAMESPACE}/${{ matrix.environment }}${SPACE_SUFFIX}" + env_dir="src/envs/${{ matrix.environment }}" + + if [ ! -d "$env_dir" ]; then + echo "Environment directory not found: $env_dir" >&2 + exit 1 + fi + + openenv push --directory "$env_dir" --repo-id "$repo_id" - name: Wait for deployment to stabilize shell: bash @@ -161,6 +175,7 @@ jobs: health_url="https://${namespace_slug}-${space_slug}.hf.space/health" live_url="https://${namespace_slug}-${space_slug}.hf.space" space_repo_url="https://huggingface.co/spaces/${HF_NAMESPACE}/${space_name}" + space_repo_id="${HF_NAMESPACE}/${space_name}" echo "namespace_slug=${namespace_slug}" >> "$GITHUB_OUTPUT" echo "space_name=${space_name}" >> "$GITHUB_OUTPUT" @@ -168,6 +183,7 @@ jobs: echo "health_url=${health_url}" >> "$GITHUB_OUTPUT" echo "live_url=${live_url}" >> "$GITHUB_OUTPUT" echo "space_repo_url=${space_repo_url}" >> "$GITHUB_OUTPUT" + echo "space_repo_id=${space_repo_id}" >> "$GITHUB_OUTPUT" - name: Perform environment health check id: health_check @@ -216,41 +232,108 @@ jobs: SPACE_NAME: ${{ steps.urls.outputs.space_name }} LIVE_URL: ${{ steps.urls.outputs.live_url }} SPACE_REPO_URL: ${{ steps.urls.outputs.space_repo_url }} + SPACE_REPO_ID: ${{ steps.urls.outputs.space_repo_id }} ENV_NAME: ${{ matrix.environment }} + COMMENT_TAG: "" with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const status = process.env.HEALTH_CONCLUSION || 'failure'; - const spaceName = process.env.SPACE_NAME; - const liveUrl = process.env.LIVE_URL; - const repoUrl = process.env.SPACE_REPO_URL; const envName = process.env.ENV_NAME; + const marker = process.env.COMMENT_TAG; const header = status === 'success' - ? `✅ Deployment succeeded for \`${envName}\`` - : `⚠️ Deployment failed for \`${envName}\``; + ? `✅ Deployment to Hugging Face succeeded for \`${envName}\`` + : `⚠️ Deployment Hugging Face failed for \`${envName}\``; const summary = status === 'success' - ? 'Nice work! Wait for a code review and we\'re ready to go.' - : 'Please resolve your environment.'; + ? 'Nice work! Wait for a code review and we\'re ready to go. You can test it with the CLI:' + : 'Please resolve your environment and test it with the CLI:'; + + const envDir = 'src/envs/' + envName; - const body = [ + const bodyLines = [ + marker, + '', header, '', - `- Space repo: [${repoUrl}](${repoUrl})`, - `- Live URL: [${liveUrl}](${liveUrl})`, '', summary, '', - 'You can iterate locally or validate fixes by running `scripts/deploy_to_hf.sh --env "' + envName + '"`.' - ].join('\n'); - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - body - }); + '- `openenv push --directory ' + envDir + ' --repo-id /' + envName + '`', + ]; + + const {owner, repo} = context.repo; + const issue_number = context.payload.pull_request.number; + const serverUrl = process.env.GITHUB_SERVER_URL || 'https://github.com'; + const runUrl = `${serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + + if (status !== 'success') { + bodyLines.push(`- Failed run: ${runUrl}`); + } else { + bodyLines.push(`- Success run: ${runUrl}`); + } + + const bodyText = bodyLines.join('\n'); + + const existing = await github.paginate( + github.rest.issues.listComments, + { owner, repo, issue_number, per_page: 100 }, + (response, done) => { + const match = response.data.find(comment => comment.body && comment.body.includes(marker)); + if (match) { + done(); + return [match]; + } + return []; + } + ); + + if (existing.length > 0) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: existing[0].id, + body: bodyText, + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body: bodyText, + }); + } + + - name: Delete preview space on Hugging Face + if: always() + continue-on-error: true + shell: bash + env: + SPACE_REPO_ID: ${{ steps.urls.outputs.space_repo_id }} + run: | + set -euo pipefail + if [ -z "${SPACE_REPO_ID:-}" ]; then + echo "No space repo id; skipping deletion" + exit 0 + fi + + TOKEN="${HF_TOKEN:-${HF_PR_TOKEN:-}}" + if [ -z "$TOKEN" ]; then + echo "HF token not available; cannot delete space" + exit 0 + fi + + set +e + hf repo delete "$SPACE_REPO_ID" --repo-type space --yes --token "$TOKEN" + status=$? + set -e + + if [ $status -eq 0 ]; then + echo "Deleted preview space $SPACE_REPO_ID" + else + echo "Failed to delete space $SPACE_REPO_ID (exit $status)" + fi - name: Fail job if health check failed if: steps.health_check.conclusion == 'failure' From 8929ce438a56db3c99293f98c2e75cdf5051d76b Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 11 Nov 2025 10:45:13 +0100 Subject: [PATCH 2/5] remove push and use validate --- .github/workflows/pr-new-env.yml | 170 +++++++------------------------ 1 file changed, 37 insertions(+), 133 deletions(-) diff --git a/.github/workflows/pr-new-env.yml b/.github/workflows/pr-new-env.yml index e7c49c53..c82b0394 100644 --- a/.github/workflows/pr-new-env.yml +++ b/.github/workflows/pr-new-env.yml @@ -94,18 +94,13 @@ jobs: echo "new_envs_json=${NEW_ENVS_JSON}" >> "$GITHUB_OUTPUT" deploy-and-health-check: - name: Deploy and Validate New Environments + name: Validate New Environments needs: detect-new-envs if: needs.detect-new-envs.outputs.has_new_envs == 'true' runs-on: ubuntu-latest strategy: matrix: environment: ${{ fromJSON(needs.detect-new-envs.outputs.new_envs_json) }} - env: - HF_TOKEN: ${{ secrets.HF_PR_TOKEN }} - HF_PR_TOKEN: ${{ secrets.HF_PR_TOKEN }} - HF_NAMESPACE: ${{ vars.HF_PR_NAMESPACE }} - SPACE_SUFFIX: -pr-${{ github.event.number }} steps: - name: Checkout PR code uses: actions/checkout@v4 @@ -115,19 +110,6 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Default Hugging Face namespace - if: env.HF_NAMESPACE == '' - shell: bash - run: echo "HF_NAMESPACE=openenv-testing" >> "$GITHUB_ENV" - - - name: Verify Hugging Face token - shell: bash - run: | - if [ -z "${HF_TOKEN:-}" ]; then - echo "HF_TOKEN secret is required for deployment." >&2 - exit 1 - fi - - name: Set up Python uses: actions/setup-python@v5 with: @@ -140,117 +122,71 @@ jobs: python -m pip install --upgrade pip pip install . - - name: Deploy environment with OpenEnv CLI + - name: Run openenv validate --verbose + id: validate shell: bash run: | - set -euo pipefail - repo_id="${HF_NAMESPACE}/${{ matrix.environment }}${SPACE_SUFFIX}" + set -u -o pipefail env_dir="src/envs/${{ matrix.environment }}" if [ ! -d "$env_dir" ]; then echo "Environment directory not found: $env_dir" >&2 + echo "status=failure" >> "$GITHUB_OUTPUT" + printf "details<> "$GITHUB_OUTPUT" exit 1 fi - openenv push --directory "$env_dir" --repo-id "$repo_id" - - - name: Wait for deployment to stabilize - shell: bash - run: sleep 180 - - - name: Compute Space URLs - id: urls - shell: bash - run: | - set -euo pipefail + cd "$env_dir" + echo "Running openenv validate --verbose in $(pwd)" - if [ -z "${HF_NAMESPACE:-}" ]; then - echo "HF_NAMESPACE is not configured; unable to compute space URLs." >&2 - exit 1 + output_file="$(mktemp)" + if openenv validate --verbose | tee "$output_file"; then + status="success" + exit_code=0 + else + status="failure" + exit_code=$? fi - namespace_slug=$(echo "${HF_NAMESPACE}" | tr '[:upper:]' '[:lower:]' | tr '_' '-') - space_name="${{ matrix.environment }}${SPACE_SUFFIX}" - space_slug=$(echo "${space_name}" | tr '[:upper:]' '[:lower:]' | tr '_' '-') - health_url="https://${namespace_slug}-${space_slug}.hf.space/health" - live_url="https://${namespace_slug}-${space_slug}.hf.space" - space_repo_url="https://huggingface.co/spaces/${HF_NAMESPACE}/${space_name}" - space_repo_id="${HF_NAMESPACE}/${space_name}" - - echo "namespace_slug=${namespace_slug}" >> "$GITHUB_OUTPUT" - echo "space_name=${space_name}" >> "$GITHUB_OUTPUT" - echo "space_slug=${space_slug}" >> "$GITHUB_OUTPUT" - echo "health_url=${health_url}" >> "$GITHUB_OUTPUT" - echo "live_url=${live_url}" >> "$GITHUB_OUTPUT" - echo "space_repo_url=${space_repo_url}" >> "$GITHUB_OUTPUT" - echo "space_repo_id=${space_repo_id}" >> "$GITHUB_OUTPUT" - - - name: Perform environment health check - id: health_check - continue-on-error: true - shell: bash - env: - HEALTH_URL: ${{ steps.urls.outputs.health_url }} - SPACE_NAME: ${{ steps.urls.outputs.space_name }} - run: | - set -euo pipefail + log_contents="$(cat "$output_file")" + rm -f "$output_file" - if [ -z "${HEALTH_URL:-}" ]; then - echo "HEALTH_URL not provided; cannot perform health check." >&2 - exit 1 - fi + echo "status=${status}" >> "$GITHUB_OUTPUT" + printf "details<> "$GITHUB_OUTPUT" - echo "Checking health for ${SPACE_NAME} at ${HEALTH_URL}" - - success=0 - for attempt in {1..5}; do - status=$(curl -sS -o response.json -w "%{http_code}" "$HEALTH_URL" || echo "000") - if [ "$status" = "200" ]; then - echo "Health check passed for ${SPACE_NAME}" - cat response.json - success=1 - break - fi - echo "Attempt ${attempt} returned status ${status}. Retrying in 30 seconds..." - sleep 30 - done - - if [ $success -ne 1 ]; then - echo "Health check failed for ${SPACE_NAME}" >&2 - if [ -f response.json ]; then - echo "Last response payload:" - cat response.json - fi - exit 1 - fi + exit "$exit_code" - name: Comment on PR with deployment status if: always() uses: actions/github-script@v7 env: - HEALTH_CONCLUSION: ${{ steps.health_check.conclusion }} - SPACE_NAME: ${{ steps.urls.outputs.space_name }} - LIVE_URL: ${{ steps.urls.outputs.live_url }} - SPACE_REPO_URL: ${{ steps.urls.outputs.space_repo_url }} - SPACE_REPO_ID: ${{ steps.urls.outputs.space_repo_id }} + VALIDATION_STATUS: ${{ steps.validate.outputs.status }} + VALIDATION_LOG: ${{ steps.validate.outputs.details }} ENV_NAME: ${{ matrix.environment }} COMMENT_TAG: "" with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const status = process.env.HEALTH_CONCLUSION || 'failure'; + const status = (process.env.VALIDATION_STATUS || '').toLowerCase() === 'success' ? 'success' : 'failure'; const envName = process.env.ENV_NAME; const marker = process.env.COMMENT_TAG; const header = status === 'success' - ? `✅ Deployment to Hugging Face succeeded for \`${envName}\`` - : `⚠️ Deployment Hugging Face failed for \`${envName}\``; + ? `✅ Validation succeeded for \`${envName}\`` + : `⚠️ Validation failed for \`${envName}\``; const summary = status === 'success' - ? 'Nice work! Wait for a code review and we\'re ready to go. You can test it with the CLI:' - : 'Please resolve your environment and test it with the CLI:'; + ? 'Nice work! Validation passed. Re-run locally with:' + : 'Validation reported issues. Review the log and re-run locally:'; const envDir = 'src/envs/' + envName; + const rawLog = process.env.VALIDATION_LOG || ''; + const trimmedLog = rawLog.trim(); + const maxLength = 6000; + let displayLog = trimmedLog; + if (displayLog.length > maxLength) { + displayLog = displayLog.slice(0, maxLength) + '\n... (truncated)'; + } const bodyLines = [ marker, @@ -260,7 +196,9 @@ jobs: '', summary, '', - '- `openenv push --directory ' + envDir + ' --repo-id /' + envName + '`', + '- `openenv validate --verbose ' + envDir + '`', + '', + '```\n' + displayLog + '\n```', ]; const {owner, repo} = context.repo; @@ -304,37 +242,3 @@ jobs: body: bodyText, }); } - - - name: Delete preview space on Hugging Face - if: always() - continue-on-error: true - shell: bash - env: - SPACE_REPO_ID: ${{ steps.urls.outputs.space_repo_id }} - run: | - set -euo pipefail - if [ -z "${SPACE_REPO_ID:-}" ]; then - echo "No space repo id; skipping deletion" - exit 0 - fi - - TOKEN="${HF_TOKEN:-${HF_PR_TOKEN:-}}" - if [ -z "$TOKEN" ]; then - echo "HF token not available; cannot delete space" - exit 0 - fi - - set +e - hf repo delete "$SPACE_REPO_ID" --repo-type space --yes --token "$TOKEN" - status=$? - set -e - - if [ $status -eq 0 ]; then - echo "Deleted preview space $SPACE_REPO_ID" - else - echo "Failed to delete space $SPACE_REPO_ID (exit $status)" - fi - - - name: Fail job if health check failed - if: steps.health_check.conclusion == 'failure' - run: exit 1 From e4db712bd7978e05de33d7f7373f72e27ad37a34 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 11 Nov 2025 11:02:06 +0100 Subject: [PATCH 3/5] add instuction to push --- .github/workflows/pr-new-env.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-new-env.yml b/.github/workflows/pr-new-env.yml index c82b0394..5984aeb7 100644 --- a/.github/workflows/pr-new-env.yml +++ b/.github/workflows/pr-new-env.yml @@ -199,6 +199,7 @@ jobs: '- `openenv validate --verbose ' + envDir + '`', '', '```\n' + displayLog + '\n```', + 'You can deploy the environment to Hugging Face Spaces by running `openenv push`.' ]; const {owner, repo} = context.repo; From 172b80f72145940f3b35b381a0770afa40536d5c Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Wed, 12 Nov 2025 17:05:36 +0100 Subject: [PATCH 4/5] improve comment note --- .github/workflows/pr-new-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-new-env.yml b/.github/workflows/pr-new-env.yml index 5984aeb7..ef3621da 100644 --- a/.github/workflows/pr-new-env.yml +++ b/.github/workflows/pr-new-env.yml @@ -176,7 +176,7 @@ jobs: : `⚠️ Validation failed for \`${envName}\``; const summary = status === 'success' - ? 'Nice work! Validation passed. Re-run locally with:' + ? 'Your env passes the vibe check. However, most environments should go straight to the hub, they will automatically be added to the official Env Hub collection on a nightly basis. Environments in the official specification repo are only meant to demonstrate usage of a specific spec feature for educational purposes. Re-run locally with:' : 'Validation reported issues. Review the log and re-run locally:'; const envDir = 'src/envs/' + envName; From b34a11b74f442d6d9937f6b53ec099ad0c732c08 Mon Sep 17 00:00:00 2001 From: Zach Wentz Date: Thu, 13 Nov 2025 09:30:59 -0500 Subject: [PATCH 5/5] Added conversion note to failures and ability to manually run --- .github/workflows/pr-new-env.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-new-env.yml b/.github/workflows/pr-new-env.yml index ef3621da..f233385c 100644 --- a/.github/workflows/pr-new-env.yml +++ b/.github/workflows/pr-new-env.yml @@ -1,6 +1,7 @@ name: PR New Environment on: + workflow_dispatch: pull_request_target: types: - opened @@ -177,7 +178,7 @@ jobs: const summary = status === 'success' ? 'Your env passes the vibe check. However, most environments should go straight to the hub, they will automatically be added to the official Env Hub collection on a nightly basis. Environments in the official specification repo are only meant to demonstrate usage of a specific spec feature for educational purposes. Re-run locally with:' - : 'Validation reported issues. Review the log and re-run locally:'; + : 'Validation reported issues. Review the log and re-run locally with `openenv validate --verbose`. Please note, we recently changed the standard template, your environment might pre-date this standard, follow the conversion guide https://github.com/meta-pytorch/OpenEnv/blob/main/scripts/CONVERT.md to convert your environment to the new standard.'; const envDir = 'src/envs/' + envName; const rawLog = process.env.VALIDATION_LOG || '';