[CI] Comment on new env PRs

.github/workflows/pr-new-env.yml

@@ -1,6 +1,7 @@
 name: PR New Environment
 
 on:
+  workflow_dispatch:
   pull_request_target:
     types:
       - opened
@@ -94,17 +95,13 @@ jobs:
           echo "new_envs_json=${NEW_ENVS_JSON}" >> "$GITHUB_OUTPUT"
 
   deploy-and-health-check:
-    name: Deploy and Validate New Environments
+    name: Validate New Environments
     needs: detect-new-envs
     if: needs.detect-new-envs.outputs.has_new_envs == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
         environment: ${{ fromJSON(needs.detect-new-envs.outputs.new_envs_json) }}
-    env:
-      HF_TOKEN: ${{ secrets.HF_PR_TOKEN }}
-      HF_NAMESPACE: ${{ vars.HF_PR_NAMESPACE }}
-      SPACE_SUFFIX: -pr-${{ github.event.number }}
     steps:
       - name: Checkout PR code
         uses: actions/checkout@v4
@@ -114,144 +111,136 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
-      - name: Default Hugging Face namespace
-        if: env.HF_NAMESPACE == ''
-        shell: bash
-        run: echo "HF_NAMESPACE=openenv-testing" >> "$GITHUB_ENV"
-
-      - name: Verify Hugging Face token
-        shell: bash
-        run: |
-          if [ -z "${HF_TOKEN:-}" ]; then
-            echo "HF_TOKEN secret is required for deployment." >&2
-            exit 1
-          fi
-
-      - name: Install Hugging Face CLI
-        shell: bash
-        run: |
-          curl -LsSf https://hf.co/cli/install.sh | bash
-          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
 
-      - name: Deploy environment to Hugging Face
+      - name: Install OpenEnv package
         shell: bash
         run: |
           set -euo pipefail
-          chmod +x scripts/deploy_to_hf.sh
-          ./scripts/deploy_to_hf.sh --env "${{ matrix.environment }}" --space-suffix "${SPACE_SUFFIX}" --hub-tag "openenv-pr"
+          python -m pip install --upgrade pip
+          pip install .
 
-      - name: Wait for deployment to stabilize
-        shell: bash
-        run: sleep 180
-
-      - name: Compute Space URLs
-        id: urls
+      - name: Run openenv validate --verbose
+        id: validate
         shell: bash
         run: |
-          set -euo pipefail
+          set -u -o pipefail
+          env_dir="src/envs/${{ matrix.environment }}"
 
-          if [ -z "${HF_NAMESPACE:-}" ]; then
-            echo "HF_NAMESPACE is not configured; unable to compute space URLs." >&2
+          if [ ! -d "$env_dir" ]; then
+            echo "Environment directory not found: $env_dir" >&2
+            echo "status=failure" >> "$GITHUB_OUTPUT"
+            printf "details<<EOF\n%s\nEOF\n" "Environment directory not found" >> "$GITHUB_OUTPUT"
             exit 1
           fi
 
-          namespace_slug=$(echo "${HF_NAMESPACE}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
-          space_name="${{ matrix.environment }}${SPACE_SUFFIX}"
-          space_slug=$(echo "${space_name}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
-          health_url="https://${namespace_slug}-${space_slug}.hf.space/health"
-          live_url="https://${namespace_slug}-${space_slug}.hf.space"
-          space_repo_url="https://huggingface.co/spaces/${HF_NAMESPACE}/${space_name}"
-
-          echo "namespace_slug=${namespace_slug}" >> "$GITHUB_OUTPUT"
-          echo "space_name=${space_name}" >> "$GITHUB_OUTPUT"
-          echo "space_slug=${space_slug}" >> "$GITHUB_OUTPUT"
-          echo "health_url=${health_url}" >> "$GITHUB_OUTPUT"
-          echo "live_url=${live_url}" >> "$GITHUB_OUTPUT"
-          echo "space_repo_url=${space_repo_url}" >> "$GITHUB_OUTPUT"
-
-      - name: Perform environment health check
-        id: health_check
-        continue-on-error: true
-        shell: bash
-        env:
-          HEALTH_URL: ${{ steps.urls.outputs.health_url }}
-          SPACE_NAME: ${{ steps.urls.outputs.space_name }}
-        run: |
-          set -euo pipefail
+          cd "$env_dir"
+          echo "Running openenv validate --verbose in $(pwd)"
 
-          if [ -z "${HEALTH_URL:-}" ]; then
-            echo "HEALTH_URL not provided; cannot perform health check." >&2
-            exit 1
+          output_file="$(mktemp)"
+          if openenv validate --verbose | tee "$output_file"; then
+            status="success"
+            exit_code=0
+          else
+            status="failure"
+            exit_code=$?
           fi
 
-          echo "Checking health for ${SPACE_NAME} at ${HEALTH_URL}"
+          log_contents="$(cat "$output_file")"
+          rm -f "$output_file"
 
-          success=0
-          for attempt in {1..5}; do
-            status=$(curl -sS -o response.json -w "%{http_code}" "$HEALTH_URL" || echo "000")
-            if [ "$status" = "200" ]; then
-              echo "Health check passed for ${SPACE_NAME}"
-              cat response.json
-              success=1
-              break
-            fi
-            echo "Attempt ${attempt} returned status ${status}. Retrying in 30 seconds..."
-            sleep 30
-          done
+          echo "status=${status}" >> "$GITHUB_OUTPUT"
+          printf "details<<EOF\n%s\nEOF\n" "$log_contents" >> "$GITHUB_OUTPUT"
 
-          if [ $success -ne 1 ]; then
-            echo "Health check failed for ${SPACE_NAME}" >&2
-            if [ -f response.json ]; then
-              echo "Last response payload:"
-              cat response.json
-            fi
-            exit 1
-          fi
+          exit "$exit_code"
 
       - name: Comment on PR with deployment status
         if: always()
         uses: actions/github-script@v7
         env:
-          HEALTH_CONCLUSION: ${{ steps.health_check.conclusion }}
-          SPACE_NAME: ${{ steps.urls.outputs.space_name }}
-          LIVE_URL: ${{ steps.urls.outputs.live_url }}
-          SPACE_REPO_URL: ${{ steps.urls.outputs.space_repo_url }}
+          VALIDATION_STATUS: ${{ steps.validate.outputs.status }}
+          VALIDATION_LOG: ${{ steps.validate.outputs.details }}
           ENV_NAME: ${{ matrix.environment }}
+          COMMENT_TAG: "<!-- openenv-pr-preview -->"
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const status = process.env.HEALTH_CONCLUSION || 'failure';
-            const spaceName = process.env.SPACE_NAME;
-            const liveUrl = process.env.LIVE_URL;
-            const repoUrl = process.env.SPACE_REPO_URL;
+            const status = (process.env.VALIDATION_STATUS || '').toLowerCase() === 'success' ? 'success' : 'failure';
             const envName = process.env.ENV_NAME;
+            const marker = process.env.COMMENT_TAG;
 
             const header = status === 'success'
-              ? `✅ Deployment succeeded for \`${envName}\``
-              : `⚠️ Deployment failed for \`${envName}\``;
+              ? `✅ Validation succeeded for \`${envName}\``
+              : `⚠️ Validation failed for \`${envName}\``;
 
             const summary = status === 'success'
-              ? 'Nice work! Wait for a code review and we\'re ready to go.'
-              : 'Please resolve your environment.';
-
-            const body = [
+              ? 'Your env passes the vibe check. However, most environments should go straight to the hub, they will automatically be added to the official Env Hub collection on a nightly basis. Environments in the official specification repo are only meant to demonstrate usage of a specific spec feature for educational purposes. Re-run locally with:'
+              : 'Validation reported issues. Review the log and re-run locally with `openenv validate --verbose`. Please note, we recently changed the standard template, your environment might pre-date this standard, follow the conversion guide https://github.com/meta-pytorch/OpenEnv/blob/main/scripts/CONVERT.md to convert your environment to the new standard.';
+
+            const envDir = 'src/envs/' + envName;
+            const rawLog = process.env.VALIDATION_LOG || '';
+            const trimmedLog = rawLog.trim();
+            const maxLength = 6000;
+            let displayLog = trimmedLog;
+            if (displayLog.length > maxLength) {
+              displayLog = displayLog.slice(0, maxLength) + '\n... (truncated)';
+            }
+
+            const bodyLines = [
+              marker,
+              '',
               header,
               '',
-              `- Space repo: [${repoUrl}](${repoUrl})`,
-              `- Live URL: [${liveUrl}](${liveUrl})`,
               '',
               summary,
               '',
-              'You can iterate locally or validate fixes by running `scripts/deploy_to_hf.sh --env "' + envName + '"`.'
-            ].join('\n');
-
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.payload.pull_request.number,
-              body
-            });
-
-      - name: Fail job if health check failed
-        if: steps.health_check.conclusion == 'failure'
-        run: exit 1
+              '- `openenv validate --verbose ' + envDir + '`',
+              '',
+              '```\n' + displayLog + '\n```',
+              'You can deploy the environment to Hugging Face Spaces by running `openenv push`.'
+            ];
+
+            const {owner, repo} = context.repo;
+            const issue_number = context.payload.pull_request.number;
+            const serverUrl = process.env.GITHUB_SERVER_URL || 'https://github.com';
+            const runUrl = `${serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+
+            if (status !== 'success') {
+              bodyLines.push(`- Failed run: ${runUrl}`);
+            } else {
+              bodyLines.push(`- Success run: ${runUrl}`);
+            }
+
+            const bodyText = bodyLines.join('\n');
+
+            const existing = await github.paginate(
+              github.rest.issues.listComments,
+              { owner, repo, issue_number, per_page: 100 },
+              (response, done) => {
+                const match = response.data.find(comment => comment.body && comment.body.includes(marker));
+                if (match) {
+                  done();
+                  return [match];
+                }
+                return [];
+              }
+            );
+
+            if (existing.length > 0) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: existing[0].id,
+                body: bodyText,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body: bodyText,
+              });
+            }