Use overrides for using the same branchname over multiple repos

Author: robtaylor

.github/workflows/coverage.yml

@@ -25,6 +25,21 @@ jobs:
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v4
 
+      - name: Generate overrides to use current branch if PR
+        if: github.event_name == 'pull_request'
+        run: |
+          pdm run tools/gen_overrides.py ${{github.head_ref}} > overrides.txt
+          echo "Generated overrides:"
+          cat overrides.txt
+
+      - name: Relock PDM
+        if: github.event_name != 'pull_request'
+        run: pdm lock -d
+
+      - name: Relock PDM (PR)
+        if: github.event_name == 'pull_request'
+        run: pdm lock -d --override overrides.txt
+
       - name: Install dependencies
         run: |
           pdm install

.github/workflows/main.yaml

@@ -19,6 +19,22 @@ jobs:
           fetch-depth: 0
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v4
+
+      - name: Generate overrides to use current branch if PR
+        if: github.event_name == 'pull_request'
+        run: |
+          pdm run tools/gen_overrides.py ${{github.head_ref}} > overrides.txt
+          echo "Generated overrides:"
+          cat overrides.txt
+
+      - name: Relock PDM
+        if: github.event_name != 'pull_request'
+        run: pdm lock -d
+
+      - name: Relock PDM (PR)
+        if: github.event_name == 'pull_request'
+        run: pdm lock -d --override overrides.txt
+
       - name: Install dependencies
         run: |
           pdm install
@@ -39,75 +55,66 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check source code licenses
-        run: |
-          docker run --platform=linux/amd64 -v ${PWD}:/src ghcr.io/google/addlicense -v -check -l BSD-2-Clause -c "ChipFlow" -s=only -ignore **/__init__.py **/*.py
+        run: ./tools/license_check.sh
 
   test-submit:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        dry: [true, false]
+    env:
+      DRY: ${{ matrix.dry && '--dry-run' || '' }}
+      IS_DRY: ${{ matrix.dry && '(dry run)' || '' }}
+    name:  ${{ matrix.dry && 'Test Submit - Dry run' || 'Test submit' }}
+
     steps:
-      - name: Check out source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          path: chipflow-lib
       - name: Check out chipflow-examples
         uses: actions/checkout@v4
         with:
           repository: ChipFlow/chipflow-examples
           fetch-depth: 0
           path: chipflow-examples
+          ref: ${{ github.head_ref || 'refs/heads/main' }}
 
       - name: Set up PDM
         uses: pdm-project/setup-pdm@v4
         with:
-          python-version: "3.10"
+          python-version: '3.10'
           cache: true
-          cache-dependency-path: "./**/pyproject.toml"
-      - name: Install dependencies
+          cache-dependency-path: './**/pyproject.toml'
+
+      - name: Generate overrides to use current branch if PR
         working-directory: ./chipflow-examples
+        if: github.event_name == 'pull_request'
         run: |
-          pdm install
-          pdm run python -m ensurepip
-          pdm run python -m pip install -e ../chipflow-lib
-      - name: Run tests
+          pdm run tools/gen_overrides.py ${{github.head_ref}} > overrides.txt
+          echo "Generated overrides:"
+          cat overrides.txt
+
+      - name: Relock PDM
         working-directory: ./chipflow-examples
-        run: |
-          pdm test
-          pdm run chipflow pin lock
-          pdm run chipflow silicon submit --wait
-        env:
-          CHIPFLOW_API_KEY: ${{ secrets.CHIPFLOW_API_KEY}}
+        if: github.event_name != 'pull_request'
+        run: pdm lock -d
 
-  test-submit-dry:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out source code
-        uses: actions/checkout@v4
-        with:
-            fetch-depth: 0
-            path: chipflow-lib
-      - name: Check out chipflow-examples
-        uses: actions/checkout@v4
-        with:
-            repository: ChipFlow/chipflow-examples
-            fetch-depth: 0
-            path: chipflow-examples
+      - name: Relock PDM (PR)
+        working-directory: ./chipflow-examples
+        if: github.event_name == 'pull_request'
+        run: pdm lock -d --override overrides.txt
 
-      - name: Set up PDM
-        uses: pdm-project/setup-pdm@v4
-        with:
-          python-version: "3.10"
-          cache: true
-          cache-dependency-path: "./**/pyproject.toml"
       - name: Install dependencies
         working-directory: ./chipflow-examples
         run: |
           pdm install
-          pdm run python -m ensurepip
-          pdm run python -m pip install -e ../chipflow-lib
+
       - name: Run tests
         working-directory: ./chipflow-examples
         run: |
           pdm test
+
+      - name: Submit build ${{ env.is_dry }}
+        working-directory: ./chipflow-examples/minimal
+        run: |
           pdm run chipflow pin lock
-          pdm run chipflow silicon submit --dry-run
+          pdm run chipflow silicon submit --wait $DRY
+        env:
+          CHIPFLOW_API_KEY: ${{ secrets.CHIPFLOW_API_KEY}}

.github/workflows/preview-docs.yml

@@ -20,6 +20,21 @@ jobs:
           python-version: 3.12
           cache: true
 
+      - name: Generate overrides to use current branch if PR
+        if: github.event_name == 'pull_request'
+        run: |
+          pdm run tools/gen_overrides.py ${{github.head_ref}} > overrides.txt
+          echo "Generated overrides:"
+          cat overrides.txt
+
+      - name: Relock PDM
+        if: github.event_name != 'pull_request'
+        run: pdm lock -d
+
+      - name: Relock PDM (PR)
+        if: github.event_name == 'pull_request'
+        run: pdm lock -d --override overrides.txt
+
       - name: Install dependencies
         run: pdm install
 

tools/gen_overrides.py

@@ -0,0 +1,54 @@
+# gen-overrides.py
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#   "pyproject-parser",
+#   "requirements-parser",
+# ]
+# ///
+# SPDX-License-Identifier: BSD-2-Clause
+import os
+import subprocess
+import sys
+import urllib
+from pathlib import Path
+
+from pyproject_parser import PyProject
+from requirements.requirement import Requirement
+
+
+rootdir = Path(os.environ["PDM_PROJECT_ROOT"])
+
+def get_branch(repo_dir):
+    """Get the current git branch"""
+    return subprocess.check_output(['git', 'branch', '--show-current'], text=True).strip()
+
+def get_remote_branch(repo, branch):
+    return subprocess.call(
+        ['git', 'ls-remote', '--exit-code', '--heads', repo, f'refs/heads/{branch}'],
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0
+
+def gen_overrides():
+    if len(sys.argv) > 1:
+        branch = sys.argv[1]
+        if branch.startswith("refs/heads/"):
+            branch = branch[11:]
+    else:
+        branch = get_branch(rootdir)
+    prj = PyProject.load(rootdir / "pyproject.toml")
+    reqs = prj.project['dependencies']
+    git_reqs = [r for r in reqs if r.url and r.url.startswith('git+')]
+    for r in git_reqs:
+        parts = urllib.parse.urlparse(r.url, allow_fragments=True)
+        # remove any branches that might already be there
+        base = parts.path.rsplit(sep="@",maxsplit=1)[0]
+        clone_url = urllib.parse.urlunparse(parts._replace(path=base, scheme="https"))
+        if get_remote_branch(clone_url, branch):
+            path = f"{parts.path}@{branch}"
+        else:
+            path = parts.path
+        r.url = urllib.parse.urlunparse(parts._replace(path=path))
+        print(str(r))
+
+if __name__ == "__main__":
+    gen_overrides()

tools/license_check.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+docker run --platform=linux/amd64 -v ${PWD}:/src ghcr.io/google/addlicense -v -check -l BSD-2-Clause -c "ChipFlow" -s=only -ignore **/__init__.py **/*.py
+
+