Restore repository indexing (#532)

nightkr · web-flow · commit c5463432b94a · 2025-06-12T08:37:39.000Z
* Slim down GH action to use the same code path for dry and wet runs * Let Ansible handle repository filtering This fixes repository indexing (fixes #411), and makes config/repositories.yaml authoritative (again) on the operator configuration. * Try to fix GH workflow syntax, maybe? * Document sharding for the local test setup * .yml is not .yaml * Satisfy pre-commit
diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml
@@ -27,83 +27,26 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: ${{ inputs.fail-fast }}
-      # If you add a new repository here, also add it to config/repositories.yaml
       matrix:
+        # This should be the list of repository names from config/repositories.yaml:
+        # yq '.repositories | map(.name)' config/repositories.yaml
         repository:
-          - name: airflow-operator
-            pretty_string: Apache Airflow
-            product_string: airflow
-            url: stackabletech/airflow-operator.git
-          - name: commons-operator
-            include_productconfig: false
-            has_product: false
-            pretty_string: Stackable Commons
-            product_string: commons
-            url: stackabletech/commons-operator.git
-          - name: druid-operator
-            pretty_string: Apache Druid
-            product_string: druid
-            url: stackabletech/druid-operator.git
-          - name: hbase-operator
-            pretty_string: Apache HBase
-            product_string: hbase
-            url: stackabletech/hbase-operator.git
-          - name: hdfs-operator
-            pretty_string: Apache HDFS
-            product_string: hdfs
-            url: stackabletech/hdfs-operator.git
-          - name: hive-operator
-            pretty_string: Apache Hive
-            product_string: hive
-            url: stackabletech/hive-operator.git
-          - name: kafka-operator
-            pretty_string: Apache Kafka
-            product_string: kafka
-            url: stackabletech/kafka-operator.git
-          - name: nifi-operator
-            pretty_string: Apache NiFi
-            product_string: nifi
-            url: stackabletech/nifi-operator.git
-          - name: listener-operator
-            include_productconfig: false
-            has_product: false
-            pretty_string: Stackable Listener Operator
-            product_string: listener-operator
-            run_as: custom
-            url: stackabletech/listener-operator.git
-          - name: opa-operator
-            extra_crates:
-              - stackable-opa-bundle-builder
-            pretty_string: OpenPolicyAgent
-            product_string: opa
-            url: stackabletech/opa-operator.git
-          - name: opensearch-operator
-            pretty_string: OpenSearch
-            product_string: opensearch
-            url: stackabletech/opensearch-operator.git
-          - name: secret-operator
-            include_productconfig: false
-            has_product: false
-            pretty_string: Stackable Secret Operator
-            product_string: secret-operator
-            run_as: custom
-            url: stackabletech/secret-operator.git
-          - name: spark-k8s-operator
-            pretty_string: Apache Spark-on-Kubernetes
-            product_string: spark-k8s
-            url: stackabletech/spark-k8s-operator.git
-          - name: superset-operator
-            pretty_string: Apache Superset
-            product_string: superset
-            url: stackabletech/superset-operator.git
-          - name: trino-operator
-            pretty_string: Trino
-            product_string: trino
-            url: stackabletech/trino-operator.git
-          - name: zookeeper-operator
-            pretty_string: Apache ZooKeeper
-            product_string: zookeeper
-            url: stackabletech/zookeeper-operator.git
+          - airflow-operator
+          - commons-operator
+          - druid-operator
+          - hbase-operator
+          - hdfs-operator
+          - hive-operator
+          - kafka-operator
+          - nifi-operator
+          - listener-operator
+          - opa-operator
+          - opensearch-operator
+          - secret-operator
+          - spark-k8s-operator
+          - superset-operator
+          - trino-operator
+          - zookeeper-operator
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -113,10 +56,10 @@ jobs:
         env:
           DEBIAN_FRONTEND: noninteractive
         run: |
-            sudo apt update &&  \
-            sudo apt install -y software-properties-common && \
-            sudo apt-add-repository ppa:ansible/ansible -y && \
-            sudo apt install -y ansible
+          sudo apt update &&  \
+          sudo apt install -y software-properties-common && \
+          sudo apt-add-repository ppa:ansible/ansible -y && \
+          sudo apt install -y ansible
 
       # NOTE (@NickLarsenNZ): This could be removed in favor of nix-shell and rrbutani/use-nix-shell-action
       - name: Install deps for operators
@@ -148,36 +91,18 @@ jobs:
           echo "AUTHOR=stackabletech/developers"
           echo "REASON=Daily run triggered" >> "$GITHUB_ENV"
 
-      - name: Overwrite repositories.yaml for ${{ matrix.repository.name }}
-        run: |
-          cat >config/repositories.yaml <<CONFIG
-          ---
-          repositories:
-          - ${{ toJSON(matrix.repository) }}
-          CONFIG
-
-          # For debugging
-          cat config/repositories.yaml
-
-      # Generate PRs
       - name: Run playbook
-        if: ${{ !inputs.dry-run }}
         env:
           CUSTOM_PR_TITLE: ${{ inputs.custom-pr-title }}
           GH_ACCESS_TOKEN: ${{ secrets.gh_access_token }}
+          REPOSITORY: ${{ matrix.repository }}
+          # tags local excludes all actions that actually generate PRs
+          DRY_RUN_FLAGS: ${{ inputs.dry-run && '--tags=local' || '' }}
+        # shellsheck disable=SC2086
         run: |
-          # Funnel via JSON to ensure that values are escaped properly
-          # Check if custom PR title is set. If so, insert it into the extra vars
-          if [ -n "$CUSTOM_PR_TITLE" ]; then
-            echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN}' --arg pwd "$(pwd)" > vars.json
-          else
-            echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN}' --arg pwd "$(pwd)" > vars.json
-          fi
-          ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json"
-
-      # Do Not Generate PRs
-      - name: Run playbook (dry-run)
-        if: ${{ inputs.dry-run }}
-        run: ./test.sh
-        env:
-          GH_ACCESS_TOKEN: ""
+          echo '{}' | jq '{commit_hash: $ENV.GITHUB_SHA, author: $ENV.AUTHOR, reason: $ENV.REASON, custom_pr_title: $ENV.CUSTOM_PR_TITLE, base_dir: $pwd, gh_access_token: $ENV.GH_ACCESS_TOKEN, shard_repositories: [$ENV.REPOSITORY]} | with_entries(select(.value != null and .value != ""))' --arg pwd "$(pwd)" > vars.json
+          # $DRY_RUN_FLAGS is intentionally not quoted, since we need to avoid
+          # passing an argument if we're not in dry mode.
+          # This is safe, since it always has one of two hard-coded values.
+          # shellcheck disable=SC2086
+          ansible-playbook playbook/playbook.yaml --extra-vars "@vars.json" $DRY_RUN_FLAGS
diff --git a/README.adoc b/README.adoc
@@ -85,7 +85,7 @@ Consult the section above to learn more about the structure of the template.
 
 === Test changes locally
 
-1. Run the `test.sh` script.
+1. Run the `test.sh` script. To limit the run to a single operator, add `--extra-vars "shard_repositories=['nifi-operator']"` (or your choice of operator).
    It will automatically delete and recreate a `work` directory.
 2. The changes can be examined with `git status`.
    When the pull request is later merged into the `main` branch then pull requests with these changes will be created automatically.
diff --git a/config/repositories.yaml b/config/repositories.yaml
@@ -1,7 +1,5 @@
-# This file exists just for the purpose of running test.sh locally.
-# The GH workflow generate_prs.yml will overwrite it with the contents if the
-# matrix run.
-# If you add a new repository here, you also need to add it to that workflow file.
+# If you add a new repository here, you also need to add it to the list of
+# repositories in ../.github/workflows/generate_prs.yml.
 ---
 repositories:
   - name: airflow-operator
diff --git a/playbook/playbook.yaml b/playbook/playbook.yaml
@@ -24,6 +24,7 @@
 
     - name: Update repositories from templates
       include_tasks: "update_repo.yaml"
+      when: shard_repositories is undefined or operator.name in shard_repositories
       with_items: "{{ repositories }}"
       loop_control:
         loop_var: operator
diff --git a/test.sh b/test.sh
@@ -2,4 +2,4 @@
 
 rm -fr work || true
 mkdir -p work
-ansible-playbook playbook/playbook.yaml --tags "local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'"
+ansible-playbook playbook/playbook.yaml --tags="local" --extra-vars "gh_access_token=unneeded base_dir=$(pwd) commit_hash=12345 reason='original message'" "$@"
