Merge remote-tracking branch 'origin/main' into feat/operator-service

sbernauer · sbernauer · commit 9610d8b50f2d · 2025-08-06T15:41:41.000+02:00
diff --git a/.github/workflows/generate_prs.yml b/.github/workflows/generate_prs.yml
@@ -51,7 +51,7 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
-      - uses: cachix/install-nix-action@17fe5fb4a23ad6cbbe47d6b3f359611ad276644c # v31.4.0
+      - uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31.4.1
       - name: Install Ansible
         env:
           DEBIAN_FRONTEND: noninteractive
diff --git a/.github/workflows/pr_pre-commit.yml b/.github/workflows/pr_pre-commit.yml
@@ -6,7 +6,7 @@ on:
 
 env:
   HADOLINT_VERSION: "v2.12.0"
-  PYTHON_VERSION: "3.12"
+  PYTHON_VERSION: "3.13"
 
 permissions: {}
 
@@ -18,7 +18,7 @@ jobs:
         with:
           persist-credentials: false
           fetch-depth: 0
-      - uses: stackabletech/actions/run-pre-commit@9aae2d1c14239021bfa33c041010f6fb7adec815 # v0.8.2
+      - uses: stackabletech/actions/run-pre-commit@4483641a7e24057bd2ba51cb4c3f2f0010ad21b7 # v0.8.4
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           hadolint: ${{ env.HADOLINT_VERSION }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,14 +15,14 @@ repos:
       - id: detect-private-key
 
   - repo: https://github.com/adrienverge/yamllint
-    rev: 81e9f98ffd059efe8aa9c1b1a42e5cce61b640c6 # 1.35.1
+    rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # 1.37.1
     hooks:
       - id: yamllint
         types: [text]
         files: \.(yml|yaml)(\.j2)*$
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: 586c3ea3f51230da42bab657c6a32e9e66c364f0 # 0.44.0
+    rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0
     hooks:
       - id: markdownlint
         types: [text]
@@ -38,7 +38,7 @@ repos:
   # If you do not, you will need to delete the cached ruff binary shown in the
   # error message
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 2c8dce6094fa2b4b668e74f694ca63ceffd38614 # 0.9.9
+    rev: 0b19ef1fd6ad680ed7752d6daba883ce1265a6de # 0.12.2
     hooks:
       # Run the linter.
       - id: ruff
diff --git a/README.adoc b/README.adoc
@@ -29,8 +29,7 @@ To remove files or directories that already exist in the target repositories the
 
 Anything that is listed here will be deleted from the target repositories.
 
-NOTE: Deletion is the last step that is performed, so if there is an overlap between files existing in the template folder and this setting, the files would not be rolled out, since they'd get deleted before creting the pull request.
-
+NOTE: Deletion is the last step that is performed, so if there is an overlap between files existing in the template folder and this setting, the files would not be rolled out, since they'd get deleted before creating the pull request.
 
 === Configuration
 
diff --git a/config/versions.yaml b/config/versions.yaml
@@ -2,7 +2,7 @@
 # IMPORTANT
 # If you change the Rust toolchain version here, make sure to also change
 # docker-images/ubi8-rust-builder/Dockerfile & docker-images/ubi9-rust-builder/Dockerfile
-rust_version: 1.85.0
+rust_version: 1.87.0
 
 # This nightly version is only used for cargo fmt invocations, because we use nightly-only
 # rustfmt config options in rustfmt.toml. The version should be kept in line with the version
@@ -22,11 +22,11 @@ hadolint_version: v2.12.0
 # IMPORTANT
 # If you change the Python version here, make sure to also change it in
 # .github/workflows/pr_pre-commit.yml
-python_version: 3.12
+python_version: 3.13
 
 # This Nix version is used in the operators pre-commit workflow as they require
 # Nix to run some checks. Check for new versions here:
 # https://github.com/NixOS/nix/tags
 # Usually the latest version should just work and as such the version here can
 # be bumped without any constraints.
-nix_pkg_manager_version: 2.28.3
+nix_pkg_manager_version: 2.30.0
diff --git a/template/.github/workflows/build.yml.j2 b/template/.github/workflows/build.yml.j2
@@ -42,18 +42,18 @@ jobs:
       RUSTC_BOOTSTRAP: 1
     steps:
       - name: Install host dependencies
-        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
+        uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1
         with:
           packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+      - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
-      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
         with:
           key: udeps
           cache-all-crates: "true"
@@ -114,7 +114,7 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - uses: EmbarkStudios/cargo-deny-action@34899fc7ba81ca6268d5947a7a16b4649013fea1 # v2.0.11
+      - uses: EmbarkStudios/cargo-deny-action@30f817c6f72275c6d54dc744fbca09ebc958599f # v2.0.12
         with:
           command: check ${{ matrix.checks }}
 
@@ -126,7 +126,7 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+      - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_NIGHTLY_TOOLCHAIN_VERSION }}
           components: rustfmt
@@ -139,19 +139,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install host dependencies
-        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
+        uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1
         with:
           packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+      - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: clippy
-      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
         with:
           key: clippy
           cache-all-crates: "true"
@@ -178,18 +178,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install host dependencies
-        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
+        uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1
         with:
           packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+      - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: rustfmt
-      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
         with:
           key: doc
           cache-all-crates: "true"
@@ -201,18 +201,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install host dependencies
-        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
+        uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1
         with:
           packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
           version: ubuntu-latest
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           submodules: recursive
-      - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+      - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
-      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
         with:
           key: test
           cache-all-crates: "true"
@@ -261,7 +261,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install host dependencies
-        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
+        uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1
         with:
           packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
           version: ubuntu-latest
@@ -274,10 +274,10 @@ jobs:
         with:
           version: v3.16.1
       - name: Set up cargo
-        uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+        uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
-      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0
         with:
           key: charts
           cache-all-crates: "true"
@@ -332,16 +332,16 @@ jobs:
       IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }}
     steps:
       - name: Install host dependencies
-        uses: awalsh128/cache-apt-pkgs-action@5902b33ae29014e6ca012c5d8025d4346556bd40 # v1.4.3
+        uses: awalsh128/cache-apt-pkgs-action@4c82c3ccdc1344ee11e9775dbdbdf43aa8a5614e # v1.5.1
         with:
           packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https
           version: ${{ matrix.runner }}
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           persist-credentials: false
           submodules: recursive
-      - uses: cachix/install-nix-action@17fe5fb4a23ad6cbbe47d6b3f359611ad276644c # v31.4.0
-      - uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0
+      - uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31.4.1
+      - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b
         with:
           toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }}
           components: rustfmt
@@ -376,9 +376,9 @@ jobs:
 
       # Recreate charts and publish charts and docker image.
       - name: Install cosign
-        uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
+        uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1
       - name: Install syft
-        uses: anchore/sbom-action/download-syft@e11c554f704a0b820cbf8c51673f6945e0731532 # v0.20.0
+        uses: anchore/sbom-action/download-syft@cee1b8e05ae5b2593a75e197229729eabaa9f8ec # v0.20.2
       - name: Build Docker image and Helm chart
         run: |
           # Installing helm and yq on ubicloud-standard-8-arm only
@@ -421,7 +421,7 @@ jobs:
       OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build"
     steps:
       - name: Install cosign
-        uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
+        uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
diff --git a/template/.github/workflows/integration-test.yml b/template/.github/workflows/integration-test.yml
@@ -86,7 +86,7 @@ jobs:
 
       - name: Run Integration Test
         id: test
-        uses: stackabletech/actions/run-integration-test@5901c3b1455488820c4be367531e07c3c3e82538 # v0.4.0
+        uses: stackabletech/actions/run-integration-test@4483641a7e24057bd2ba51cb4c3f2f0010ad21b7 # v0.8.4
         with:
           test-platform: ${{ env.TEST_PLATFORM }}-${{ env.TEST_ARCHITECTURE }}
           test-run: ${{ env.TEST_RUN }}
@@ -117,3 +117,21 @@ jobs:
                   }
                 ]
               }
+        # TODO: Update to version 2.1.0. This could look something like the following.
+        # The workflow is currently not in use, testing that the new version still works imposes effort.
+        # So I left it as a future exercise, but saved the current state.
+        #
+        # uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0
+        # with:
+        #   method: chat.postMessage
+        #   token:  ${{ secrets.SLACK_INTEGRATION_TEST_TOKEN }}
+        #   payload: |
+        #     channel: "C07UYJYSMSN" # notifications-integration-tests
+        #     text: "Integration Test *${{ github.repository }}* failed"
+        #     attachments:
+        #       - pretext: "Started at ${{ steps.test.outputs.start-time }}, failed at ${{ steps.test.outputs.end-time }}"
+        #         color: "#aa0000"
+        #         actions:
+        #           - type: button
+        #             text: Go to integration test run
+        #             url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
diff --git a/template/.pre-commit-config.yaml.j2 b/template/.pre-commit-config.yaml.j2
@@ -36,7 +36,7 @@ repos:
   # If you do not, you will need to delete the cached ruff binary shown in the
   # error message
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: d19233b89771be2d89273f163f5edc5a39bbc34a # 0.11.12
+    rev: 0b19ef1fd6ad680ed7752d6daba883ce1265a6de # 0.12.2
     hooks:
       # Run the linter.
       - id: ruff-check
diff --git a/template/.readme/partials/borrowed/footer.md.j2.j2 b/template/.readme/partials/borrowed/footer.md.j2.j2
@@ -36,6 +36,7 @@ These are the operators that are currently part of the Stackable Data Platform:
 * [Stackable Operator for Apache Hive](https://github.com/stackabletech/hive-operator)
 * [Stackable Operator for Apache Kafka](https://github.com/stackabletech/kafka-operator)
 * [Stackable Operator for Apache NiFi](https://github.com/stackabletech/nifi-operator)
+* [Stackable Operator for OpenSearch](https://github.com/stackabletech/opensearch-operator)
 * [Stackable Operator for Apache Spark](https://github.com/stackabletech/spark-k8s-operator)
 * [Stackable Operator for Apache Superset](https://github.com/stackabletech/superset-operator)
 * [Stackable Operator for Trino](https://github.com/stackabletech/trino-operator)
diff --git a/template/deploy/helm/[[operator]]/templates/deployment.yaml.j2 b/template/deploy/helm/[[operator]]/templates/deployment.yaml.j2
@@ -43,17 +43,38 @@ spec:
             - mountPath: /etc/stackable/{{ include "operator.appname" . }}/config-spec
               name: config-spec
           env:
+            # The following env vars are passed as clap (think CLI) arguments to the operator.
+            # They are picked up by clap using the structs defied in the operator.
+            # (which is turn pulls in https://github.com/stackabletech/operator-rs/blob/main/crates/stackable-operator/src/cli.rs)
+            # You can read there about the expected values and purposes.
+
+            # Sometimes products need to know the operator image, e.g. the opa-bundle-builder OPA
+            # sidecar uses the operator image.
             - name: OPERATOR_IMAGE
               # Tilt can use annotations as image paths, but not env variables
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.annotations['internal.stackable.tech/image']
+
+            # Namespace the operator Pod is running in, e.g. used to construct the conversion
+            # webhook endpoint.
             - name: OPERATOR_NAMESPACE
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+
+            # The name of the Kubernetes Service that point to the operator Pod, e.g. used to
+            # construct the conversion webhook endpoint.
             - name: OPERATOR_SERVICE_NAME
               value: {{ include "operator.fullname" . }}
+
+            # Operators need to know the node name they are running on, to e.g. discover the
+            # Kubernetes domain name from the kubelet API.
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+
             {{- if .Values.kubernetesClusterDomain }}
             - name: KUBERNETES_CLUSTER_DOMAIN
               value: {{ .Values.kubernetesClusterDomain | quote }}
diff --git a/template/docker/Dockerfile.j2 b/template/docker/Dockerfile.j2
@@ -26,6 +26,7 @@ ARG RELEASE="1"
 # These are chosen at random and are this high on purpose to have very little chance to clash with an existing user or group on the host system
 ARG STACKABLE_USER_GID="574654813"
 ARG STACKABLE_USER_UID="782252253"
+ARG STACKABLE_USER_NAME="stackable"
 
 # Sets the default shell to Bash with strict error handling and robust pipeline processing.
 # "-e": Exits immediately if a command exits with a non-zero status
@@ -95,6 +96,12 @@ RUN <<EOF
 # Update image and install kerberos client libraries as well as some other utilities
 microdnf update
 
+# **findutils**
+# Needed to find all patch files, used in `apply_patches.sh`, and helpful for debugging
+# Added 2024-10: Last vulnerability in 2007, only two vulnerabilities in total, a risk we accept
+# https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&isCpeNameSearch=false&cpe_vendor=cpe%3A%2F%3Agnu&cpe_product=cpe%3A%2F%3Agnu%3Afindutils
+# cpe:2.3:a:gnu:findutils:*:*:*:*:*:*:*:*
+#
 # **iputils**
 # To make debugging easier, includes things like ping
 # Added 2024-03: We cannot find any vulnerabilities in the past years
@@ -120,6 +127,7 @@ microdnf update
 # NOTE (@NickLarsenNZ): Maybe we should consider pinning package versions?
 # hadolint ignore=DL3041
 microdnf install \
+  findutils \
   iputils \
   krb5-libs \
   less \
@@ -128,7 +136,7 @@ microdnf install \
   shadow-utils \
   tar
 
-groupadd --gid ${STACKABLE_USER_GID} --system stackable
+groupadd --gid ${STACKABLE_USER_GID} --system ${STACKABLE_USER_NAME}
 # The --no-log-init is required to work around a bug/problem in Go/Docker when very large UIDs are used
 # See https://github.com/moby/moby/issues/5419#issuecomment-41478290 for more context
 # Making this a system user prevents a mail dir from being created, expiry of passwords etc. but it will warn:
@@ -142,7 +150,7 @@ useradd \
   --system \
   --create-home \
   --home-dir /stackable \
-   stackable
+   ${STACKABLE_USER_NAME}
 microdnf remove shadow-utils
 microdnf clean all
 rm -rf /var/cache/yum
diff --git a/template/nix/sources.json b/template/nix/sources.json
diff --git a/template/scripts/auto-retry-tests.py b/template/scripts/auto-retry-tests.py
diff --git a/template/scripts/run-tests b/template/scripts/run-tests
