Make uid/gid configurable & change group of files - part 2 (#890)

* Make uid/gid configurable & change group of files

This is a follow-up for #849 and includes:

- The missing bits for Hive
- Kafka

* More tools now migrated but not tested yet:

- Kafka Testing Tools
- KCat
- NiFi
- Omid

* - OPA
- Spark (WIP)

* Adds Spark and a changelog entry

* Update CHANGELOG.md

Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com>

* Update comment

---------

Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com>

Author: lfrancke

CHANGELOG.md

@@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file.
 - Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]).
 - java: migrate to temurin jdk/jre ([#894]).
 - tools: bump kubectl to `1.31.1` and jq to `1.7.1` ([#896]).
+- Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890]).
 
 ### Removed
 
@@ -66,6 +67,7 @@ All notable changes to this project will be documented in this file.
 [#822]: https://github.com/stackabletech/docker-images/pull/822
 [#846]: https://github.com/stackabletech/docker-images/pull/846
 [#848]: https://github.com/stackabletech/docker-images/pull/848
+[#849]: https://github.com/stackabletech/docker-images/pull/849
 [#851]: https://github.com/stackabletech/docker-images/pull/851
 [#852]: https://github.com/stackabletech/docker-images/pull/852
 [#853]: https://github.com/stackabletech/docker-images/pull/853
@@ -80,6 +82,7 @@ All notable changes to this project will be documented in this file.
 [#880]: https://github.com/stackabletech/docker-images/pull/880
 [#881]: https://github.com/stackabletech/docker-images/pull/881
 [#882]: https://github.com/stackabletech/docker-images/pull/882
+[#890]: https://github.com/stackabletech/docker-images/pull/890
 [#894]: https://github.com/stackabletech/docker-images/pull/894
 [#896]: https://github.com/stackabletech/docker-images/pull/896
 

druid/Dockerfile

@@ -120,8 +120,8 @@ ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid
 # Force to overwrite the existing 'run-druid'
 ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hadoop/Dockerfile

@@ -169,8 +169,8 @@ find . -name 'hadoop-*tests.jar' -type f -delete
 # It is so non-root users (as we are) can mount a FUSE device and let other users access it
 echo "user_allow_other" > /etc/fuse.conf
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hbase/Dockerfile

@@ -354,8 +354,8 @@ ln --symbolic --logical --verbose "/stackable/hbase-${PRODUCT}" /stackable/hbase
 ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}" /stackable/hbase-operator-tools
 ln --symbolic --logical --verbose "/stackable/phoenix/phoenix-server-hbase-${HBASE_PROFILE}.jar" "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar"
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hello-world/Dockerfile

@@ -22,8 +22,8 @@ rm -rf /var/cache/yum
 
 curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o /stackable/hello-world.jar
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF

hive/Dockerfile

@@ -103,40 +103,47 @@ LABEL io.openshift.tags="ubi9,stackable,hive,sdp"
 LABEL io.k8s.description="${DESCRIPTION}"
 LABEL io.k8s.display-name="${NAME}"
 
-RUN <<EOF
-microdnf update
-microdnf clean all
-rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
-rm -rf /var/cache/yum
-EOF
-
-USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin
-RUN ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore
 
 # It is useful to see which version of Hadoop is used at a glance
 # Therefore the use of the full name here
 # TODO: Do we really need all of Hadoop in here?
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP}
-RUN ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop
+
+RUN <<EOF
+microdnf update
+microdnf clean all
+rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
+rm -rf /var/cache/yum
+
+ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore
+ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop
 
 # The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards
 # This way the build will fail should one of the files not be available anymore in a later Hadoop version!
 
 # Add S3 Support for Hive (support for s3a://)
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar /stackable/hive-metastore/lib/
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/hive-metastore/lib/
 
 # Add Azure ABFS support (support for abfs://)
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackable/hive-metastore/lib/
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/
-RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/
+cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/
+
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx
 COPY hive/licenses /licenses
 
+USER ${STACKABLE_USER_UID}
+
 ENV HADOOP_HOME=/stackable/hadoop
 ENV HIVE_HOME=/stackable/hive-metastore
 ENV PATH="${PATH}":/stackable/hadoop/bin:/stackable/hive-metastore/bin

kafka-testing-tools/Dockerfile

@@ -8,6 +8,7 @@ FROM stackable/image/stackable-base AS final
 ARG PRODUCT
 ARG KCAT
 ARG RELEASE
+ARG STACKABLE_USER_UID
 
 LABEL name="Kafka Testing Tools" \
       maintainer="info@stackable.tech" \
@@ -29,11 +30,10 @@ RUN microdnf install  \
     && rm -rf /var/cache/yum
 
 # Store kcat version with binary name and add softlink
-COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT}
 RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat
-COPY --chown=stackable:stackable --from=kcat /licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
 
-
-COPY --chown=stackable:stackable kafka-testing-tools/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses
 
 ENTRYPOINT ["/stackable/kcat"]

kafka/Dockerfile

@@ -9,8 +9,9 @@ ARG PRODUCT
 ARG SCALA
 ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
+ARG STACKABLE_USER_UID
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \
@@ -27,35 +28,20 @@ RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}
 RUN curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
     -o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
 
-COPY --chown=stackable:stackable kafka/stackable/jmx/ /stackable/jmx/
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/
 RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
     -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \
     ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar
 
-# For earlier versions this script removes the .class file that contains the
-# vulnerable code.
-# TODO: This can be restricted to target only versions which do not honor the environment
-#   varible that has been set above but this has not currently been implemented
-COPY shared/log4shell.sh /bin
-RUN /bin/log4shell.sh /stackable/kafka_${SCALA}-${PRODUCT}
-
-# Ensure no vulnerable files are left over
-# This will currently report vulnerable files being present, as it also alerts on
-# SocketNode.class, which we do not remove with our scripts.
-# Further investigation will be needed whether this should also be removed.
-COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
-COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
-COPY shared/log4shell_scanner /bin/log4shell_scanner
-RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT}
-# ===
 
 FROM stackable/image/java-base AS final
 
 ARG RELEASE
 ARG PRODUCT
 ARG SCALA
 ARG KCAT
+ARG STACKABLE_USER_UID
 
 LABEL name="Apache Kafka" \
       maintainer="info@stackable.tech" \
@@ -67,32 +53,38 @@ LABEL name="Apache Kafka" \
 
 # This is needed for kubectl
 COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo
-RUN microdnf update && \
-    microdnf install \
-    # needed by kcat for kerberos
-    cyrus-sasl-gssapi \
-    # Can be removed once listener-operator integration is used
-    kubectl && \
-    microdnf clean all && \
-    rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt && \
-    rm -rf /var/cache/yum
-
-USER stackable
-WORKDIR /stackable
-
-COPY --chown=stackable:stackable kafka/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses
 
-# We copy opa-authorizer.jar and jmx-exporter through the builder image to have an absolutely minimal final image
-# (e.g. we don't even need curl in it).
-COPY --chown=stackable:stackable --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
-COPY --chown=stackable:stackable --from=kafka-builder /stackable/jmx/ /stackable/jmx/
-COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT}
-COPY --chown=stackable:stackable --from=kcat /licenses /licenses
+WORKDIR /stackable
 
-RUN ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat && \
-    # kcat was located in /stackable/kcat - legacy
-    ln -s /stackable/bin/kcat /stackable/kcat && \
-    ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
+RUN <<EOF
+microdnf update
+# cyrus-sasl-gssapi: needed by kcat for kerberos
+# kubectl: Can be removed once listener-operator integration is used
+microdnf install \
+  cyrus-sasl-gssapi \
+  kubectl
+
+microdnf clean all
+rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
+rm -rf /var/cache/yum
+
+ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat
+# kcat was located in /stackable/kcat - legacy
+ln -s /stackable/bin/kcat /stackable/kcat
+ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
+
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
+
+USER ${STACKABLE_USER_UID}
 
 ENV PATH="${PATH}:/stackable/bin:/stackable/kafka/bin"
 

kcat/Dockerfile

@@ -7,6 +7,7 @@
 FROM stackable/image/java-base AS builder
 
 ARG PRODUCT
+ARG STACKABLE_USER_UID
 
 RUN microdnf update \
     && microdnf install \
@@ -32,7 +33,7 @@ RUN curl -O https://repo.stackable.tech/repository/packages/kcat/kcat-${PRODUCT}
     && cd kcat-${PRODUCT} \
     && ./bootstrap.sh
 
-COPY --chown=stackable:stackable kcat/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 kcat/licenses /licenses
 
 # SNIPPET 1
 # 145.2 gcc  -I/stackable/kcat-1.7.0/tmp-bootstrap/usr/include -I/stackable/kcat-1.7.0/tmp-bootstrap/usr/include -g -O2 -Wall -Wsign-compare -Wfloat-equal -Wpointer-arith -Wcast-align -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -Wl,-rpath-link=/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -Wl,-rpath-link=/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib kcat.o format.o tools.o input.o json.o avro.o -o kcat  -lm -ldl -lpthread -lrt -lpthread -lrt  -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib  /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libavro.a /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libjansson.a -lcurl /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libserdes.a -Wl,-Bstatic -lavro -Wl,-Bdynamic /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libyajl_s.a -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib //stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/librdkafka.a -lm -ldl -lpthread -lrt -lz -lcrypto -lssl -lsasl2   -lm -ldl -lpthread -lrt -lpthread -lrt  -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib  /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libavro.a /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libjansson.a -lcurl

nifi/Dockerfile

@@ -5,6 +5,7 @@ FROM stackable/image/java-devel AS nifi-builder
 
 ARG PRODUCT
 ARG MAVEN_VERSION="3.9.8"
+ARG STACKABLE_USER_UID
 
 RUN microdnf update && \
     microdnf clean all && \
@@ -22,10 +23,10 @@ RUN if [[ "${PRODUCT}" == 2.* ]] ; then \
         ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \
     fi
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --chown=stackable:stackable nifi/stackable/patches /stackable/patches
+COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches /stackable/patches
 
 # NOTE: NiFi 1.21.0 source build does not work with the current arm64 git runners due to java heap issues:
 #
@@ -82,28 +83,11 @@ RUN if [[ "${PRODUCT}" == "1.21.0" ]] ; then \
         rm -rf /stackable/nifi-${PRODUCT}/docs ; \
     fi
 
-# ===
-# For earlier versions this script removes the .class file that contains the
-# vulnerable code.
-# TODO: This can be restricted to target only versions which do not honor the environment
-#   varible that has been set above but this has not currently been implemented
-COPY shared/log4shell.sh /bin
-RUN /bin/log4shell.sh /stackable/nifi-${PRODUCT}
-
-# Ensure no vulnerable files are left over
-# This will currently report vulnerable files being present, as it also alerts on
-# SocketNode.class, which we do not remove with our scripts.
-# Further investigation will be needed whether this should also be removed.
-COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
-COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
-COPY shared/log4shell_scanner /bin/log4shell_scanner
-RUN /bin/log4shell_scanner s /stackable/nifi-${PRODUCT}
-# ===
-
 FROM stackable/image/java-base AS final
 
 ARG PRODUCT
 ARG RELEASE
+ARG STACKABLE_USER_UID
 
 LABEL name="Apache NiFi" \
       maintainer="info@stackable.tech" \
@@ -113,28 +97,39 @@ LABEL name="Apache NiFi" \
       summary="The Stackable image for Apache NiFi." \
       description="This image is deployed by the Stackable Operator for Apache NiFi."
 
-RUN microdnf update && \
-    microdnf install \
-    # Required to install nipyapi
-    python-pip && \
-    microdnf clean all && \
-    rm -rf /var/cache/yum && \
-    # The nipyapi is required for the ReportingTaskJob
-    pip install --no-cache-dir nipyapi==0.19.1 && \
-    # For backwards compatibility we create a softlink in /bin where the jar used to be as long as we are root
-    # This can be removed once older versions / operators using this are no longer supported
-    ln -s /stackable/stackable-bcrypt.jar /bin/stackable-bcrypt.jar
+COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar
+
+COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/bin /stackable/bin
+COPY --chown=${STACKABLE_USER_UID}:0 nifi/licenses /licenses
+COPY --chown=${STACKABLE_USER_UID}:0 nifi/python /stackable/python
+
+RUN <<EOF
+ln -s /stackable/nifi-${PRODUCT} /stackable/nifi
+
+microdnf update
+
+# python-pip: Required to install nipyapi
+microdnf install \
+  python-pip
+
+microdnf clean all
+rm -rf /var/cache/yum
 
-USER stackable
+# The nipyapi is required for the ReportingTaskJob
+pip install --no-cache-dir nipyapi==0.19.1 && \
 
-COPY --chown=stackable:stackable --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/
-COPY --chown=stackable:stackable --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar
+# For backwards compatibility we create a softlink in /bin where the jar used to be as long as we are root
+# This can be removed once older versions / operators using this are no longer supported
+ln -s /stackable/stackable-bcrypt.jar /bin/stackable-bcrypt.jar
 
-COPY --chown=stackable:stackable nifi/stackable/bin /stackable/bin
-COPY --chown=stackable:stackable nifi/licenses /licenses
-COPY --chown=stackable:stackable nifi/python /stackable/python
+# All files and folders owned by root group to support running as arbitrary users.
+# This is best practice as all container users will belong to the root group (0).
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
 
-RUN ln -s /stackable/nifi-${PRODUCT} /stackable/nifi
+USER ${STACKABLE_USER_UID}
 
 ENV HOME=/stackable
 ENV NIFI_HOME=/stackable/nifi