Merge pull request #937 from stackhpc/yoga-secure-container-builds

CI: Avoid leaking credentials in container image builds

Author: markgoddard

.github/workflows/stackhpc-all-in-one.yml

@@ -86,9 +86,9 @@ jobs:
         with:
            submodules: true
 
-      - name: Make sure dockerd is running and test Docker.
+      - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Output image tag
         id: image_tag

.github/workflows/stackhpc-container-image-build.yml

@@ -125,7 +125,7 @@ jobs:
 
       - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Install Kayobe
         run: |
@@ -136,10 +136,11 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      # Required for Docker registry login. Normally installed during host configure.
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          pip install --user docker
+          sudo pip install docker
 
       - name: Configure localhost as a seed
         run: |
@@ -150,11 +151,23 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
       - name: Build and push kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -169,6 +182,7 @@ jobs:
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi

.gitignore

@@ -58,3 +58,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring
 
 # Tempest logs
 tempest-artifacts
+
+# Ansible Galaxy roles & collections
+etc/kayobe/ansible/roles/*\.*/
+etc/kayobe/ansible/collections/

doc/source/contributor/environments/ci-builder.rst

@@ -95,6 +95,34 @@ Next, configure the host OS & services.
 
    kayobe seed host configure
 
+.. _authenticating-pulp-proxy:
+
+Authenticating Pulp proxy
+-------------------------
+
+If you are building against authenticated package repositories such as those in
+`Ark <https://ark.stackhpc.com>`_, you will need to provide secure access to
+the repositories without leaking credentials into the built images or their
+metadata.  This is typically not the case for a client-local Pulp, which
+provides unauthenticated read-only access to the repositories on a trusted
+network.
+
+Docker provides `build
+secrets <https://docs.docker.com/build/building/secrets/>`_, but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+StackHPC Kayobe Configuration provides support for deploying an authenticating
+Pulp proxy that injects an HTTP basic auth header into requests that it
+proxies. Because this proxy bypasses Pulp's authentication, it must not be
+exposed to any untrusted environment.
+
+To deploy the proxy:
+
+.. parsed-literal::
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml
+
 Building images
 ===============
 
@@ -105,6 +133,9 @@ At this point you are ready to build and push some container images.
    kayobe seed container image build --push
    kayobe overcloud container image build --push
 
+If using an :ref:`authenticating Pulp proxy <authenticating-pulp-proxy>`,
+append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands.
+
 The container images are tagged as |current_release|-<datetime>.
 
 To use the new images, edit

etc/kayobe/ansible/pulp-auth-proxy.yml

@@ -0,0 +1,14 @@
+---
+# See roles/pulp_auth_proxy/README.md for details.
+
+- name: Deploy Pulp auth proxy
+  hosts: container-image-builders
+  gather_facts: false
+  tasks:
+    - import_role:
+        name: pulp_auth_proxy
+      vars:
+        pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}"
+        pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}"
+        pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}"
+        pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy"

etc/kayobe/ansible/roles/pulp_auth_proxy/README.md

@@ -0,0 +1,26 @@
+# Pulp Auth Proxy
+
+There is currently no practical, secure way to provide credentials for
+accessing Ark's authenticated package repositories from within a Kolla build.
+Docker provides [build
+secrets](https://docs.docker.com/build/building/secrets/), but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+This role deploys an Nginx container that runs as a reverse proxy, injecting an
+HTTP basic authentication header into requests.
+
+Because this proxy bypasses Pulp's authentication, it must not be exposed to
+any untrusted environment.
+
+## Role variables
+
+* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx
+  configuration.
+* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to
+  listen. Default is `127.0.0.1`.
+* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen.
+  Default is 80.

etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+pulp_auth_proxy_url:
+pulp_auth_proxy_username:
+pulp_auth_proxy_password:
+pulp_auth_proxy_conf_path:
+pulp_auth_proxy_listen_ip: 127.0.0.1
+pulp_auth_proxy_listen_port: 80

etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists"
+  ansible.builtin.file:
+    path: "{{ pulp_auth_proxy_conf_path }}"
+    state: directory
+    mode: 0700
+  become: true
+
+- name: Ensure pulp_proxy.conf is templated
+  ansible.builtin.template:
+    src: pulp_proxy.conf.j2
+    dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf"
+    mode: 0600
+  become: true
+  register: pulp_proxy_conf
+
+- name: Ensure pulp_proxy container is running
+  community.docker.docker_container:
+    name: pulp_proxy
+    image: nginx:stable-alpine
+    ports:
+      - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80"
+    restart_policy: "no"
+    restart: "{{ pulp_proxy_conf is changed }}"
+    volumes:
+      - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro"

etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2

@@ -0,0 +1,17 @@
+server {
+    listen {{ pulp_auth_proxy_listen_port }};
+    server_name pulp_proxy;
+    location / {
+        proxy_pass {{ pulp_auth_proxy_url }};
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }};
+        # The important part: add basic auth header
+        proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}";
+        proxy_pass_header Authorization;
+        # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027
+        proxy_ssl_server_name on;
+        proxy_ssl_protocols TLSv1.2;
+    }
+}

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -43,7 +43,7 @@ resolv_is_managed: false
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
-stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"