Pointed dnf repos back at ark for now + refactor

Author: wtripp180901

ansible/adhoc/sync-pulp.yml

@@ -8,4 +8,3 @@
         pulp_site_target_distribution: "rocky"
         pulp_site_target_distribution_version: "9.4"
         pulp_site_target_distribution_version_major: "9"
-        pulp_site_version_timestamps: "{{ appliances_repo_timestamps[pulp_site_target_distribution_version] }}"

ansible/fatimage.yml

@@ -25,7 +25,7 @@
       tasks_from: sync.yml
       apply:
         delegate_to: localhost
-    when: appliances_mode != 'configure' and appliances_sync_pulp_on_build
+    when: appliances_mode != 'configure'
 
 - hosts: dnf_repos
   become: yes

ansible/roles/dnf_repos/defaults/main.yml

@@ -2,23 +2,24 @@ dnf_repos_pulp_url: "{{ appliances_pulp_url }}"
 dnf_repos_pulp_content_url: "{{ dnf_repos_pulp_url }}/pulp/content"
 dnf_repos_rocky_prefix: "{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}"
 dnf_repos_epel_prefix: "epel/{{ ansible_distribution_major_version }}"
-dnf_repos_disable: true
-dnf_repos_version_timestamps: "{{ appliances_repo_timestamps[ansible_distribution_version] }}"
+dnf_repos_version_timestamps: "{{ appliances_repo_minor_timestamps[ansible_distribution_version] }}"
+dnf_repos_username: "{{ omit }}"
+dnf_repos_password: "{{ omit }}"
 
 # epel installed separately
 dnf_repos_repolist:
 - file: rocky
   name: baseos
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/baseos/{{ dnf_repos_version_timestamps.baseos }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/BaseOS/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].baseos }}"
 - file: rocky
   name: appstream
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/appstream/{{ dnf_repos_version_timestamps.appstream }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/AppStream/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].appstream }}"
 - file: rocky
   name: crb
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/crb/{{ dnf_repos_version_timestamps.crb }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/CRB/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].crb }}"
 - file: rocky-extras
   name: extras
-  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ dnf_repos_version_timestamps.extras }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ ansible_architecture }}/os/{{ appliances_repo_minor_timestamps[ansible_distribution_version].extras }}"
 
-dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_version }}/{{ dnf_repos_version_timestamps.epel }}"
+dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ appliances_repo_major_timestamps[ansible_distribution_major_version].epel }}"
 dnf_repos_epel_description: "epel"

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -6,6 +6,8 @@
     name: "{{ item.name }}"
     baseurl: "{{ item.base_url }}"
     description: "{{ item.name }}"
+    username: "{{ dnf_repos_username }}"
+    password: "{{ dnf_repos_password }}"
   loop: "{{ dnf_repos_repolist }}"
 
 - name: Install epel-release
@@ -20,3 +22,5 @@
     description: "{{ dnf_repos_epel_description }}"
     gpgcheck: false
     baseurl: "{{ dnf_repos_epel_baseurl }}"
+    username: "{{ dnf_repos_username }}"
+    password: "{{ dnf_repos_password }}"

ansible/roles/pulp_site/defaults/main.yml

@@ -14,7 +14,8 @@ pulp_site_target_arch: "{{ pulp_site_target_facts['architecture'] }}"
 pulp_site_target_distribution: "{{ pulp_site_target_facts['distribution'] | lower }}"
 pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}"
 pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}"
-pulp_site_version_timestamps: "{{ appliances_repo_timestamps[pulp_site_target_distribution_version] }}"
+pulp_site_version_timestamps: "{{ appliances_repo_minor_timestamps[pulp_site_target_distribution_version] }}"
+pulp_site_major_version_timestamps: "{{ appliances_repo_major_timestamps[pulp_site_target_distribution_version_major] }}"
 
 pulp_site_rpm_info:
 - name: "baseos-{{ pulp_site_target_distribution_version }}-{{ pulp_site_version_timestamps.baseos }}"
@@ -30,7 +31,7 @@ pulp_site_rpm_info:
   url: "{{ pulp_site_default_upstream_prefix }}/extras/{{ pulp_site_default_upstream_suffix }}/{{ pulp_site_version_timestamps.extras }}"
   base_path: "rocky/{{ pulp_site_target_distribution_version }}/extras/{{ pulp_site_version_timestamps.extras }}"
 - name: "epel-{{ pulp_site_target_distribution_version }}-{{ pulp_site_version_timestamps.epel }}"
-  url: "https://ark.stackhpc.com/pulp/content/epel/{{ pulp_site_target_distribution_version_major }}/Everything/{{ pulp_site_target_arch }}/{{ pulp_site_version_timestamps.epel }}"
+  url: "https://ark.stackhpc.com/pulp/content/epel/{{ pulp_site_target_distribution_version_major }}/Everything/{{ pulp_site_target_arch }}/{{ pulp_site_major_version_timestamps.epel }}"
   base_path: "epel/{{ pulp_site_target_distribution_version }}/{{ pulp_site_version_timestamps.epel }}"
 
 pulp_site_rpm_repo_defaults:

docs/experimental/pulp.md

@@ -14,4 +14,4 @@ An existing Pulp server can be used to host Ark repos by overriding `pulp_site_p
 
 ## Syncing Pulp content with Ark
 
-By default, the appliance will sync repos for the targetted distribution during build (can be disabled by setting `appliances_sync_pulp_on_build` to `false`). You must supply your Ark credentials, either by overriding `pulp_site_upstream_password` or setting environment variable `ARK_PASSWORD`. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`, optionally setting extravars for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version`.
+If the `pulp` group is added to the Packer build groups, the local Pulp server will be synced with Ark on build. You must supply your Ark credentials, either by overriding `pulp_site_upstream_password` or setting environment variable `ARK_PASSWORD`. Content can also be synced by running `ansible/adhoc/sync-pulp.yml`, optionally setting extravars for `pulp_site_target_arch`, `pulp_site_target_distribution`, `pulp_site_target_distribution_version` and `pulp_site_target_distribution_version`.

environments/.stackhpc/inventory/group_vars/builder.yml

@@ -1,9 +1,14 @@
 # update_enable: false # Can uncomment for speed debugging non-update related build issues
-pulp_server_config:
-  LEAFCLOUD:
-    url: http://192.168.10.157:8080
-    password: lookup('env','LEAFCLOUD_PULP_PASSWORD')
-
-appliances_pulp_url: "{{ pulp_server_config[lookup('env','CI_CLOUD')].url }}"
-pulp_site_password: "{{ pulp_server_config[lookup('env','CI_CLOUD')].password }}"
-appliances_sync_pulp_on_build: false
+
+# Uncomment below to use CI pulp servers
+
+# pulp_server_config:
+#   LEAFCLOUD:
+#     url: http://192.168.10.157:8080
+#     password: lookup('env','LEAFCLOUD_PULP_PASSWORD')
+
+# appliances_pulp_url: "{{ pulp_server_config[lookup('env','CI_CLOUD')].url }}"
+# pulp_site_password: "{{ pulp_server_config[lookup('env','CI_CLOUD')].password }}"
+
+dnf_repos_username: slurm-app-ci
+dnf_repos_password: "{{ lookup('env','ARK_PASSWORD') }}"

environments/common/inventory/group_vars/all/defaults.yml

@@ -7,7 +7,7 @@ appliances_environment_name: "{{ appliances_environment_root | basename | regex_
 appliances_cockpit_state: absent # RHEL cockpit installed but not enabled in genericcloud images; appliance defaults to removing it
 #appliances_state_dir: # define an absolute path here to use for persistent state: NB: This is defined as /var/lib/state in inventory by the default Terraform
 appliances_mode: configure
-#appliances_pulp_url: #override required
+appliances_pulp_url: https://ark.stackhpc.com
 
 # Address(ip/dns) for internal communication between services. This is
 # normally traffic you do no want to expose to users.
@@ -82,11 +82,13 @@ appliances_local_users: "{{ appliances_local_users_default + appliances_local_us
 
 ###########################################################################################
 
-appliances_sync_pulp_on_build: true
-appliances_repo_timestamps:
+appliances_repo_minor_timestamps:
   '9.4':
     baseos: 20240816T002610
     appstream: 20240816T002610
     crb: 20240816T002610
     extras: 20240816T002610
+
+appliances_repo_major_timestamps:
+  '9':
     epel: 20240902T080424

environments/common/inventory/groups

@@ -147,9 +147,7 @@ freeipa_client
 
 [dnf_repos:children]
 # Hosts to replace system repos with Pulp repos
-cluster
 builder
 
 [pulp:children]
 # Hosts used to run Pulp API commands
-builder