From 798731f3e5f56f292290f4e156a6d4bdb386a117 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Fri, 27 Jun 2025 21:17:39 -0700 Subject: [PATCH 1/5] (PA-7586) Add puppetcore support for macOS Install puppetcore packages on macOS via SSH: ``` /opt/puppetlabs/bolt/bin/bolt task run puppet_agent::install \ collection=puppetcore8 \ version=8.13.1 \ username=forge-key \ password=${PUPPET_FORGE_TOKEN} \ --targets www.example.com ``` --- tasks/install_shell.sh | 49 +++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tasks/install_shell.sh b/tasks/install_shell.sh index f5c64a8e..6cabc51d 100644 --- a/tasks/install_shell.sh +++ b/tasks/install_shell.sh @@ -162,10 +162,18 @@ fi if [ -n "$PT_mac_source" ]; then mac_source=$PT_mac_source else - if [ "$nightly" = true ]; then - mac_source='http://nightlies.puppet.com/downloads' + if [[ "$collection" == "puppetcore"* ]]; then + mac_source='https://artifacts-puppetcore.puppet.com/v1/download' + if [ -z "$password" ]; then + echo "A password parameter is required to install from ${mac_source}" + exit 1 + fi else - mac_source='http://downloads.puppet.com' + if [ "$nightly" = true ]; then + mac_source='http://nightlies.puppet.com/downloads' + else + mac_source='http://downloads.puppet.com' + fi fi fi @@ -396,10 +404,14 @@ run_cmd() { return $rc } -# do_wget URL FILENAME +# do_wget URL FILENAME [USERNAME] [PASSWORD] do_wget() { info "Trying wget..." - run_cmd "wget -O '$2' '$1' 2>$tmp_stderr" + if [[ -n "$3" && -n "$4" ]]; then + run_cmd "wget -O '$2' --user '$3' --password '$4' '$1' 2>$tmp_stderr" + else + run_cmd "wget -O '$2' '$1' 2>$tmp_stderr" + fi rc=$? # check for 404 @@ -418,10 +430,14 @@ do_wget() { return 0 } -# do_curl URL FILENAME +# do_curl URL FILENAME [USERNAME] [PASSWORD] do_curl() { info "Trying curl..." - run_cmd "curl -1 -sL -D $tmp_stderr '$1' > '$2'" + if [[ -n "$3" && -n "$4" ]]; then + run_cmd "curl -1 -sL -u'$3:$4' -D $tmp_stderr '$1' > '$2'" + else + run_cmd "curl -1 -sL -D $tmp_stderr '$1' > '$2'" + fi rc=$? # check for 404 @@ -544,7 +560,7 @@ do_perl_ff() { return 1 } -# do_download URL FILENAME +# do_download URL FILENAME [USERNAME] [PASSWORD] do_download() { info "Downloading $1" info " to file $2" @@ -553,11 +569,11 @@ do_download() { # perl, in particular may be present but LWP::Simple may not be installed if exists wget; then - do_wget $1 $2 && return 0 + do_wget $1 $2 $3 $4 && return 0 fi if exists curl; then - do_curl $1 $2 && return 0 + do_curl $1 $2 $3 $4 && return 0 fi if exists fetch; then @@ -822,7 +838,16 @@ case $platform in if [[ $(uname -p) == "arm" ]]; then arch="arm64" fi - download_url="${mac_source}/mac/${collection}/${platform_version}/${arch}/${filename}" + if [[ "$collection" =~ "puppetcore" ]]; then + dots=$(echo "${version}" | grep -o '\.' | wc -l) + if (( dots >= 3 )); then + download_url="${mac_source}?version=${version}&os_name=osx&os_version=${platform_version}&os_arch=${arch}&dev=true" + else + download_url="${mac_source}?version=${version}&os_name=osx&os_version=${platform_version}&os_arch=${arch}" + fi + else + download_url="${mac_source}/mac/${collection}/${platform_version}/${arch}/${filename}" + fi ;; *) critical "Sorry $platform is not supported yet!" @@ -837,7 +862,7 @@ fi if [[ $PT__noop != true ]]; then download_filename="${tmp_dir}/${filename}" - do_download "$download_url" "$download_filename" + do_download "$download_url" "$download_filename" "$username" "$password" install_file $filetype "$download_filename" From 224bb069c910f47334a8cac6144d90fc983226da Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Fri, 27 Jun 2025 21:27:00 -0700 Subject: [PATCH 2/5] (PA-7586) Move puppetcore password check If yum_source wasn't specified, such as when installing puppetcore on apt or mac, and a password wasn't specified, the error message incorrectly referenced yum: A password parameter is required to install from https://yum-puppetcore.puppet.com/public Move the password check prior to handling yum, apt, etc sources. If installing puppetcore packages and password isn't specified, we now report: A password parameter is required to install from puppetcore --- tasks/install_shell.sh | 59 ++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 39 deletions(-) diff --git a/tasks/install_shell.sh b/tasks/install_shell.sh index 6cabc51d..44e9300c 100644 --- a/tasks/install_shell.sh +++ b/tasks/install_shell.sh @@ -123,58 +123,39 @@ else collection='puppet' fi +if [[ "$collection" == "puppetcore"* && -z "$password" ]]; then + echo "A password parameter is required to install from puppetcore" + exit 1 +fi + if [ -n "$PT_yum_source" ]; then yum_source=$PT_yum_source +elif [[ "$collection" == "puppetcore"* ]]; then + yum_source='https://yum-puppetcore.puppet.com/public' +elif [ "$nightly" = true ]; then + yum_source='http://nightlies.puppet.com/yum' else - if [[ "$collection" == "puppetcore"* ]]; then - yum_source='https://yum-puppetcore.puppet.com/public' - if [ -z "$password" ]; then - echo "A password parameter is required to install from ${yum_source}" - exit 1 - fi - else - if [ "$nightly" = true ]; then - yum_source='http://nightlies.puppet.com/yum' - else - yum_source='http://yum.puppet.com' - fi - fi + yum_source='http://yum.puppet.com' fi if [ -n "$PT_apt_source" ]; then apt_source=$PT_apt_source +elif [[ "$collection" == "puppetcore"* ]]; then + apt_source='https://apt-puppetcore.puppet.com/public' +elif [ "$nightly" = true ]; then + apt_source='http://nightlies.puppet.com/apt' else - if [[ "$collection" == "puppetcore"* ]]; then - apt_source='https://apt-puppetcore.puppet.com/public' - if [ -z "$password" ]; then - echo "A password parameter is required to install from ${apt_source}" - exit 1 - fi - else - if [ "$nightly" = true ]; then - apt_source='http://nightlies.puppet.com/apt' - else - apt_source='http://apt.puppet.com' - fi - fi + apt_source='http://apt.puppet.com' fi if [ -n "$PT_mac_source" ]; then mac_source=$PT_mac_source +elif [[ "$collection" == "puppetcore"* ]]; then + mac_source='https://artifacts-puppetcore.puppet.com/v1/download' +elif [ "$nightly" = true ]; then + mac_source='http://nightlies.puppet.com/downloads' else - if [[ "$collection" == "puppetcore"* ]]; then - mac_source='https://artifacts-puppetcore.puppet.com/v1/download' - if [ -z "$password" ]; then - echo "A password parameter is required to install from ${mac_source}" - exit 1 - fi - else - if [ "$nightly" = true ]; then - mac_source='http://nightlies.puppet.com/downloads' - else - mac_source='http://downloads.puppet.com' - fi - fi + mac_source='http://downloads.puppet.com' fi if [ -n "$PT_retry" ]; then From 061b17219a48a04cde39407113be6265814a34d9 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Fri, 27 Jun 2025 21:36:45 -0700 Subject: [PATCH 3/5] (PA-7586) Handle failed authentication Since wget and curl may attempt to use credentials, report if authentication fails. --- tasks/install_shell.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tasks/install_shell.sh b/tasks/install_shell.sh index 44e9300c..05aba5ed 100644 --- a/tasks/install_shell.sh +++ b/tasks/install_shell.sh @@ -402,6 +402,13 @@ do_wget() { unable_to_retrieve_package fi + # check for 401 + grep "ERROR 401" $tmp_stderr 2>&1 >/dev/null + if test $? -eq 0; then + critical "ERROR 401" + unable_to_retrieve_package + fi + # check for bad return status or empty output if test $rc -ne 0 || test ! -s "$2"; then capture_tmp_stderr "wget" @@ -428,6 +435,13 @@ do_curl() { unable_to_retrieve_package fi + # check for 401 + grep "401 Unauthorized" $tmp_stderr 2>&1 >/dev/null + if test $? -eq 0; then + critical "ERROR 401" + unable_to_retrieve_package + fi + # check for bad return status or empty output if test $rc -ne 0 || test ! -s "$2"; then capture_tmp_stderr "curl" From f7cd3cd5f9d2e1661bed7c6ad87c8813e38298eb Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Fri, 27 Jun 2025 21:50:06 -0700 Subject: [PATCH 4/5] (PA-7586) Don't assume curl uses HTTP/1.1 The script assumed curl was using HTTP/1.1 as it was tried to match: 404 Not Found When using 2.0, the HTTP status text is not included in the response: HTTP/2 404 Update the grep pattern so we handle both 1.1 and 2. If the credentials are invalid, the task will report: 21:49:40 -0700 CRIT: ERROR 401 21:49:40 -0700 CRIT: Unable to retrieve a valid package! If the requested package doesn't exist, the task will report: 21:56:33 -0700 CRIT: ERROR 404 21:56:33 -0700 CRIT: Unable to retrieve a valid package! --- tasks/install_shell.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/install_shell.sh b/tasks/install_shell.sh index 05aba5ed..cfa81528 100644 --- a/tasks/install_shell.sh +++ b/tasks/install_shell.sh @@ -429,14 +429,14 @@ do_curl() { rc=$? # check for 404 - grep "404 Not Found" $tmp_stderr 2>&1 >/dev/null + grep "HTTP/.* 404" $tmp_stderr 2>&1 >/dev/null if test $? -eq 0; then critical "ERROR 404" unable_to_retrieve_package fi # check for 401 - grep "401 Unauthorized" $tmp_stderr 2>&1 >/dev/null + grep "HTTP/.* 401" $tmp_stderr 2>&1 >/dev/null if test $? -eq 0; then critical "ERROR 401" unable_to_retrieve_package From bd2787ac15a1631d7658cbfe161dc7ccee6c88c6 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Tue, 1 Jul 2025 12:20:34 -0700 Subject: [PATCH 5/5] (PA-7586) Upgrade puppetcore* dmg from artifacts-puppetcore.puppet.com When using the puppetcore collection on macOS, if we detect the version does not match, then upgrade the DMG. Due to a puppet bug, we cannot pass credentials in the `source` parameter, so curl using 'netrc' to pass credentials securely. Note facter's `os.release.major` returns the Darwin kernel version (23), but our packages are named after the OS version (14), so use `os.macosx.version.major`. ``` class { 'puppet_agent': package_version => '8.13.1', collection => 'puppetcore8', username => 'forge-key', password => Sensitive(...) } include 'puppet_agent' ``` --- manifests/osfamily/darwin.pp | 11 ++++++++++- manifests/prepare/package.pp | 23 +++++++++++++++++++++++ templates/download_puppet.sh.epp | 25 +++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 templates/download_puppet.sh.epp diff --git a/manifests/osfamily/darwin.pp b/manifests/osfamily/darwin.pp index ab3dfd15..7ee8ed74 100644 --- a/manifests/osfamily/darwin.pp +++ b/manifests/osfamily/darwin.pp @@ -20,12 +20,21 @@ } else { $source = "puppet:///pe_packages/${pe_server_version}/${facts['platform_tag']}/${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-1.osx${$productversion_major}.dmg" } + } elsif $puppet_agent::collection =~ /core/ { + $source = 'https://artifacts-puppetcore.puppet.com/v1/download' } else { $source = "${puppet_agent::mac_source}/mac/${puppet_agent::collection}/${productversion_major}/${puppet_agent::arch}/${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-1.osx${$productversion_major}.dmg" } + $destination_name = if $puppet_agent::collection =~ /core/ { + "${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-1.osx${$productversion_major}.dmg" + } else { + undef + } + class { 'puppet_agent::prepare::package': - source => $source, + source => $source, + destination_name => $destination_name, } contain puppet_agent::prepare::package diff --git a/manifests/prepare/package.pp b/manifests/prepare/package.pp index a1b1e549..7b20ddb3 100644 --- a/manifests/prepare/package.pp +++ b/manifests/prepare/package.pp @@ -56,6 +56,29 @@ creates => $local_package_file_path, require => File[$puppet_agent::params::local_packages_dir], } + } elsif $puppet_agent::collection =~ /core/ and $facts['os']['family'] =~ /Darwin/ { + $download_username = getvar('puppet_agent::username', 'forge-key') + $download_password = unwrap(getvar('puppet_agent::password')) + $osname = 'osx' + $osversion = $facts['os']['macosx']['version']['major'] + $osarch = $facts['os']['architecture'] + $fips = 'false' + $dev = count(split($puppet_agent::prepare::package_version, '\.')) > 3 + + $_download_puppet = "${puppet_agent::params::local_packages_dir}/download_puppet.sh" + file { $_download_puppet: + ensure => file, + owner => $puppet_agent::params::user, + group => $puppet_agent::params::group, + mode => '0700', + content => Sensitive(epp('puppet_agent/download_puppet.sh.epp')), + } + + exec { 'Download Puppet Agent': + command => [$_download_puppet], + creates => $local_package_file_path, + require => File[$puppet_agent::params::local_packages_dir], + } } else { file { $local_package_file_path: ensure => file, diff --git a/templates/download_puppet.sh.epp b/templates/download_puppet.sh.epp new file mode 100644 index 00000000..9303d8dc --- /dev/null +++ b/templates/download_puppet.sh.epp @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -x +netrc=$(mktemp) +trap 'rm -f "$netrc"' EXIT +chmod 0600 "$netrc" +cat < "$netrc" +machine artifacts-puppetcore.puppet.com +login <%= $puppet_agent::prepare::package::download_username %> +password <%= $puppet_agent::prepare::package::download_password %> +EOF +/opt/puppetlabs/puppet/bin/curl \ + --get \ + --fail \ + --location \ + --netrc-file "$netrc" \ + --retry 3 \ + --data-urlencode "version=<%= $puppet_agent::prepare::package_version %>" \ + --data-urlencode "dev=<%= $puppet_agent::prepare::package::dev %>" \ + --data-urlencode "os_name=<%= $puppet_agent::prepare::package::osname %>" \ + --data-urlencode "os_version=<%= $puppet_agent::prepare::package::osversion %>" \ + --data-urlencode "os_arch=<%= $puppet_agent::prepare::package::osarch %>" \ + --data-urlencode "fips=<%= $puppet_agent::prepare::package::fips %>" \ + --output "<%= $puppet_agent::prepare::package::local_package_file_path %>" \ + "<%= $puppet_agent::prepare::package::source %>" +