Ensure nightly pipelines are correctly detected in Canonical

- Ensures package is built for both branch pipelines and
  the nightly job
- Fixes package download URL calculations

Closes https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/8451

Signed-off-by: Balasankar 'Balu' C <balasankar@gitlab.com>

Author: balasankarc

gitlab-ci-config/dev-gitlab-org.yml

@@ -433,37 +433,6 @@ Raspberry-Pi-2-Bookworm-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/rpi_12:${BUILDER_IMAGE_REVISION}"
   extends: .rpi_branch_template
 
-# This job runs only on nightly EE builds
-RAT-Nightly:
-  stage: slow_jobs
-  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
-  script:
-    - bundle exec rake qa:rat:nightly
-  cache: !reference [.branch-cache]
-  needs:
-    - Ubuntu-22.04-branch
-  allow_failure: true
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
-
-# This job runs only on nightly EE builds
-RAT-Nightly-FIPS:
-  stage: slow_jobs
-  image: "${BUILDER_IMAGE_REGISTRY}/almalinux_8_fips:${BUILDER_IMAGE_REVISION}"
-  variables:
-    USE_SYSTEM_SSL: "true"
-    RAT_REFERENCE_ARCHITECTURE: "omnibus-gitlab-mrs-fips"
-  script:
-    - bundle exec rake qa:rat:nightly
-  cache: !reference [.branch-cache]
-  needs:
-    - AlmaLinux-8-fips-branch
-  allow_failure: true
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
-
 ############################
 #     Release pipeline     #
 ############################

gitlab-ci-config/gitlab-com.yml

@@ -667,9 +667,15 @@ RAT:
     - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"'
       when: manual
       allow_failure: true
+    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
+      allow_failure: true
   needs:
+    - job: Ubuntu-22.04-branch
+      artifacts: false
+      optional: true
     - job: Trigger:package
       artifacts: false
+      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -679,8 +685,12 @@ RAT:FIPS:
     PACKAGE_URL: ${RAT_FIPS_PACKAGE_URL}
     REFERENCE_ARCHITECTURE: ${RAT_FIPS_REFERENCE_ARCHITECTURE}
   needs:
+    - job: Ubuntu-20.04-fips-branch
+      artifacts: false
+      optional: true
     - job: Trigger:package:fips
       artifacts: false
+      optional: true
     - job: generate-facts
       artifacts: true
 

gitlab-ci-config/workflow-rules.yml

@@ -43,9 +43,6 @@ workflow:
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_PIPELINE_SOURCE == "merge_request_event"'
       variables:
         PIPELINE_TYPE: GITLAB_MR_PIPELINE
-    - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_COMMIT_REF_PROTECTED == "true"'
-      variables:
-        PIPELINE_TYPE: PROTECTED_TEST_PIPELINE
     - if: '$PIPELINE_TYPE == null && $NIGHTLY && $CI_COMMIT_BRANCH && ($ee == "true" || $GITLAB_VERSION =~ /-ee$/)'
       variables:
         PIPELINE_TYPE: EE_NIGHTLY_BUILD_PIPELINE
@@ -94,6 +91,9 @@ workflow:
         PIPELINE_TYPE: EE_TAG_BUILD_PIPELINE
         CACHE_EDITION: "EE"
         GITLAB_EDITION: "Enterprise Edition"
+    - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_COMMIT_REF_PROTECTED == "true"'
+      variables:
+        PIPELINE_TYPE: PROTECTED_TEST_PIPELINE
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $SECURITY_PROJECT_PATH && $CI_PIPELINE_SOURCE == "merge_request_event"'
       variables:
         PIPELINE_TYPE: GITLAB_MR_PIPELINE

lib/gitlab/aws_helper.rb

@@ -29,7 +29,7 @@ def create_ami
       architecture = 'amd64'
     end
 
-    @download_url = Build::Info::CI.package_download_url(job_name: "Ubuntu-20.04", arch: architecture)
+    @download_url = Build::Info::CI.branch_build_package_download_url(job_name: "Ubuntu-20.04", arch: architecture)
 
     system(*%W[support/packer/packer_ami.sh #{@version} #{@type} #{@download_url} #{Build::Info::CI.job_token} #{@license_file}])
   end

lib/gitlab/build/facts.rb

@@ -30,9 +30,17 @@ def get_component_shas(version_manifest_file = 'version-manifest.json')
         softwares = version_manifest['software']
         results = {}
         Gitlab::Version::COMPONENTS_ENV_VARS.keys.map do |component|
-          next unless softwares.key?(component)
+          # For both `gitlab-rails` and `gitlab-rails-ee`, the key in
+          # version-manifest.json is `gitlab-rails`
+          version_manifest_key = if component == 'gitlab-rails' || component == 'gitlab-rails-ee'
+                                   'gitlab-rails'
+                                 else
+                                   component
+                                 end
 
-          results[component] = softwares[component]['locked_version']
+          next unless softwares.key?(version_manifest_key)
+
+          results[component] = softwares[version_manifest_key]['locked_version']
         end
 
         results
@@ -83,8 +91,8 @@ def qa_trigger_vars
           GITLAB_SEMVER_VERSION=#{Build::Info::Git.latest_stable_tag.tr('+', '-')}
           RAT_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs'}
           RAT_FIPS_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_FIPS_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs-fips-ubuntu'}
-          RAT_PACKAGE_URL=#{Gitlab::Util.get_env('PACKAGE_URL') || Build::Info::CI.triggered_package_download_url(fips: false)}
-          RAT_FIPS_PACKAGE_URL=#{Gitlab::Util.get_env('FIPS_PACKAGE_URL') || Build::Info::CI.triggered_package_download_url(fips: true)}
+          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url}
+          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.fips_package_download_url}
         ]
       end
 

lib/gitlab/build/info/ci.rb

@@ -38,6 +38,26 @@ def commit_ref_slug
           Gitlab::Util.get_env('CI_COMMIT_REF_SLUG')
         end
 
+        def package_download_url
+          return Gitlab::Util.get_env('PACKAGE_URL') if Gitlab::Util.get_env('PACKAGE_URL')
+
+          # For builds running in dev.gitlab.org, or nightly pipelines in .com,
+          # use the artifact from `Ubuntu-22.04-branch` job. For the other build,
+          # which is essentially in triggered pipeline, use artifact from
+          # `Trigger:package` job.
+          if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
+            Build::Info::CI.branch_build_package_download_url
+          else
+            Build::Info::CI.triggered_package_download_url(fips: false)
+          end
+        end
+
+        def fips_package_download_url
+          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if Gitlab::Util.get_env('FIPS_PACKAGE_URL')
+
+          Build::Info::CI.triggered_package_download_url(fips: false)
+        end
+
         def artifact_url(job_name, file_path)
           client = Gitlab::APIClient.new
           target_job_id = client.get_job_id(job_name)
@@ -55,7 +75,7 @@ def triggered_package_download_url(fips: Build::Check.use_system_ssl?)
           artifact_url(job_name, package_path)
         end
 
-        def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
+        def branch_build_package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
           case job_name
           when /AlmaLinux-8/
             # In EL world, amd64 is called x86_64

lib/gitlab/build/info/docker.rb

@@ -14,12 +14,7 @@ def tag
         def release_file_contents
           repo = Gitlab::Util.get_env('PACKAGECLOUD_REPO') # Target repository
 
-          download_url = if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
-                           Build::Info::CI.package_download_url
-                         else
-                           Build::Info::CI.triggered_package_download_url
-                         end
-
+          download_url = Build::Info::CI.package_download_url
           raise "Unable to identify package download URL." unless download_url
 
           contents = []

spec/lib/gitlab/build/facts_spec.rb

@@ -69,6 +69,7 @@
   let(:component_shas) do
     {
       'gitlab-rails' => '6f286d7717f419489a08a9918621f438256e397b',
+      'gitlab-rails-ee' => '6f286d7717f419489a08a9918621f438256e397b',
       'gitaly' => 'b55578ec476e8bc8ecd9775ee7e9960b52e0f6e0',
       'gitlab-shell' => '264d63e81cbf08e3ae75e84433b8d09af15f351f',
       'gitlab-pages' => 'b0cb1f0c0783db2f5176301e6528fe41e1b42abf',

spec/lib/gitlab/tasks/aws_spec.rb

@@ -49,7 +49,7 @@ def deregister_image(parameters)
       allow(Build::Check).to receive(:on_tag?).and_return(true)
       allow(Build::Check).to receive(:is_auto_deploy?).and_return(false)
       allow(Build::Check).to receive(:is_rc_tag?).and_return(false)
-      allow(Build::Info::CI).to receive(:package_download_url).and_return('http://example.com')
+      allow(Build::Info::CI).to receive(:branch_build_package_download_url).and_return('http://example.com')
     end
 
     it 'should identify ce category correctly, if specified' do
@@ -129,7 +129,7 @@ def deregister_image(parameters)
       allow(Build::Check).to receive(:on_tag?).and_return(true)
       allow(Build::Check).to receive(:is_auto_deploy?).and_return(false)
       allow(Build::Check).to receive(:is_rc_tag?).and_return(true)
-      allow(Build::Info::CI).to receive(:package_download_url).and_return('http://example.com')
+      allow(Build::Info::CI).to receive(:branch_build_package_download_url).and_return('http://example.com')
     end
 
     it 'does not do anything' do
@@ -144,7 +144,7 @@ def deregister_image(parameters)
       allow(Build::Check).to receive(:on_tag?).and_return(true)
       allow(Build::Check).to receive(:is_auto_deploy?).and_return(true)
       allow(Build::Check).to receive(:is_rc_tag?).and_return(false)
-      allow(Build::Info::CI).to receive(:package_download_url).and_return('http://example.com')
+      allow(Build::Info::CI).to receive(:branch_build_package_download_url).and_return('http://example.com')
     end
 
     it 'does not do anything' do