Revert recently introduced cache changes

- Recent pipeline jobs failed with strange cache issues. After review of
  recent changes, it is possible that changes from trigger based jobs to
  normal pipeline jobs may have introduced hidden behavioral changes that
  polluted the build caches. This reverts those changes as an attempt
  to unblock auto deploy while the team continues to investigate.
- https://dev.gitlab.org/gitlab/omnibus-gitlab/-/jobs/19887684
- https://dev.gitlab.org/gitlab/omnibus-gitlab/-/jobs/19883123
- https://gitlab.com/gitlab-org/omnibus-gitlab/-/jobs/6535645918

This reverts commit 373d38d.
This reverts commit b7cb856.

Signed-off-by: Robert Marshall <rmarshall@gitlab.com>

Author: rmarshall-gitlab

.gitlab-ci.yml

@@ -60,7 +60,6 @@ variables:
   DOCS_GITLAB_REPO_SUFFIX: "omnibus"
   CACHE_KEY_SUFFIX: '-v3'
   CACHE_EDITION: "CE"
-  CACHE_POLICY: 'pull-push'
   ISSUE_BOT_LABELS_EXTRA: "group::distribution"
   BUNDLER_VERSION: "2.5.6"
   # NOTE: When `NEXT_RUBY_VERSION` is updated, flip

gitlab-ci-config/dev-gitlab-org.yml

@@ -34,13 +34,11 @@
 .branch-cache: &branch-cache
   key: "${CI_JOB_NAME}-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
   paths: !reference [.cache-paths]
-  policy: $CACHE_POLICY
 
 .tag-cache: &tag-cache
   # Use the same cache key as branch builds, by manually appending `-branch` substring.
   key: "${CI_JOB_NAME}-branch-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
   paths: !reference [.cache-paths]
-  policy: $CACHE_POLICY
 
 .branch_template: &branch_build
   stage: package
@@ -284,11 +282,6 @@ Ubuntu-20.04-branch:
 Ubuntu-22.04-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 
 Debian-10-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/debian_10:${BUILDER_IMAGE_REVISION}"
@@ -314,12 +307,6 @@ Ubuntu-20.04-arm64-branch:
 Ubuntu-22.04-arm64-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04_arm64:${BUILDER_IMAGE_REVISION}"
   extends: .arm64_branch_template
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-      when: manual
-      allow_failure: true
 CentOS-7-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/centos_7:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
@@ -381,15 +368,6 @@ Ubuntu-18.04-fips-branch:
 Ubuntu-20.04-fips-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04_fips:${BUILDER_IMAGE_REVISION}"
   extends: .fips_branch_template
-  rules:
-    - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
-    - if: '$PIPELINE_TYPE == "EE_BRANCH_BUILD_PIPELINE"'
-      when: manual
-      allow_failure: true
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-      when: manual
-      allow_failure: true
 
 Docker-branch:
   extends: .docker_job
@@ -410,8 +388,6 @@ Docker-branch:
   rules:
     - !reference [.default_rules, rules]
     - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
-    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
-    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 
 QA-branch:
   extends: .docker_job

gitlab-ci-config/gitlab-com.yml

@@ -42,7 +42,7 @@
       - gems
       - assets_cache
       - node_modules
-    policy: $CACHE_POLICY
+    policy: pull
 
 .trigger-fips-package-cache:
   cache:
@@ -52,7 +52,7 @@
       - gems
       - assets_cache
       - node_modules
-    policy: $CACHE_POLICY
+    policy: pull
 
 .install-gems: &install-gems
   - gem install bundler:${BUNDLER_VERSION}
@@ -133,8 +133,6 @@
       junit: junit_rspec.xml
 
 .base-trigger-job-variables:
-  # In trigger jobs, we don't want to pollute cache
-  CACHE_POLICY: 'pull'
   ALTERNATIVE_SOURCES: 'true'
   SECURITY_SOURCES: ${SECURITY_SOURCES}
   BUILDER_IMAGE_REVISION: ${BUILDER_IMAGE_REVISION}
@@ -180,6 +178,17 @@
       when: manual
       variables: !reference [.ce-trigger-job-variables]
 
+.trigger-pipeline:
+  stage: trigger-qa
+  image: "${RUBY_IMAGE}"
+  allow_failure: true
+  rules:
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+      when: manual
+  needs:
+    - job: Trigger:package
+      artifacts: false
+
 .review-docs:
   image: "${RUBY_IMAGE}-alpine"
   stage: post-test
@@ -250,6 +259,22 @@ update-gems-cache-for-docker-jobs:
     - update-gems-cache
     - .docker_job
 
+update-trigger-package-cache:
+  extends: .trigger-package-cache
+  stage: update-cache
+  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
+  script:
+    - !reference [.build-package]
+    - echo "Cache is up to date!"
+  cache:
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+  tags: !reference [.distribution-amd64-tags]
+  rules:
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
+  needs:
+    - job: fetch-assets
+      optional: true
+
 ###########################
 #     Branch pipeline     #
 ###########################
@@ -501,6 +526,25 @@ update-knapsack:
 #     Trigger Pipeline     #
 ############################
 
+Trigger:package:
+  extends: .trigger-package-common
+  variables:
+    BASE_OS: "ubuntu_22.04"
+    PACKAGE_DIRECTORY: "pkg/ubuntu-jammy"
+
+Trigger:package:fips:
+  extends:
+    - .trigger-package-common
+    - .trigger-fips-package-cache
+  variables:
+    USE_SYSTEM_SSL: "true"
+    BASE_OS: "ubuntu_20.04_fips"
+    PACKAGE_DIRECTORY: "pkg/ubuntu-focal_fips"
+  allow_failure: true
+  rules:
+    - if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"'
+      when: manual
+
 package_size_check:
   extends: .trigger-package-cache
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
@@ -509,11 +553,29 @@ package_size_check:
     - bundle exec rake build:package:generate_sizefile
     - bundle exec rake check:package_size
   needs:
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
   rules:
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
 
+Trigger:gitlab-docker:
+  extends:
+    - .docker_job
+    - .gems-cache
+  stage: trigger-docker
+  script:
+    - bundle exec rake docker:build:image
+    - bundle exec rake docker:push:triggered
+  rules:
+    - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
+  needs:
+    - job: Trigger:package
+      artifacts: false
+    - job: generate-facts
+      optional: true
+      artifacts: true
+
 qa-subset-test:
   extends:
     - .qa-template
@@ -540,10 +602,9 @@ qa-subset-test:
   needs:
     - job: generate-facts
       artifacts: true
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
-    - job: Docker-branch
-      optional: true
+    - job: Trigger:gitlab-docker
       artifacts: false
 
 qa-remaining-test-manual:
@@ -568,10 +629,9 @@ qa-remaining-test-manual:
   needs:
     - job: generate-facts
       artifacts: true
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
-    - job: Docker-branch
-      optional: true
+    - job: Trigger:gitlab-docker
       artifacts: false
 
 letsencrypt-test:
@@ -587,7 +647,7 @@ letsencrypt-test:
       when: never
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
   needs:
-    - job: Docker-branch
+    - job: Trigger:gitlab-docker
       artifacts: false
 
 RAT:
@@ -612,6 +672,10 @@ RAT:
   needs:
     - job: Ubuntu-22.04-branch
       artifacts: false
+      optional: true
+    - job: Trigger:package
+      artifacts: false
+      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -623,6 +687,10 @@ RAT:FIPS:
   needs:
     - job: Ubuntu-20.04-fips-branch
       artifacts: false
+      optional: true
+    - job: Trigger:package:fips
+      artifacts: false
+      optional: true
     - job: generate-facts
       artifacts: true
 
@@ -644,7 +712,7 @@ GET:Geo:
       when: manual
       allow_failure: true
   needs:
-    - job: Ubuntu-22.04-branch
+    - job: Trigger:package
       artifacts: false
     - job: generate-facts
       artifacts: true
@@ -725,7 +793,7 @@ pages:
 
 .build-package-on-all-os-vars:
   extends: .base-trigger-job-variables
-  SKIP_JOB_REGEX: '/Ubuntu-22.04|Ubuntu-20.04-fips|Docker|QA/'
+  SKIP_JOB_REGEX: '/Ubuntu-22.04|Docker|QA/'
   PIPELINE_TYPE: "${EDITION}_BRANCH_BUILD_PIPELINE"
   CACHE_EDITION: ${EDITION}
   ee: ${ee}

gitlab-ci-config/workflow-rules.yml

@@ -28,8 +28,6 @@ workflow:
       variables:
         PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE
         CACHE_EDITION: "GITLAB"
-        # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-        CACHE_POLICY: 'push'
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $QA_PROJECT_PATH && $CACHE_UPDATE == "true"'
       variables:
         PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE

lib/gitlab/build/facts.rb

@@ -91,8 +91,8 @@ def qa_trigger_vars
           GITLAB_SEMVER_VERSION=#{Build::Info::Git.latest_stable_tag.tr('+', '-')}
           RAT_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs'}
           RAT_FIPS_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_FIPS_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs-fips-ubuntu'}
-          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url(fips: false)}
-          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.package_download_url(job_name: 'Ubuntu-20.04', fips: true)}
+          RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url}
+          RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.fips_package_download_url}
         ]
       end
 

lib/gitlab/build/info/ci.rb

@@ -38,6 +38,26 @@ def commit_ref_slug
           Gitlab::Util.get_env('CI_COMMIT_REF_SLUG')
         end
 
+        def package_download_url
+          return Gitlab::Util.get_env('PACKAGE_URL') if Gitlab::Util.get_env('PACKAGE_URL')
+
+          # For builds running in dev.gitlab.org, or nightly pipelines in .com,
+          # use the artifact from `Ubuntu-22.04-branch` job. For the other build,
+          # which is essentially in triggered pipeline, use artifact from
+          # `Trigger:package` job.
+          if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
+            Build::Info::CI.branch_build_package_download_url
+          else
+            Build::Info::CI.triggered_package_download_url(fips: false)
+          end
+        end
+
+        def fips_package_download_url
+          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if Gitlab::Util.get_env('FIPS_PACKAGE_URL')
+
+          Build::Info::CI.triggered_package_download_url(fips: false)
+        end
+
         def artifact_url(job_name, file_path)
           client = Gitlab::APIClient.new
           target_job_id = client.get_job_id(job_name)
@@ -47,10 +67,15 @@ def artifact_url(job_name, file_path)
           URI("#{api_v4_url}/projects/#{project_id}/jobs/#{target_job_id}/artifacts/#{file_path}")
         end
 
-        def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64', fips: Build::Check.use_system_ssl?)
-          return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if fips && Gitlab::Util.get_env('FIPS_PACKAGE_URL')
-          return Gitlab::Util.get_env('PACKAGE_URL') if !fips && Gitlab::Util.get_env('PACKAGE_URL')
+        def triggered_package_download_url(fips: Build::Check.use_system_ssl?)
+          folder = fips ? 'ubuntu-focal_fips' : 'ubuntu-jammy'
+          job_name = fips ? 'Trigger:package:fips' : 'Trigger:package'
+          package_path = "pkg/#{folder}/gitlab.deb"
+
+          artifact_url(job_name, package_path)
+        end
 
+        def branch_build_package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
           case job_name
           when /AlmaLinux-8/
             # In EL world, amd64 is called x86_64
@@ -70,8 +95,6 @@ def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64', fips: Build::C
             folder = "#{folder}_aarch64"
           end
 
-          job_name = "#{job_name}-fips" if fips
-
           job_name = "#{job_name}-branch" unless Build::Info::CI.tag_name
 
           package_path = "pkg/#{folder}/#{package_file_name}"

lib/gitlab/package_size.rb

@@ -13,7 +13,7 @@ class PackageSizeCheck
 
   class << self
     def fetch_sizefile
-      sizefile_url = URI("#{Build::Info::CI.package_download_url}.size")
+      sizefile_url = Build::Info::CI.artifact_url("Trigger:package", "pkg/ubuntu-jammy/gitlab.deb.size")
 
       # We have to use net/http here because `gitlab` gem's `download_job_artifact_file`
       # method doesn't support plain text files. It has to be either binary or valid JSON.

spec/lib/gitlab/build/image_spec.rb

@@ -159,7 +159,7 @@ def self.dockerhub_image_name
             "PACKAGECLOUD_REPO=download-package",
             "RELEASE_PACKAGE=gitlab-ce",
             "RELEASE_VERSION=12.121.12-ce.0",
-            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ce_12.121.12-ce.0_amd64.deb",
+            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
             "CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
           ].join("\n")
         end
@@ -180,7 +180,7 @@ def self.dockerhub_image_name
             "PACKAGECLOUD_REPO=download-package",
             "RELEASE_PACKAGE=gitlab-ee",
             "RELEASE_VERSION=12.121.12-ee.0",
-            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ee_12.121.12-ee.0_amd64.deb",
+            "DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
             "CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
           ].join("\n")
         end