Merge pull request #1181 from elezar/integrate-ngc-automation

Integrate ngc automation

Author: elezar

.common-ci.yml

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-include:
-  - local: '.common-ci.yml'
-
 default:
+  image: docker
+  services:
+    - name: docker:dind
+      command: ["--experimental"]
   tags:
     - cnt
     - container-dev
@@ -25,6 +26,8 @@ default:
     - type/docker
 
 variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+
   DOCKER_DRIVER: overlay2
   DOCKER_TLS_CERTDIR: "/certs"
   # Release "devel"-tagged images off the main branch
@@ -39,6 +42,40 @@ variables:
   KITMAKER_RELEASE_FOLDER: "kitmaker"
   PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
+stages:
+  - pull
+  - scan
+  - release
+  - ngc-publish
+
+.pipeline-trigger-rules:
+  rules:
+    # We trigger the pipeline if started manually
+    - if: $CI_PIPELINE_SOURCE == "web"
+    # We trigger the pipeline on the main branch
+    - if: $CI_COMMIT_BRANCH == "main"
+    # We trigger the pipeline on the release- branches
+    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
+    # We trigger the pipeline on tags
+    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
+
+workflow:
+  rules:
+    # We trigger the pipeline on a merge request
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    # We then add all the regular triggers
+    - !reference [.pipeline-trigger-rules, rules]
+
+# Download the regctl binary for use in the release steps
+.regctl-setup:
+  before_script:
+    - export REGCTL_VERSION=v0.4.5
+    - apk add --no-cache curl
+    - mkdir -p bin
+    - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
+    - chmod a+x bin/regctl
+    - export PATH=$(pwd)/bin:${PATH}
+
 # .copy-images copies the required application and packaging images from the
 # IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
 # to
@@ -195,98 +232,59 @@ push-images-to-staging:
     OUT_IMAGE_NAME: "${NGC_STAGING_REGISTRY}/container-toolkit"
     OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
 
-.release-images:
-  extends:
-    - .copy-images
-  stage: release
+.publish-images:
+  stage: ngc-publish
   needs:
     - scan-images
     - push-images-to-staging
+  image:
+    name: "${CNT_NGC_PUBLISH_IMAGE}"
+    pull_policy: always
   variables:
-    IN_REGISTRY: "${CI_REGISTRY}"
-    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
-    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
-
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-    OUT_IMAGE_TAG: "${CI_COMMIT_TAG}"
-
-release-images-to-ngc:
-  extends:
-    - .release-images
-  rules:
-    - if: $CI_COMMIT_TAG
-
-release-images-dummy:
-  extends:
-    - .release-images
-  variables:
-    REGCTL: "echo [DUMMY] regctl"
-  rules:
-    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
 
-# .sign-images forms the base of the jobs which sign images in the NGC registry.
-.sign-images:
-  stage: sign
-  image: ubuntu:latest
-  parallel:
-    matrix:
-      - TAG_SUFFIX: ["", "-packaging"]
   variables:
-    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-    IMAGE_TAG: "${CI_COMMIT_TAG}"
-    NGC_CLI: "ngc-cli/ngc"
-  before_script:
-    - !reference [.ngccli-setup, before_script]
-  script:
-    - |
-      # We ensure that the IMAGE_NAME and IMAGE_TAG is set
-      echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1
-      echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1
+    GITLAB_ACCESS_TOKEN: "${CNT_GITLAB_TOKEN}"
 
-      export IMAGE=${IMAGE_NAME}:${IMAGE_TAG}${TAG_SUFFIX}
-      echo "Signing the image ${IMAGE}"
-      ${NGC_CLI} registry image publish --source ${IMAGE} ${IMAGE} --public --discoverable --allow-guest --sign --org nvidia
+    OUT_IMAGE_TAG: "${CI_COMMIT_TAG}"
 
-# Define the external image signing steps for NGC
-# Download the ngc cli binary for use in the sign steps
-.ngccli-setup:
+    VERSION_FILE: "build-info-${CI_PIPELINE_ID}.txt"
+    PROJECT_NAME: "nvidia-container-toolkit"
   before_script:
-    - apt-get update && apt-get install -y curl unzip jq
     - |
-      if [ -z "${NGCCLI_VERSION}" ]; then
-        NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions"
-        # Extract the latest version from the JSON data using jq
-        export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr')
+      if [ -z "$NGC_PUBLISHING_PROJECT_PATH" ]; then
+        echo "NGC_PUBLISHING_PROJECT_PATH not set"
+        exit 1
+      else
+        echo "publishing to ${NGC_PUBLISHING_PROJECT_PATH}"
       fi
-      echo "NGCCLI_VERSION ${NGCCLI_VERSION}"
-    - curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip
-    - unzip ngccli_linux.zip
-    - chmod u+x ngc-cli/ngc
 
-sign-ngc-images:
+      rm -f ${VERSION_FILE}
+      echo "${IN_IMAGE_TAG} ${OUT_IMAGE_TAG}" >> ${VERSION_FILE}
+      echo "${IN_IMAGE_TAG}-packaging ${OUT_IMAGE_TAG}-packaging" >> ${VERSION_FILE}"
+      cat ${VERSION_FILE}
+  script:
+    - cnt-ngc-publish render --project-name "${PROJECT_NAME}" --versions-file "${VERSION_FILE}" --output "${PROJECT_NAME}".yaml
+    - cnt-ngc-publish merge-request --files "${PROJECT_NAME}.yaml"
+  artifacts:
+    paths:
+      - "${VERSION_FILE}"
+      - "${PROJECT_NAME}.yaml"
+
+
+publish-images-to-ngc:
   extends:
-    - .sign-images
-  needs:
-    - release-images-to-ngc
+    - .publish-images
   rules:
     - if: $CI_COMMIT_TAG
-  variables:
-    NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
-  retry:
-    max: 2
 
-sign-images-dummy:
+# We create a dummy MR that exercises the publishing logic.
+# TODO: This MR should be closed automatically.
+publish-images-dummy:
   extends:
-    - .sign-images
-  needs:
-    - release-images-dummy
+    - .publish-images
   variables:
-    NGC_CLI: "echo [DUMMY] ngc-cli/ngc"
-    IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
+    NGC_PUBLISHING_PROJECT_PATH: dl/container-dev/ngc-automation
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
   rules:
     - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
+