refactor: redesign CI pipelines (#339)

* run integration (bash) tests only when `engine` files are changed
* run integration (bash) tests for the first commit as well (not only for the second and next ones)
* run UI builds and deployments only when `ui` files are changed
* enable integration tests for contributors
* include SAST jobs from GitLab templates
* avoid duplication of pipelines

Author: agneum

.gitlab-ci.yml

@@ -2,7 +2,13 @@ variables:
   SAST_EXCLUDED_ANALYZERS: "semgrep-sast,gosec-sast"
   DOCKER_DRIVER: overlay2
 
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+      when: never
+    - if: $CI_COMMIT_TAG
+
 include:
-  - template: Security/SAST.gitlab-ci.yml
   - local: 'engine/.gitlab-ci.yml'
   - local: 'ui/.gitlab-ci.yml'

engine/.gitlab-ci.yml

@@ -12,7 +12,10 @@ stages:
 .only_engine: &only_engine
   rules:
     - if: $CI_COMMIT_TAG =~ /^v[a-zA-Z0-9_.-]*/
-    - if: $CI_COMMIT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - engine/**/*
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
       changes:
         - engine/**/*
 
@@ -32,7 +35,7 @@ stages:
 
 .only_dle_feature: &only_feature
   rules:
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         - engine/**/*
 
@@ -346,18 +349,14 @@ build-image-swagger-latest:
     - export LATEST_TAG=$(echo ${CLEAN_TAG%.*}-latest)
     - export TAGS="${DOCKER_NAME}:${LATEST_TAG}"
 
-
 .bash-test: &bash_test
   stage: integration-test
   variables:
     IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
   rules:
-    - changes:
-        - ui/**/*
-      when: never
     - if: '$CI_PROJECT_NAMESPACE != "postgres-ai"'
       when: never
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         - engine/**/*
   artifacts:

ui/.gitlab-ci.yml

@@ -1,7 +1,12 @@
+include:
+  - template: Security/SAST.gitlab-ci.yml
+  - local: 'ui/packages/ce/.gitlab-ci.yml'
+  - local: 'ui/packages/platform/.gitlab-ci.yml'
+
 .only_ui: &only_ui
   rules:
     - if: $CI_COMMIT_TAG =~ /^ui\/[0-9.]+$/
-    - if: $CI_COMMIT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         - ui/**/*
 
@@ -14,6 +19,20 @@ check_code_style:
     - npm --prefix ui/ run lint -w packages/ce
     - npm --prefix ui/ run lint -w packages/platform
 
-include:
-  - local: 'ui/packages/ce/.gitlab-ci.yml'
-  - local: 'ui/packages/platform/.gitlab-ci.yml'
+eslint-sast:
+  <<: *only_ui
+  extends: .sast-analyzer
+  image:
+    name: "$SAST_ANALYZER_IMAGE"
+  variables:
+    SAST_ANALYZER_IMAGE_TAG: 2
+    SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG"
+
+nodejs-scan-sast:
+  <<: *only_ui
+  extends: .sast-analyzer
+  image:
+    name: "$SAST_ANALYZER_IMAGE"
+  variables:
+    SAST_ANALYZER_IMAGE_TAG: 2
+    SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG"

ui/packages/ce/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 # Conditions.
 .only_ui_feature: &only_ui_feature
   rules:
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
         - ui/**/*
 

ui/packages/platform/.gitlab-ci.yml

@@ -5,12 +5,23 @@
 # Proprietary and confidential
 #--------------------------------------------------------------------------
 
-workflow:
+# Conditions.
+.only_ui_tag_release: &only_ui_tag_release
+  rules:
+    - if: $CI_COMMIT_TAG =~ /^ui\/[0-9.]+$/
+
+.only_ui_staging: &only_ui_staging
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      changes:
+        - ui/**/*
+
+.only_ui_feature: &only_ui_feature
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: never
-    - if: $CI_COMMIT_BRANCH
-    - if: $CI_COMMIT_TAG
+      changes:
+        - ui/**/*
+      when: manual
 
 # Environments.
 .environment_production: &env_production
@@ -64,24 +75,6 @@ workflow:
     # Deploy to k8s cluster.
     - kubectl apply --filename /tmp/platform-console.yaml -n $NAMESPACE
 
-# Conditions.
-.only_ui_tag_release: &only_ui_tag_release
-  rules:
-    - if: $CI_COMMIT_TAG =~ /^ui\/[0-9.]+$/
-
-.only_ui_staging: &only_ui_staging
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-      changes:
-        - ui/**/*
-
-.only_ui_feature: &only_ui_feature
-  rules:
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-      changes:
-        - ui/**/*
-      when: manual
-
 # Jobs.
 # Production.
 ui_build_platform_image_tag_release: