CLOUDP-337727: Add operator with race checker image in init variant (#341)

# Summary

This patch adds a separate task in the `init_test_run` variant for
building the operator image with the race-checker enabled.

This patch also moved the tests under `scripts/release` according to
[standard pytest best
practices](https://doc.pytest.org/en/latest/explanation/goodpractices.html#tests-as-part-of-application-code)
as currently our python test targets were not capturing all the tests.

## Proof of Work

[EVG Job](https://spruce.mongodb.com/version/689b8a097e5e41000778b82e)

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: mircea-cosbuc

.evergreen.yml

@@ -156,6 +156,25 @@ variables:
       - name: build_agent_images_ubi
         variant: init_test_run
 
+  - &base_om7_dependency_with_race
+    depends_on:
+      - name: build_om_images
+        variant: build_om70_images
+      - name: build_operator_race_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image
+        variant: init_test_run
+      - name: build_init_appdb_images_ubi
+        variant: init_test_run
+      - name: build_init_om_images_ubi
+        variant: init_test_run
+      - name: build_agent_images_ubi
+        variant: init_test_run
+
   - &base_om8_dependency
     depends_on:
       - name: build_om_images
@@ -439,18 +458,23 @@ tasks:
       - func: setup_building_host
       - func: pipeline
         vars:
-          skip_tags: ubuntu,release
-          distro: ubi
           image_name: operator
 
+  - name: build_operator_race_ubi
+    commands:
+      - func: clone
+      - func: setup_building_host
+      - func: pipeline
+        vars:
+          image_name: operator-race
+
   - name: build_init_om_images_ubi
     commands:
       - func: clone
       - func: setup_building_host
       - func: pipeline
         vars:
           image_name: init-ops-manager
-          skip_tags: ubuntu,release
 
   - name: build_init_appdb_images_ubi
     commands:
@@ -459,7 +483,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: init-appdb
-          skip_tags: ubuntu,release
 
   - name: build_agent_images_ubi
     commands:
@@ -468,7 +491,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: agent
-          skip_tags: ubuntu,release
 
   - name: build_init_database_image_ubi
     commands:
@@ -477,7 +499,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: init-database
-          skip_tags: ubuntu,release
 
   - name: build_database_image_ubi
     commands:
@@ -486,7 +507,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: database
-          skip_tags: ubuntu,release
 
   - name: build_readiness_probe_image
     commands:
@@ -495,7 +515,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: readiness-probe
-          skip_tags: ubuntu,release
 
   - name: build_upgrade_hook_image
     commands:
@@ -504,7 +523,6 @@ tasks:
       - func: pipeline
         vars:
           image_name: upgrade-hook
-          skip_tags: ubuntu,release
 
   - name: prepare_aws
     priority: 59
@@ -1434,7 +1452,7 @@ buildvariants:
     tags: [ "e2e_test_suite" ]
     run_on:
       - ubuntu1804-xlarge
-    <<: *base_om7_dependency
+    <<: *base_om7_dependency_with_race
     tasks:
       - name: e2e_operator_race_with_telemetry_task_group
 
@@ -1653,6 +1671,7 @@ buildvariants:
       - ubuntu2204-small
     tasks:
       - name: build_operator_ubi
+      - name: build_operator_race_ubi
       - name: build_test_image
       - name: build_mco_test_image
       - name: build_init_appdb_images_ubi

build_info.json

@@ -25,6 +25,23 @@
         ]
       }
     },
+    "operator-race": {
+      "dockerfile-path": "docker/mongodb-kubernetes-operator/Dockerfile.atomic",
+      "patch": {
+        "repository": "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes",
+        "platforms": [
+          "linux/amd64"
+        ]
+      },
+      "staging": {
+        "sign": true,
+        "repository": "268558157000.dkr.ecr.us-east-1.amazonaws.com/staging/mongodb-kubernetes",
+        "platforms": [
+          "linux/arm64",
+          "linux/amd64"
+        ]
+      }
+    },
     "init-database": {
       "dockerfile-path": "docker/mongodb-kubernetes-init-database/Dockerfile.atomic",
       "patch": {

scripts/release/atomic_pipeline.py

@@ -117,17 +117,20 @@ def build_mco_tests_image(build_configuration: ImageBuildConfiguration):
     )
 
 
-def build_operator_image(build_configuration: ImageBuildConfiguration):
+def build_operator_image(build_configuration: ImageBuildConfiguration, with_race_detection: bool = False):
     """Calculates arguments required to build the operator image, and starts the build process."""
     # In evergreen, we can pass test_suffix env to publish the operator to a quay
     # repository with a given suffix.
     test_suffix = os.getenv("test_suffix", "")
     log_automation_config_diff = os.getenv("LOG_AUTOMATION_CONFIG_DIFF", "false")
 
+    build_configuration.version = f"{build_configuration.version}{'-race' if with_race_detection else ''}"
+
     args = {
         "version": build_configuration.version,
         "log_automation_config_diff": log_automation_config_diff,
         "test_suffix": test_suffix,
+        "use_race": "true" if with_race_detection else "false",
     }
 
     logger.info(f"Building Operator args: {args}")

scripts/release/build/build_info.py

@@ -3,11 +3,17 @@
 from typing import Dict
 
 from scripts.release.build.build_scenario import BuildScenario
-from scripts.release.constants import DEFAULT_REPOSITORY_PATH, DEFAULT_CHANGELOG_PATH, RELEASE_INITIAL_VERSION_ENV_VAR, \
-    get_initial_version, get_initial_commit_sha
+from scripts.release.constants import (
+    DEFAULT_REPOSITORY_PATH,
+    DEFAULT_CHANGELOG_PATH,
+    RELEASE_INITIAL_VERSION_ENV_VAR,
+    get_initial_version,
+    get_initial_commit_sha,
+)
 
 MEKO_TESTS_IMAGE = "meko-tests"
 OPERATOR_IMAGE = "operator"
+OPERATOR_RACE_IMAGE = "operator-race"
 MCO_TESTS_IMAGE = "mco-tests"
 READINESS_PROBE_IMAGE = "readiness-probe"
 UPGRADE_HOOK_IMAGE = "upgrade-hook"
@@ -50,11 +56,13 @@ class BuildInfo:
     helm_charts: Dict[str, HelmChartInfo]
 
 
-def load_build_info(scenario: BuildScenario,
-                    repository_path: str = DEFAULT_REPOSITORY_PATH,
-                    changelog_sub_path: str = DEFAULT_CHANGELOG_PATH,
-                    initial_commit_sha: str = None,
-                    initial_version: str = None) -> BuildInfo:
+def load_build_info(
+    scenario: BuildScenario,
+    repository_path: str = DEFAULT_REPOSITORY_PATH,
+    changelog_sub_path: str = DEFAULT_CHANGELOG_PATH,
+    initial_commit_sha: str = None,
+    initial_version: str = None,
+) -> BuildInfo:
     f"""
     Load build information based on the specified scenario.
 

scripts/release/build/conftest.py

@@ -22,7 +22,7 @@ def is_release_scenario(self) -> bool:
         return self.scenario == BuildScenario.RELEASE
 
     def base_registry(self) -> str:
-        return self.registry.rpartition('/')[0]
+        return self.registry.rpartition("/")[0]
 
     def image_name(self) -> str:
-        return self.registry.rpartition('/')[2]
+        return self.registry.rpartition("/")[2]

scripts/release/build/image_build_configuration.py

@@ -1,5 +1,6 @@
 import argparse
 import os
+from functools import partial
 from typing import Callable, Dict
 
 from opentelemetry import context, trace
@@ -37,6 +38,7 @@
     MCO_TESTS_IMAGE,
     MEKO_TESTS_IMAGE,
     OPERATOR_IMAGE,
+    OPERATOR_RACE_IMAGE,
     OPS_MANAGER_IMAGE,
     READINESS_PROBE_IMAGE,
     UPGRADE_HOOK_IMAGE,
@@ -67,6 +69,7 @@ def get_builder_function_for_image_name() -> Dict[str, Callable]:
     image_builders = {
         MEKO_TESTS_IMAGE: build_meko_tests_image,
         OPERATOR_IMAGE: build_operator_image,
+        OPERATOR_RACE_IMAGE: partial(build_operator_image, with_race_detection=True),
         MCO_TESTS_IMAGE: build_mco_tests_image,
         READINESS_PROBE_IMAGE: build_readiness_probe_image,
         UPGRADE_HOOK_IMAGE: build_upgrade_hook_image,
@@ -96,7 +99,6 @@ def image_build_config_from_args(args) -> ImageBuildConfiguration:
     image = args.image
 
     build_scenario = get_scenario_from_arg(args.build_scenario) or BuildScenario.infer_scenario_from_environment()
-
     build_info = load_build_info(build_scenario)
     logger.info(f"image is {image}")
     logger.info(f"images are {build_info.images}")