Add documentation for pipeline.py (#130)

# Summary

- Add documentation for pipeline.py
- Add env variables documentation used by `pipeline.py` for easier
references.
- Some docs were in wrong locations, likely merging errors. I've looked
into the history and moved the to their original locations.

Author: SimonBaeumer

PIPELINE.md

@@ -0,0 +1,70 @@
+# Pipeline
+
+## Environment Variables (env vars)
+
+This listing contains all environment variables used in `pipeline.py`.
+Default evergreen-ci expansions can be looked up [here](https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#expansions).
+
+| Environment Variable           | Usage / Description                                                                |
+|-------------------------------|------------------------------------------------------------------------------------|
+| `otel_trace_id`                | OpenTelemetry tracing: trace ID. Default evergreen-ci expansion.                   |
+| `otel_parent_id`               | OpenTelemetry tracing: parent span ID. Default evergreen-ci expansion.             |
+| `otel_collector_endpoint`      | OpenTelemetry tracing: collector endpoint. Default evergreen-ci expansion.         |
+| `distro`                       | Image type (defaults to `ubi`)                                                     |
+| `BASE_REPO_URL`                | Base repository URL for images                                                     |
+| `namespace`                    | Kubernetes namespace (defaults to `default`)                                       |
+| `skip_tags`                    | Tags to skip during build                                                          |
+| `include_tags`                 | Tags to include during build                                                       |
+| `all_agents`                   | Whether to build all agent images                                                  |
+| `RUNNING_IN_EVG`               | Whether running in Evergreen pipeline                                              |
+| `is_patch`                     | Whether running as a patch build. Default evergreen-ci expansion.                  |
+| `pin_tag_at`                   | Time to pin image tag (format: `HH:MM`)                                            |
+| `created_at`                   | Build creation time (format: `%y_%m_%d_%H_%M_%S`). Default evergreen-ci expansion. |
+| `triggered_by_git_tag`         | Git tag that triggered the build. Default evergreen-ci expansion. Default evergreen-ci expansion.                 |
+| `version_id`                   | Patch ID or version for non-release builds. Default evergreen-ci expansion.        |
+| `test_suffix`                  | Suffix for test images                                                             |
+| `LOG_AUTOMATION_CONFIG_DIFF`   | Whether to log automation config diff                                              |
+| `PYTHON_VERSION`               | Python version for test images                                                     |
+| `GOLANG_VERSION`               | Go version for community images and tests                                          |
+| `QUAY_REGISTRY`                | Quay registry URL (defaults to `quay.io/mongodb`)                                  |
+| `REGISTRY`                     | ECR registry URL (defaults to `268558157000.dkr.ecr.us-east-1.amazonaws.com/dev`)  |
+| `om_version`                   | Ops Manager version for OM image builds                                            |
+| `om_download_url`              | Download URL for Ops Manager (optional, can be auto-detected)                      |
+
+## Context Image Build Process
+
+```
+                      ┌─────────────────────────────┐
+                      │       Release Pipeline      │
+                      └────────────┬────────────────┘
+                                   │
+                                   ▼
+                     ┌─────────────────────────────────┐
+                     │  Build context image            │
+                     │  Tag: opsmanager-context:1.33.0 │
+                     └────────────┬────────────────────┘
+                                   │
+                                   ▼
+                     ┌───────────────────────────────┐
+                     │         Daily Build           │
+                     │   Base: opsmanager-context    │
+                     │   Input tag: 1.33.0           │
+                     └────────────┬──────────────────┘
+                                  │
+                                  ▼
+                    ┌────────────────────────────────────┐
+                    │       Push Two Image Tags          │
+                    └────────────┬───────────────┬───────┘
+                                 ▼               ▼
+             ┌────────────────────────┐ ┌──────────────────────────────┐
+             │ Rolling Tag (latest)   │ │ Immutable Tag (daily stamp)  │
+             │ opsmanager:1.33.0      │ │ opsmanager:1.33.0-2025-01-01 │
+             └────────────────────────┘ └──────────────────────────────┘
+
+                                 ▼ (next day build)
+             ┌────────────────────────┐ ┌──────────────────────────────┐
+             │ opsmanager:1.33.0      │ │ opsmanager:1.33.0-2025-01-02 │
+             └────────────────────────┘ └──────────────────────────────┘
+               ↑ now updated to point       ↑ new image pushed
+                 to the 2025-01-02 build
+```

pipeline.py

@@ -264,6 +264,7 @@ def get_release() -> Dict:
 
 
 def get_git_release_tag() -> tuple[str, bool]:
+    """Returns the git tag of the current run on releases, on non-release returns the patch id."""
     release_env_var = os.getenv("triggered_by_git_tag")
 
     # that means we are in a release and only return the git_tag; otherwise we want to return the patch_id
@@ -290,22 +291,19 @@ def copy_into_container(client, src, dst):
         container.put_archive(os.path.dirname(dst), fd.read())
 
 
-"""
-Generates docker manifests by running the following commands:
-1. Clear existing manifests
-docker manifest rm config.repo_url/image:tag
-2. Create the manifest
-docker manifest create config.repo_url/image:tag --amend config.repo_url/image:tag-amd64 --amend config.repo_url/image:tag-arm64
-3. Push the manifest
-docker manifest push config.repo_url/image:tag
-"""
-
-
-# This method calls docker directly on the command line, this is different from the rest of the code which uses
-# Sonar as an interface to docker. We decided to keep this asymmetry for now, as Sonar will be removed soon.
-
-
 def create_and_push_manifest(image: str, tag: str, architectures: list[str]) -> None:
+    """
+    Generates docker manifests by running the following commands:
+    1. Clear existing manifests
+    docker manifest rm config.repo_url/image:tag
+    2. Create the manifest
+    docker manifest create config.repo_url/image:tag --amend config.repo_url/image:tag-amd64 --amend config.repo_url/image:tag-arm64
+    3. Push the manifest
+    docker manifest push config.repo_url/image:tag
+
+    This method calls docker directly on the command line, this is different from the rest of the code which uses
+    Sonar as an interface to docker. We decided to keep this asymmetry for now, as Sonar will be removed soon.
+    """
     final_manifest = image + ":" + tag
 
     args = [
@@ -343,14 +341,12 @@ def try_get_platform_data(client, image):
         return None
 
 
-"""
-Checks if a docker image supports AMD and ARM platforms by inspecting the registry data.
-
-:param str image: The image name and tag
-"""
-
-
 def check_multi_arch(image: str, suffix: str) -> bool:
+    """
+    Checks if a docker image supports AMD and ARM platforms by inspecting the registry data.
+
+    :param str image: The image name and tag
+    """
     client = docker.from_env()
     platforms = ["linux/amd64", "linux/arm64"]
 
@@ -741,17 +737,6 @@ def submit(self, fn, *args, **kwargs):
             return super().submit(lambda: fn(*args, **kwargs))
 
 
-"""
-Starts the daily build process for an image. This function works for all images we support, for community and
-enterprise operator. The list of supported image_name is defined in get_builder_function_for_image_name.
-Builds an image for each version listed in ./release.json
-The registry used to pull base image and output the daily build is configured in the image_config function, it is passed
-as an argument to the inventories/daily.yaml file.
-
-If the context image supports both ARM and AMD architectures, both will be built.
-"""
-
-
 def should_skip_arm64():
     """
     Determines if arm64 builds should be skipped based on environment.
@@ -766,7 +751,15 @@ def build_image_daily(
     max_version: str = None,
     operator_version: str = None,
 ):
-    """Builds a daily image."""
+    """
+    Starts the daily build process for an image. This function works for all images we support, for community and
+    enterprise operator. The list of supported image_name is defined in get_builder_function_for_image_name.
+    Builds an image for each version listed in ./release.json
+    The registry used to pull base image and output the daily build is configured in the image_config function, it is passed
+    as an argument to the inventories/daily.yaml file.
+
+    If the context image supports both ARM and AMD architectures, both will be built.
+    """
 
     def get_architectures_set(build_configuration, args):
         """Determine the set of architectures to build for"""
@@ -992,6 +985,11 @@ def build_image_generic(
     multi_arch_args_list: list = None,
     is_run_in_parallel: bool = False,
 ):
+    """Build image generic builds context images and is used for triggering release. During releases
+    it signs and verifies the context image.
+    The release process uses the daily images build process.
+    """
+
     if not multi_arch_args_list:
         multi_arch_args_list = [extra_args or {}]
 
@@ -1011,8 +1009,12 @@ def build_image_generic(
             # But since we don't run daily rebuilds on ecr image builds, we can do that step instead here.
             # We only need to push manifests for multi-arch images.
             create_and_push_manifest(registry_address, version, architectures=architectures)
+
+    # Sign and verify the context image if on releases if requied.
     if config.sign and config.is_release_step_executed():
         sign_and_verify_context_image(registry, version)
+
+    # Release step. Release images via the daily image process.
     if config.is_release_step_executed() and version and QUAY_REGISTRY_URL in registry:
         logger.info(
             f"finished building context images, releasing them now via daily builds process for"
@@ -1546,11 +1548,12 @@ def calculate_images_to_build(
 
 def main():
     _setup_tracing()
+    _setup_tracing()
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--include", action="append")
-    parser.add_argument("--exclude", action="append")
-    parser.add_argument("--builder", default="docker", type=str)
+    parser.add_argument("--include", action="append", help="list of images to include")
+    parser.add_argument("--exclude", action="append", help="list of images to exclude")
+    parser.add_argument("--builder", default="docker", type=str, help="docker or podman")
     parser.add_argument("--list-images", action="store_true")
     parser.add_argument("--parallel", action="store_true", default=False)
     parser.add_argument("--debug", action="store_true", default=False)