From 6ec608efeed64297dfb345f5d341f0b47f120b62 Mon Sep 17 00:00:00 2001 From: Marius <24592972+gilbN@users.noreply.github.com> Date: Tue, 28 Oct 2025 22:57:34 +0100 Subject: [PATCH] Add commit sha, build number to create a build cache url for buildx sbom generation. Update readme. --- README.md | 4 +++- ci/ci.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- tests/test_ci.py | 14 +++++++++++++- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0d4df95..95d5ddc 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ This container is an automated testing tool for Docker images. It's designed to 1. **Spins up the container:** It runs the target Docker image with a specified tag. 2. **Checks for successful startup:** It tails the container's logs, waiting for the `[services.d] done.` message, which confirms the init system has finished and the services are running. -3. **Generates an SBOM:** It uses `syft` to create a Software Bill of Materials, providing a complete list of all packages inside the image. +3. **Generates an SBOM:** It uses `buildx imagetools inspect` or `syft`(fallback) to create a Software Bill of Materials, providing a complete list of all packages inside the image. 4. **Tests the Web UI (optional):** If the container runs a web service, it attempts to connect to the UI and take a screenshot to verify it's accessible and renders correctly. 5. **Generates a report:** It gathers all the results—container logs, build info, SBOM, screenshots, and test statuses—into a comprehensive HTML report. 6. **Uploads the report (CI only):** In a CI environment, it uploads the final report to an S3 bucket for review. @@ -115,6 +115,8 @@ sudo docker run --rm -i \ -e NODE_NAME= \ -e RELEASE_TAG= \ -e SYFT_IMAGE_TAG= \ +-e COMMIT_SHA= \ +-e BUILD_NUMBER= -t lsiodev/ci:latest \ python3 test_build.py ``` diff --git a/ci/ci.py b/ci/ci.py index 54a68c9..6d9f607 100755 --- a/ci/ci.py +++ b/ci/ci.py @@ -93,6 +93,13 @@ class Platform(enum.Enum): RISCV64 = "riscv64" UNKNOWN = "amd64" +class BuildCacheTag(enum.Enum): + """Enum for the different build cache tags""" + AMD64 = "amd64" + ARM64 = "arm64v8" + RISCV64 = "riscv64" + UNKNOWN = "amd64" + class SetEnvs(): """Simple helper class that sets up the ENVs""" def __init__(self) -> None: @@ -119,6 +126,9 @@ def __init__(self) -> None: self.bucket: str = os.environ.get("S3_BUCKET", "ci-tests.linuxserver.io") self.release_tag: str = os.environ.get("RELEASE_TAG", "latest") self.syft_image_tag: str = os.environ.get("SYFT_IMAGE_TAG", "v1.26.1") + self.commit_sha: str = os.environ.get("COMMIT_SHA", "") + self.build_number: str = os.environ.get("BUILD_NUMBER", "") + self.build_cache_registry: str = os.environ.get("BUILD_CACHE_REGISTRY", "ghcr.io/linuxserver/lsiodev-buildcache") if os.environ.get("DELAY_START"): self.logger.warning("DELAY_START env is obsolete, and not in use anymore") @@ -169,6 +179,9 @@ def __init__(self) -> None: S3_REGION: '{os.environ.get("S3_REGION")}' S3_BUCKET: '{os.environ.get("S3_BUCKET")}' SYFT_IMAGE_TAG: '{os.environ.get("SYFT_IMAGE_TAG")}' + COMMIT_SHA: '{os.environ.get("COMMIT_SHA")}' + BUILD_NUMBER: '{os.environ.get("BUILD_NUMBER")}' + BUILD_CACHE_REGISTRY: '{os.environ.get("BUILD_CACHE_REGISTRY")}' Docker Engine Version: '{self.get_docker_engine_version()}' """) self.logger.info(env_data) @@ -630,6 +643,39 @@ def get_sbom_syft(self, tag: str) -> str | CITestResult: self.logger.exception("Failed to remove the Syft container, %s",tag) return CITestResult.ERROR + def get_build_cache_url(self, tag: str) -> str: + """Get the build cache URL for the given tag. + + Args: + tag (str): The tag we are testing + Returns: + str: The build cache URL ex `ghcr.io/linuxserver/lsiodev-buildcache:arm64v8--` + """ + platform: str = self.get_build_cache_platform(tag) + build_cache_url: str = f"{self.build_cache_registry}:{platform}-{self.commit_sha}-{self.build_number}" + return build_cache_url + + def get_build_cache_platform(self, tag: str) -> str: + """Get the build cache platform for the given tag. + + Args: + tag (str): The tag we are testing + + Returns: + str: The build cache platform + """ + + platform = self.get_platform(tag) + match platform: + case Platform.AMD64.value: + return BuildCacheTag.AMD64.value + case Platform.ARM64.value: + return BuildCacheTag.ARM64.value + case Platform.RISCV64.value: + return BuildCacheTag.RISCV64.value + case _: + return BuildCacheTag.UNKNOWN.value + def get_sbom_buildx_blob(self, tag: str) -> str | CITestResult: """Get the SBOM for the image tag using docker buildx imagetools inspect. @@ -640,7 +686,7 @@ def get_sbom_buildx_blob(self, tag: str) -> str | CITestResult: str: SBOM output if successful, otherwise "CITestResult.ERROR". """ try: - image_ref = f"{self.image}:{tag}" + image_ref:str = self.get_build_cache_url(tag) self.logger.info("Generating SBOM for %s using buildx imagetools inspect", image_ref) cmd = f'docker buildx imagetools inspect {image_ref} --format "{{{{ json (index .SBOM).SPDX.packages }}}}"' result: subprocess.CompletedProcess[str] = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=self.sbom_timeout, check=False) diff --git a/tests/test_ci.py b/tests/test_ci.py index 834dd52..98dba30 100644 --- a/tests/test_ci.py +++ b/tests/test_ci.py @@ -8,7 +8,7 @@ from docker import DockerClient from moto import mock_aws -from ci.ci import CI, SetEnvs, CITestResult, CITests, Platform +from ci.ci import CI, SetEnvs, CITestResult, CITests, Platform, BuildCacheTag os.environ["DRY_RUN"] = "false" os.environ["IMAGE"] = "linuxserver/test" @@ -23,6 +23,8 @@ os.environ["PORT"] = "443" os.environ["WEB_SCREENSHOT"] = "true" os.environ["WEB_AUTH"] = "" +os.environ["COMMIT_SHA"] = "test-commit-sha" +os.environ["BUILD_NUMBER"] = "1234" @pytest.fixture def sbom_blob() -> bytes: @@ -241,3 +243,13 @@ def test_get_image_name(ci: CI) -> None: assert ci.get_image_name() == "linuxserver/lspipepr-plex" ci.image = "lsiobase/ubuntu" assert ci.get_image_name() == "linuxserver/docker-baseimage-ubuntu" + +def test_get_build_cache_url(ci: CI) -> None: + for tag in ci.tags: + cache_tag = ci.get_build_cache_platform(tag) + expected_url = f"{ci.build_cache_registry}:{cache_tag}-{ci.commit_sha}-{ci.build_number}" + assert ci.get_build_cache_url(tag) == expected_url + +def test_get_build_cache_platform(ci: CI) -> None: + assert ci.get_build_cache_platform(ci.tags[0]) == BuildCacheTag.AMD64.value + assert ci.get_build_cache_platform(ci.tags[1]) == BuildCacheTag.ARM64.value