From c211b091334215d66e78b98b6e1fb2980608a0ff Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Fri, 28 Nov 2025 15:33:47 +0100 Subject: [PATCH 1/3] Add regional location support to GCP Image Builder Users can now specify a `location` parameter to run Cloud Build in a specific GCP region instead of the global endpoint. This enables: - Data residency compliance (GDPR, etc.) - Lower latency when building near GCS buckets/Artifact Registry - Required for Cloud Build private pools The Service Connector region does not control Cloud Build region, so this must be set explicitly on the Image Builder component. --- docs/book/component-guide/image-builders/gcp.md | 12 ++++++++++-- .../gcp/flavors/gcp_image_builder_flavor.py | 15 ++++++++++++++- .../gcp/image_builders/gcp_image_builder.py | 17 ++++++++++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/book/component-guide/image-builders/gcp.md b/docs/book/component-guide/image-builders/gcp.md index 54b1673024d..445f49d83dc 100644 --- a/docs/book/component-guide/image-builders/gcp.md +++ b/docs/book/component-guide/image-builders/gcp.md @@ -38,6 +38,11 @@ To use the Google Cloud image builder, we need: * the Docker image used by Google Cloud Build to execute the steps to build and push the Docker image. By default, the builder image will be `'gcr.io/cloud-builders/docker'`. * The network to which the container used to build the ZenML pipeline Docker image will be attached. More information: [Cloud build network](https://cloud.google.com/build/docs/build-config-file-schema#network). * The build timeout for the build, and for the blocking operation waiting for the build to finish. More information: [Build Timeout](https://cloud.google.com/build/docs/build-config-file-schema#timeout_2). + * The location to run Cloud Build (e.g., `us-central1`, `europe-west1`) when you need regional data residency, lower latency to nearby GCS buckets or Artifact Registry, or to use Cloud Build private pools. + +{% hint style="info" %} +Even if your GCP Service Connector is scoped to a specific region, the GCP Image Builder uses the **global** Cloud Build endpoint by default. To run builds in a specific region, set the `location` parameter on the Image Builder. The Service Connector only supplies authentication and does not influence which Cloud Build region is used. +{% endhint %} We can register the image builder and use it in our active stack: @@ -46,7 +51,8 @@ zenml image-builder register \ --flavor=gcp \ --cloud_builder_image= \ --network= \ - --build_timeout= + --build_timeout= \ + --location= # Register and activate a stack with the new image builder zenml stack register -i ... --set @@ -127,7 +133,8 @@ zenml image-builder register \ --flavor=gcp \ --cloud_builder_image= \ --network= \ - --build_timeout= + --build_timeout= \ + --location= # Connect the GCP Image Builder to GCP via a GCP Service Connector zenml image-builder connect -i @@ -175,6 +182,7 @@ zenml image-builder register \ --service_account_path= \ --cloud_builder_image= \ --network= \ + --location= \ --build_timeout= # Register and set a stack with the new image builder diff --git a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py index 63de9cac3e2..a5b4032d53d 100644 --- a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py +++ b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Optional, Type -from pydantic import PositiveInt +from pydantic import Field, PositiveInt from zenml.image_builders import BaseImageBuilderConfig, BaseImageBuilderFlavor from zenml.integrations.gcp import ( @@ -53,11 +53,24 @@ class GCPImageBuilderConfig( about this parameter: https://cloud.google.com/build/docs/build-config-file-schema#timeout_2 Defaults to `3600`. + location: Optional GCP region for running Cloud Build (e.g., + 'us-central1', 'europe-west1'). Controls data residency and latency + and is required when using Cloud Build private pools. If not set, + the global endpoint is used. """ cloud_builder_image: str = DEFAULT_CLOUD_BUILDER_IMAGE network: str = DEFAULT_CLOUD_BUILDER_NETWORK build_timeout: PositiveInt = DEFAULT_CLOUD_BUILD_TIMEOUT + location: Optional[str] = Field( + default=None, + description=( + "GCP region for Cloud Build execution to control data residency and " + "latency. Examples: 'us-central1', 'europe-west1'. Required when " + "using Cloud Build private pools. If omitted, the global Cloud Build " + "endpoint is used." + ), + ) class GCPImageBuilderFlavor(BaseImageBuilderFlavor): diff --git a/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py b/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py index b83bda7e291..7721b62287a 100644 --- a/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py +++ b/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast from urllib.parse import urlparse +from google.api_core.client_options import ClientOptions from google.cloud.devtools import cloudbuild_v1 from zenml.enums import StackComponentType @@ -223,7 +224,21 @@ def _run_cloud_build(self, build: cloudbuild_v1.Build) -> str: RuntimeError: If the Cloud Build run has failed. """ credentials, project_id = self._get_authentication() - client = cloudbuild_v1.CloudBuildClient(credentials=credentials) + client_options = None + if self.config.location: + location = self.config.location.strip() + endpoint = f"{location}-cloudbuild.googleapis.com" + client_options = ClientOptions(api_endpoint=endpoint) + logger.info( + "Using regional Cloud Build endpoint `%s`.", + endpoint, + ) + else: + logger.info("Using global Cloud Build endpoint.") + + client = cloudbuild_v1.CloudBuildClient( + credentials=credentials, client_options=client_options + ) operation = client.create_build(project_id=project_id, build=build) log_url = operation.metadata.build.log_url From 579b674813ba1406aee764467f44d68746b424ab Mon Sep 17 00:00:00 2001 From: Alex Strick van Linschoten Date: Mon, 1 Dec 2025 15:11:40 +0100 Subject: [PATCH 2/3] Add validator to normalize empty location strings Adds a Pydantic field_validator that strips whitespace and converts empty strings to None at parse time. This prevents edge cases where location="" would produce an invalid endpoint like "-cloudbuild.googleapis.com". --- .../gcp/flavors/gcp_image_builder_flavor.py | 12 +++++++++++- .../gcp/image_builders/gcp_image_builder.py | 3 +-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py index a5b4032d53d..f90758f411e 100644 --- a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py +++ b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Optional, Type -from pydantic import Field, PositiveInt +from pydantic import Field, PositiveInt, field_validator from zenml.image_builders import BaseImageBuilderConfig, BaseImageBuilderFlavor from zenml.integrations.gcp import ( @@ -72,6 +72,16 @@ class GCPImageBuilderConfig( ), ) + @field_validator("location", mode="before") + @classmethod + def validate_location(cls, v: Optional[str]) -> Optional[str]: + """Normalize location field, treating empty strings as unset.""" + if v is not None: + v = v.strip() + if not v: + return None + return v + class GCPImageBuilderFlavor(BaseImageBuilderFlavor): """Google Cloud Builder image builder flavor.""" diff --git a/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py b/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py index 7721b62287a..b0c035d9ce1 100644 --- a/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py +++ b/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py @@ -226,8 +226,7 @@ def _run_cloud_build(self, build: cloudbuild_v1.Build) -> str: credentials, project_id = self._get_authentication() client_options = None if self.config.location: - location = self.config.location.strip() - endpoint = f"{location}-cloudbuild.googleapis.com" + endpoint = f"{self.config.location}-cloudbuild.googleapis.com" client_options = ClientOptions(api_endpoint=endpoint) logger.info( "Using regional Cloud Build endpoint `%s`.", From 0c723295fa9711e23fda6016d4ae2c74f1a4ac52 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Tue, 2 Dec 2025 09:23:51 +0800 Subject: [PATCH 3/3] Docstring --- .../integrations/gcp/flavors/gcp_image_builder_flavor.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py index f90758f411e..8002876c797 100644 --- a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py +++ b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py @@ -75,7 +75,14 @@ class GCPImageBuilderConfig( @field_validator("location", mode="before") @classmethod def validate_location(cls, v: Optional[str]) -> Optional[str]: - """Normalize location field, treating empty strings as unset.""" + """Normalize location field, treating empty strings as unset. + + Args: + v: The location to validate. + + Returns: + The validated location. + """ if v is not None: v = v.strip() if not v: