diff --git a/docs/book/component-guide/image-builders/gcp.md b/docs/book/component-guide/image-builders/gcp.md index 54b1673024d..445f49d83dc 100644 --- a/docs/book/component-guide/image-builders/gcp.md +++ b/docs/book/component-guide/image-builders/gcp.md @@ -38,6 +38,11 @@ To use the Google Cloud image builder, we need: * the Docker image used by Google Cloud Build to execute the steps to build and push the Docker image. By default, the builder image will be `'gcr.io/cloud-builders/docker'`. * The network to which the container used to build the ZenML pipeline Docker image will be attached. More information: [Cloud build network](https://cloud.google.com/build/docs/build-config-file-schema#network). * The build timeout for the build, and for the blocking operation waiting for the build to finish. More information: [Build Timeout](https://cloud.google.com/build/docs/build-config-file-schema#timeout_2). + * The location to run Cloud Build (e.g., `us-central1`, `europe-west1`) when you need regional data residency, lower latency to nearby GCS buckets or Artifact Registry, or to use Cloud Build private pools. + +{% hint style="info" %} +Even if your GCP Service Connector is scoped to a specific region, the GCP Image Builder uses the **global** Cloud Build endpoint by default. To run builds in a specific region, set the `location` parameter on the Image Builder. The Service Connector only supplies authentication and does not influence which Cloud Build region is used. +{% endhint %} We can register the image builder and use it in our active stack: @@ -46,7 +51,8 @@ zenml image-builder register \ --flavor=gcp \ --cloud_builder_image= \ --network= \ - --build_timeout= + --build_timeout= \ + --location= # Register and activate a stack with the new image builder zenml stack register -i ... --set @@ -127,7 +133,8 @@ zenml image-builder register \ --flavor=gcp \ --cloud_builder_image= \ --network= \ - --build_timeout= + --build_timeout= \ + --location= # Connect the GCP Image Builder to GCP via a GCP Service Connector zenml image-builder connect -i @@ -175,6 +182,7 @@ zenml image-builder register \ --service_account_path= \ --cloud_builder_image= \ --network= \ + --location= \ --build_timeout= # Register and set a stack with the new image builder diff --git a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py index 63de9cac3e2..8002876c797 100644 --- a/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py +++ b/src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Optional, Type -from pydantic import PositiveInt +from pydantic import Field, PositiveInt, field_validator from zenml.image_builders import BaseImageBuilderConfig, BaseImageBuilderFlavor from zenml.integrations.gcp import ( @@ -53,11 +53,41 @@ class GCPImageBuilderConfig( about this parameter: https://cloud.google.com/build/docs/build-config-file-schema#timeout_2 Defaults to `3600`. + location: Optional GCP region for running Cloud Build (e.g., + 'us-central1', 'europe-west1'). Controls data residency and latency + and is required when using Cloud Build private pools. If not set, + the global endpoint is used. """ cloud_builder_image: str = DEFAULT_CLOUD_BUILDER_IMAGE network: str = DEFAULT_CLOUD_BUILDER_NETWORK build_timeout: PositiveInt = DEFAULT_CLOUD_BUILD_TIMEOUT + location: Optional[str] = Field( + default=None, + description=( + "GCP region for Cloud Build execution to control data residency and " + "latency. Examples: 'us-central1', 'europe-west1'. Required when " + "using Cloud Build private pools. If omitted, the global Cloud Build " + "endpoint is used." + ), + ) + + @field_validator("location", mode="before") + @classmethod + def validate_location(cls, v: Optional[str]) -> Optional[str]: + """Normalize location field, treating empty strings as unset. + + Args: + v: The location to validate. + + Returns: + The validated location. + """ + if v is not None: + v = v.strip() + if not v: + return None + return v class GCPImageBuilderFlavor(BaseImageBuilderFlavor): diff --git a/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py b/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py index 2f025dff4a1..7c71b7bc85c 100644 --- a/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py +++ b/src/zenml/integrations/gcp/image_builders/gcp_image_builder.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Optional, Tuple, cast from urllib.parse import urlparse +from google.api_core.client_options import ClientOptions from google.cloud.devtools import cloudbuild_v1 from zenml.enums import StackComponentType @@ -216,7 +217,20 @@ def _run_cloud_build(self, build: cloudbuild_v1.Build) -> str: RuntimeError: If the Cloud Build run has failed. """ credentials, project_id = self._get_authentication() - client = cloudbuild_v1.CloudBuildClient(credentials=credentials) + client_options = None + if self.config.location: + endpoint = f"{self.config.location}-cloudbuild.googleapis.com" + client_options = ClientOptions(api_endpoint=endpoint) + logger.info( + "Using regional Cloud Build endpoint `%s`.", + endpoint, + ) + else: + logger.info("Using global Cloud Build endpoint.") + + client = cloudbuild_v1.CloudBuildClient( + credentials=credentials, client_options=client_options + ) operation = client.create_build(project_id=project_id, build=build) log_url = operation.metadata.build.log_url