Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions docs/book/component-guide/image-builders/gcp.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ To use the Google Cloud image builder, we need:
* the Docker image used by Google Cloud Build to execute the steps to build and push the Docker image. By default, the builder image will be `'gcr.io/cloud-builders/docker'`.
* The network to which the container used to build the ZenML pipeline Docker image will be attached. More information: [Cloud build network](https://cloud.google.com/build/docs/build-config-file-schema#network).
* The build timeout for the build, and for the blocking operation waiting for the build to finish. More information: [Build Timeout](https://cloud.google.com/build/docs/build-config-file-schema#timeout_2).
* The location to run Cloud Build (e.g., `us-central1`, `europe-west1`) when you need regional data residency, lower latency to nearby GCS buckets or Artifact Registry, or to use Cloud Build private pools.

{% hint style="info" %}
Even if your GCP Service Connector is scoped to a specific region, the GCP Image Builder uses the **global** Cloud Build endpoint by default. To run builds in a specific region, set the `location` parameter on the Image Builder. The Service Connector only supplies authentication and does not influence which Cloud Build region is used.
{% endhint %}

We can register the image builder and use it in our active stack:

Expand All @@ -46,7 +51,8 @@ zenml image-builder register <IMAGE_BUILDER_NAME> \
--flavor=gcp \
--cloud_builder_image=<BUILDER_IMAGE_NAME> \
--network=<DOCKER_NETWORK> \
--build_timeout=<BUILD_TIMEOUT_IN_SECONDS>
--build_timeout=<BUILD_TIMEOUT_IN_SECONDS> \
--location=<GCP_REGION>

# Register and activate a stack with the new image builder
zenml stack register <STACK_NAME> -i <IMAGE_BUILDER_NAME> ... --set
Expand Down Expand Up @@ -127,7 +133,8 @@ zenml image-builder register <IMAGE_BUILDER_NAME> \
--flavor=gcp \
--cloud_builder_image=<BUILDER_IMAGE_NAME> \
--network=<DOCKER_NETWORK> \
--build_timeout=<BUILD_TIMEOUT_IN_SECONDS>
--build_timeout=<BUILD_TIMEOUT_IN_SECONDS> \
--location=<GCP_REGION>

# Connect the GCP Image Builder to GCP via a GCP Service Connector
zenml image-builder connect <IMAGE_BUILDER_NAME> -i
Expand Down Expand Up @@ -175,6 +182,7 @@ zenml image-builder register <IMAGE_BUILDER_NAME> \
--service_account_path=<PATH_TO_SERVICE_ACCOUNT_KEY> \
--cloud_builder_image=<BUILDER_IMAGE_NAME> \
--network=<DOCKER_NETWORK> \
--location=<GCP_REGION> \
--build_timeout=<BUILD_TIMEOUT_IN_SECONDS>

# Register and set a stack with the new image builder
Expand Down
32 changes: 31 additions & 1 deletion src/zenml/integrations/gcp/flavors/gcp_image_builder_flavor.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from typing import TYPE_CHECKING, Optional, Type

from pydantic import PositiveInt
from pydantic import Field, PositiveInt, field_validator

from zenml.image_builders import BaseImageBuilderConfig, BaseImageBuilderFlavor
from zenml.integrations.gcp import (
Expand Down Expand Up @@ -53,11 +53,41 @@ class GCPImageBuilderConfig(
about this parameter:
https://cloud.google.com/build/docs/build-config-file-schema#timeout_2
Defaults to `3600`.
location: Optional GCP region for running Cloud Build (e.g.,
'us-central1', 'europe-west1'). Controls data residency and latency
and is required when using Cloud Build private pools. If not set,
the global endpoint is used.
"""

cloud_builder_image: str = DEFAULT_CLOUD_BUILDER_IMAGE
network: str = DEFAULT_CLOUD_BUILDER_NETWORK
build_timeout: PositiveInt = DEFAULT_CLOUD_BUILD_TIMEOUT
location: Optional[str] = Field(
default=None,
description=(
"GCP region for Cloud Build execution to control data residency and "
"latency. Examples: 'us-central1', 'europe-west1'. Required when "
"using Cloud Build private pools. If omitted, the global Cloud Build "
"endpoint is used."
),
)

@field_validator("location", mode="before")
@classmethod
def validate_location(cls, v: Optional[str]) -> Optional[str]:
"""Normalize location field, treating empty strings as unset.

Args:
v: The location to validate.

Returns:
The validated location.
"""
if v is not None:
v = v.strip()
if not v:
return None
return v


class GCPImageBuilderFlavor(BaseImageBuilderFlavor):
Expand Down
16 changes: 15 additions & 1 deletion src/zenml/integrations/gcp/image_builders/gcp_image_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from typing import TYPE_CHECKING, Optional, Tuple, cast
from urllib.parse import urlparse

from google.api_core.client_options import ClientOptions
from google.cloud.devtools import cloudbuild_v1

from zenml.enums import StackComponentType
Expand Down Expand Up @@ -216,7 +217,20 @@ def _run_cloud_build(self, build: cloudbuild_v1.Build) -> str:
RuntimeError: If the Cloud Build run has failed.
"""
credentials, project_id = self._get_authentication()
client = cloudbuild_v1.CloudBuildClient(credentials=credentials)
client_options = None
if self.config.location:
endpoint = f"{self.config.location}-cloudbuild.googleapis.com"
client_options = ClientOptions(api_endpoint=endpoint)
logger.info(
"Using regional Cloud Build endpoint `%s`.",
endpoint,
)
else:
logger.info("Using global Cloud Build endpoint.")

client = cloudbuild_v1.CloudBuildClient(
credentials=credentials, client_options=client_options
)

operation = client.create_build(project_id=project_id, build=build)
log_url = operation.metadata.build.log_url
Expand Down
Loading