From 0f2e583fa25d5ab3c4780e005a127c0f2f218eed Mon Sep 17 00:00:00 2001 From: Kevin Smith Date: Thu, 27 Oct 2022 11:17:19 -0400 Subject: [PATCH] Adding Argo Events --- gcp/terraform/main.tf | 7 +- gcp/terraform/services/argo.tf | 83 +++++++++++++++++-- .../argo_events/service_accounts.yaml | 31 +++++++ .../services/argo_events/webhook_source.yaml | 15 ++++ gcp/terraform/services/variables.tf | 30 ++++--- gcp/terraform/variables.tf | 29 ++++--- 6 files changed, 161 insertions(+), 34 deletions(-) create mode 100644 gcp/terraform/services/argo_events/service_accounts.yaml create mode 100644 gcp/terraform/services/argo_events/webhook_source.yaml diff --git a/gcp/terraform/main.tf b/gcp/terraform/main.tf index cf5f353..b735edf 100644 --- a/gcp/terraform/main.tf +++ b/gcp/terraform/main.tf @@ -45,8 +45,8 @@ data "google_sql_database_instance" "default" { } provider "kubernetes" { - host = "https://${data.google_container_cluster.default.endpoint}" - token = data.google_client_config.default.access_token + host = "https://${data.google_container_cluster.default.endpoint}" + token = data.google_client_config.default.access_token cluster_ca_certificate = base64decode( data.google_container_cluster.default.master_auth[0].cluster_ca_certificate, ) @@ -93,4 +93,5 @@ module "services" { metaflow_workload_identity_ksa_name = local.metaflow_workload_identity_ksa_name metadata_service_image = local.metadata_service_image kubeconfig_path = local_file.kubeconfig.filename -} \ No newline at end of file + deploy_argo_events = var.deploy_argo_events +} diff --git a/gcp/terraform/services/argo.tf b/gcp/terraform/services/argo.tf index 0b15654..0da5a08 100644 --- a/gcp/terraform/services/argo.tf +++ b/gcp/terraform/services/argo.tf @@ -4,11 +4,23 @@ resource "kubernetes_namespace" "argo" { } } +resource "kubernetes_namespace" "argo-events" { + count = var.deploy_argo_events ? 1 : 0 + metadata { + name = "argo-events" + } +} + locals { is_windows = substr(pathexpand("~"), 0, 1) == "/" ? false : true - _apply_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml" + _argo_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-workflows/master/manifests/quick-start-postgres.yaml" # we need to annotate the "argo" kubernetes service account for workload identity integration - _annotate_cmd = "kubectl annotate -n argo serviceaccount argo iam.gke.io/gcp-service-account=${var.metaflow_workload_identity_gsa_name}@${var.project}.iam.gserviceaccount.com" + _argo_annotate_cmd = "kubectl annotate -n argo serviceaccount argo iam.gke.io/gcp-service-account=${var.metaflow_workload_identity_gsa_name}@${var.project}.iam.gserviceaccount.com" + _argo_events_cmd = "kubectl apply -f https://raw.githubusercontent.com/argoproj/argo-events/v1.7.3/manifests/install.yaml" + _service_accts_cmd = "kubectl apply -n argo -f ${path.module}/argo_events/service_accounts.yaml" + _event_bus_cmd = "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo-events/v1.7.3/examples/eventbus/native.yaml" + _webhook_source_cmd = "kubectl apply -n argo -f ${path.module}/argo_events/webhook_source.yaml" + _argo_events_annotate_cmd = "kubectl annotate -n argo serviceaccount operate-workflow-sa iam.gke.io/gcp-service-account=${var.metaflow_workload_identity_gsa_name}@${var.project}.iam.gserviceaccount.com" } # Yes local-exec is unfortunate. @@ -17,21 +29,80 @@ locals { # The main challenge is that the Argo yaml contains multiple k8s resources, and terraform does not accept that natively. resource "null_resource" "argo-quick-start-installation" { triggers = { - cmd = local._apply_cmd + cmd = local._argo_cmd } provisioner "local-exec" { interpreter = local.is_windows ? ["PowerShell"] : null - command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._apply_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._apply_cmd}" + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._argo_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._argo_cmd}" } } resource "null_resource" "argo-annotate-service-account" { depends_on = [null_resource.argo-quick-start-installation] triggers = { - cmd = local._annotate_cmd + cmd = local._argo_annotate_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._argo_annotate_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._argo_annotate_cmd}" + } +} + +resource "null_resource" "argo-events-quick-start" { + count = var.deploy_argo_events ? 1 : 0 + depends_on = [null_resource.argo-quick-start-installation] + triggers = { + cmd = local._argo_events_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._argo_events_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._argo_events_cmd}" + } +} + +resource "null_resource" "argo-events-service-account" { + depends_on = [null_resource.argo-events-quick-start] + count = var.deploy_argo_events ? 1 : 0 + triggers = { + cmd = local._service_accts_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._service_accts_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._service_accts_cmd}" + } +} + +resource "null_resource" "argo-events-event-bus" { + depends_on = [null_resource.argo-events-quick-start] + count = var.deploy_argo_events ? 1 : 0 + triggers = { + cmd = local._event_bus_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._event_bus_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._event_bus_cmd}" + } +} + +resource "null_resource" "argo-events-webhook-source" { + depends_on = [null_resource.argo-events-quick-start] + count = var.deploy_argo_events ? 1 : 0 + triggers = { + cmd = local._webhook_source_cmd + } + provisioner "local-exec" { + interpreter = local.is_windows ? ["PowerShell"] : null + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._webhook_source_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._webhook_source_cmd}" + } +} + +resource "null_resource" "argo-events-annotate-service-account" { + depends_on = [null_resource.argo-events-service-account] + triggers = { + cmd = local._argo_events_annotate_cmd } provisioner "local-exec" { interpreter = local.is_windows ? ["PowerShell"] : null - command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._annotate_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._annotate_cmd}" + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._argo_events_annotate_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._argo_events_annotate_cmd}" } } diff --git a/gcp/terraform/services/argo_events/service_accounts.yaml b/gcp/terraform/services/argo_events/service_accounts.yaml new file mode 100644 index 0000000..7cee29e --- /dev/null +++ b/gcp/terraform/services/argo_events/service_accounts.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operate-workflow-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operate-workflow-role +rules: + - apiGroups: + - argoproj.io + verbs: + - "*" + resources: + - workflows + - workflowtemplates + - cronworkflows + - clusterworkflowtemplates +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operate-workflow-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operate-workflow-role +subjects: + - kind: ServiceAccount + name: operate-workflow-sa \ No newline at end of file diff --git a/gcp/terraform/services/argo_events/webhook_source.yaml b/gcp/terraform/services/argo_events/webhook_source.yaml new file mode 100644 index 0000000..3ba72c5 --- /dev/null +++ b/gcp/terraform/services/argo_events/webhook_source.yaml @@ -0,0 +1,15 @@ +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: + name: metaflow-webhook + namespace: argo +spec: + service: + ports: + - port: 12000 + targetPort: 12000 + webhook: + event: + port: "12000" + endpoint: /event + method: POST \ No newline at end of file diff --git a/gcp/terraform/services/variables.tf b/gcp/terraform/services/variables.tf index 10565ad..a079024 100644 --- a/gcp/terraform/services/variables.tf +++ b/gcp/terraform/services/variables.tf @@ -1,40 +1,40 @@ -variable metaflow_ui_static_service_image { +variable "metaflow_ui_static_service_image" { type = string } -variable metaflow_datastore_sysroot_gs { +variable "metaflow_datastore_sysroot_gs" { type = string } -variable metaflow_db_name { +variable "metaflow_db_name" { type = string } -variable metaflow_db_user { +variable "metaflow_db_user" { type = string } -variable metaflow_db_host { +variable "metaflow_db_host" { type = string } -variable metaflow_ui_backend_service_image { +variable "metaflow_ui_backend_service_image" { type = string } -variable metaflow_db_port { +variable "metaflow_db_port" { type = string } -variable metaflow_db_password { +variable "metaflow_db_password" { type = string } -variable project { +variable "project" { type = string } -variable db_connection_name { +variable "db_connection_name" { type = string } @@ -42,18 +42,22 @@ variable "metaflow_workload_identity_gsa_name" { type = string } -variable metaflow_workload_identity_gsa_id { +variable "metaflow_workload_identity_gsa_id" { type = string } -variable metaflow_workload_identity_ksa_name { +variable "metaflow_workload_identity_ksa_name" { type = string } -variable metadata_service_image { +variable "metadata_service_image" { type = string } variable "kubeconfig_path" { type = string } + +variable "deploy_argo_events" { + type = bool +} diff --git a/gcp/terraform/variables.tf b/gcp/terraform/variables.tf index ae1d06b..c5d21bb 100644 --- a/gcp/terraform/variables.tf +++ b/gcp/terraform/variables.tf @@ -1,4 +1,4 @@ -resource random_id database_server_name_suffix { +resource "random_id" "database_server_name_suffix" { byte_length = 4 keepers = { db_generation_number = var.db_generation_number @@ -8,18 +8,18 @@ resource random_id database_server_name_suffix { locals { database_server_name_prefix = "psql-metaflow-${terraform.workspace}" - database_server_name = "${local.database_server_name_prefix}-${random_id.database_server_name_suffix.hex}" - kubernetes_cluster_name = "gke-metaflow-${terraform.workspace}" - region = "us-west2" - zone = "us-west2-a" + database_server_name = "${local.database_server_name_prefix}-${random_id.database_server_name_suffix.hex}" + kubernetes_cluster_name = "gke-metaflow-${terraform.workspace}" + region = "us-west2" + zone = "us-west2-a" - storage_bucket_name = "storage-${var.org_prefix}-metaflow-${terraform.workspace}" + storage_bucket_name = "storage-${var.org_prefix}-metaflow-${terraform.workspace}" metaflow_datastore_sysroot_gs = "gs://${local.storage_bucket_name}/tf-full-stack-sysroot" metaflow_ui_static_service_image = "public.ecr.aws/outerbounds/metaflow_ui:v1.1.4" # metaflow_ui_backend_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" metaflow_ui_backend_service_image = "jackieob/metadata_service:gcp.rc1" - metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" + metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" # TODO gsa-metaflow-workload-id- metaflow_workload_identity_gsa_name = "gsa-metaflow-${terraform.workspace}" @@ -27,15 +27,20 @@ locals { service_account_key_file = "${path.root}/metaflow_gsa_key_${terraform.workspace}.json" } -variable project { +variable "project" { type = string } -variable org_prefix { +variable "org_prefix" { type = string } -variable db_generation_number { - type = number +variable "db_generation_number" { + type = number default = 0 -} \ No newline at end of file +} + +variable "deploy_argo_events" { + type = bool + default = true +}