feat: Configure cloudwatch logging for step functions module (#9)

Author: ahummel25

.github/workflows/pre-commit.yml

@@ -94,7 +94,7 @@ jobs:
       - name: Install pre-commit dependencies
         run: |
           pip install pre-commit
-          curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E "https://.+?-v0.12.0-linux-amd64" | head -n1)" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/
+          curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E "https://.+?-v0.12\..+?-linux-amd64" | head -n1)" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/
           curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/
       - name: Execute pre-commit
         # Run all pre-commit checks on max version supported

README.md

@@ -4,18 +4,16 @@ Terraform module, which creates AWS Step Functions as well as required IAM role
 
 This Terraform module is the part of [serverless.tf framework](https://github.com/antonbabenko/serverless.tf), which aims to simplify all operations when working with the serverless in Terraform.
 
-
 ## Features
 
 - [x] Creates AWS Step Function
 - [x] Conditional creation for many types of resources
 - [x] Support IAM policy attachments for [Integrated Services (eg, Lambda, SQS, ECS, EKS, Batch, DynamoDB, etc)](https://docs.aws.amazon.com/step-functions/latest/dg/service-integration-iam-templates.html) and various ways to create and attach additional policies
 
-
 ## Usage
 
 ### Step Function
-  
+
 ```hcl
 module "step_function" {
   source = "terraform-aws-modules/step-functions/aws"
@@ -39,7 +37,7 @@ module "step_function" {
   }
 }
 EOF
-  
+
   service_integrations = {
     dynamodb = {
       dynamodb = ["arn:aws:dynamodb:eu-west-1:052212379155:table/Test"]
@@ -79,7 +77,7 @@ module "step_function" {
     sqs = {
       sqs = "arn:aws:sqs:..."  # sqs queue ARN is required because there is no default_resources key for such integration
     }
- 
+
     # Special case to deny all actions for the step function (this will override all IAM policies allowed for the function)
     no_tasks = {
       deny_all = true
@@ -88,7 +86,6 @@ module "step_function" {
 }
 ```
 
-
 ## Additional IAM policies for Step Function
 
 In addition to all supported AWS service integrations you may want to create and attach additional policies.
@@ -101,7 +98,6 @@ There are 5 supported ways to attach additional IAM policies to IAM role used by
 1. `policies` - List of ARNs of existing IAM policies, when `attach_policies = true` and `number_of_policies > 0`.
 1. `policy_statements` - Map of maps to define IAM statements which will be generated as IAM policy. Requires `attach_policy_statements = true`. See `examples/complete` for more information.
 
-
 ## Conditional creation
 
 Sometimes you need to have a way to create resources conditionally, so the solution is to specify `create` arguments.
@@ -117,11 +113,9 @@ module "step_function" {
 }
 ```
 
-
 ## Examples
 
-* [Complete](https://github.com/terraform-aws-modules/terraform-aws-step-functions/tree/master/examples/complete) - Create Step Function and required IAM resources in various combinations with all supported features.
-
+- [Complete](https://github.com/terraform-aws-modules/terraform-aws-step-functions/tree/master/examples/complete) - Create Step Function and required IAM resources in various combinations with all supported features.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
@@ -145,37 +139,48 @@ No modules.
 
 | Name | Type |
 |------|------|
+| [aws_cloudwatch_log_group.sfn](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_iam_policy.additional_inline](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.additional_json](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.additional_jsons](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy_attachment.additional_inline](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
 | [aws_iam_policy_attachment.additional_json](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
 | [aws_iam_policy_attachment.additional_jsons](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
+| [aws_iam_policy_attachment.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
 | [aws_iam_policy_attachment.service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource |
 | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.additional_many](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.additional_one](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_sfn_state_machine.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sfn_state_machine) | resource |
+| [aws_cloudwatch_log_group.sfn](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/cloudwatch_log_group) | data source |
 | [aws_iam_policy_document.additional_inline](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_attach_cloudwatch_logs_policy"></a> [attach\_cloudwatch\_logs\_policy](#input\_attach\_cloudwatch\_logs\_policy) | Controls whether CloudWatch Logs policy should be added to IAM role for Lambda Function | `bool` | `true` | no |
 | <a name="input_attach_policies"></a> [attach\_policies](#input\_attach\_policies) | Controls whether list of policies should be added to IAM role | `bool` | `false` | no |
 | <a name="input_attach_policies_for_integrations"></a> [attach\_policies\_for\_integrations](#input\_attach\_policies\_for\_integrations) | Whether to attach AWS Service policies to IAM role | `bool` | `true` | no |
 | <a name="input_attach_policy"></a> [attach\_policy](#input\_attach\_policy) | Controls whether policy should be added to IAM role | `bool` | `false` | no |
 | <a name="input_attach_policy_json"></a> [attach\_policy\_json](#input\_attach\_policy\_json) | Controls whether policy\_json should be added to IAM role | `bool` | `false` | no |
 | <a name="input_attach_policy_jsons"></a> [attach\_policy\_jsons](#input\_attach\_policy\_jsons) | Controls whether policy\_jsons should be added to IAM role | `bool` | `false` | no |
 | <a name="input_attach_policy_statements"></a> [attach\_policy\_statements](#input\_attach\_policy\_statements) | Controls whether policy\_statements should be added to IAM role | `bool` | `false` | no |
 | <a name="input_aws_region_assume_role"></a> [aws\_region\_assume\_role](#input\_aws\_region\_assume\_role) | Name of AWS regions where IAM role can be assumed by the Step Function | `string` | `""` | no |
+| <a name="input_cloudwatch_log_group_kms_key_id"></a> [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | The ARN of the KMS Key to use when encrypting log data. | `string` | `null` | no |
+| <a name="input_cloudwatch_log_group_name"></a> [cloudwatch\_log\_group\_name](#input\_cloudwatch\_log\_group\_name) | Name of Cloudwatch Logs group name to use. | `string` | `null` | no |
+| <a name="input_cloudwatch_log_group_retention_in_days"></a> [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Specifies the number of days you want to retain log events in the specified log group. Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `null` | no |
+| <a name="input_cloudwatch_log_group_tags"></a> [cloudwatch\_log\_group\_tags](#input\_cloudwatch\_log\_group\_tags) | A map of tags to assign to the resource. | `map(string)` | `{}` | no |
 | <a name="input_create"></a> [create](#input\_create) | Whether to create Step Function resource | `bool` | `true` | no |
 | <a name="input_create_role"></a> [create\_role](#input\_create\_role) | Whether to create IAM role for the Step Function | `bool` | `true` | no |
 | <a name="input_definition"></a> [definition](#input\_definition) | The Amazon States Language definition of the Step Function | `string` | `""` | no |
+| <a name="input_logging_configuration"></a> [logging\_configuration](#input\_logging\_configuration) | Defines what execution history events are logged and where they are logged | `map(string)` | `{}` | no |
 | <a name="input_name"></a> [name](#input\_name) | The name of the Step Function | `string` | `""` | no |
 | <a name="input_number_of_policies"></a> [number\_of\_policies](#input\_number\_of\_policies) | Number of policies to attach to IAM role | `number` | `0` | no |
 | <a name="input_number_of_policy_jsons"></a> [number\_of\_policy\_jsons](#input\_number\_of\_policy\_jsons) | Number of policies JSON to attach to IAM role | `number` | `0` | no |
@@ -195,6 +200,7 @@ No modules.
 | <a name="input_tags"></a> [tags](#input\_tags) | Maps of tags to assign to the Step Function | `map(string)` | `{}` | no |
 | <a name="input_trusted_entities"></a> [trusted\_entities](#input\_trusted\_entities) | Step Function additional trusted entities for assuming roles (trust relationship) | `list(string)` | `[]` | no |
 | <a name="input_type"></a> [type](#input\_type) | Determines whether a Standard or Express state machine is created. The default is STANDARD. Valid Values: STANDARD \| EXPRESS | `string` | `"STANDARD"` | no |
+| <a name="input_use_existing_cloudwatch_log_group"></a> [use\_existing\_cloudwatch\_log\_group](#input\_use\_existing\_cloudwatch\_log\_group) | Whether to use an existing CloudWatch log group or create new | `bool` | `false` | no |
 | <a name="input_use_existing_role"></a> [use\_existing\_role](#input\_use\_existing\_role) | Whether to use an existing IAM role for this Step Function | `bool` | `false` | no |
 
 ## Outputs
@@ -215,7 +221,6 @@ Module managed by [Anton Babenko](https://github.com/antonbabenko). Check out [s
 
 Please reach out to [Betajob](https://www.betajob.com/) if you are looking for commercial support for your Terraform, AWS, or serverless project.
 
-
 ## License
 
 Apache 2 Licensed. See LICENSE for full details.

examples/complete/README.md

@@ -39,11 +39,13 @@ Note that this example may create resources which cost money. Run `terraform des
 |------|--------|---------|
 | <a name="module_disabled_step_function"></a> [disabled\_step\_function](#module\_disabled\_step\_function) | ../../ |  |
 | <a name="module_step_function"></a> [step\_function](#module\_step\_function) | ../../ |  |
+| <a name="module_step_function_with_existing_log_group"></a> [step\_function\_with\_existing\_log\_group](#module\_step\_function\_with\_existing\_log\_group) | ../../ |  |
 
 ## Resources
 
 | Name | Type |
 |------|------|
+| [aws_cloudwatch_log_group.external](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_sqs_queue.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
 | [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
 

examples/complete/main.tf

@@ -30,7 +30,6 @@ locals {
 EOF
 }
 
-
 module "step_function" {
   source = "../../"
 
@@ -40,6 +39,11 @@ module "step_function" {
 
   definition = local.definition_template
 
+  logging_configuration = {
+    include_execution_data = true
+    level                  = "ALL"
+  }
+
   service_integrations = {
 
     dynamodb = {
@@ -61,9 +65,10 @@ module "step_function" {
       xray = true
     }
 
-    no_tasks = {
-      deny_all = true
-    }
+    #    # NB: This will "Deny" everything (including logging)!
+    #    no_tasks = {
+    #      deny_all = true
+    #    }
   }
 
   ######################
@@ -135,6 +140,34 @@ EOF
   }
 }
 
+###############################################
+# With CloudWatch log group created externally
+###############################################
+
+resource "aws_cloudwatch_log_group" "external" {
+  name = "${random_pet.this.id}-my-log-group"
+}
+
+module "step_function_with_existing_log_group" {
+  source = "../../"
+
+  name = "${random_pet.this.id}-existing-log-group"
+
+  type = "express"
+
+  definition = local.definition_template
+
+  use_existing_cloudwatch_log_group = true
+  cloudwatch_log_group_name         = aws_cloudwatch_log_group.external.name
+
+  logging_configuration = {
+    include_execution_data = false
+    level                  = "ERROR"
+  }
+
+  depends_on = [aws_cloudwatch_log_group.external]
+}
+
 ###########
 # Disabled
 ###########

main.tf

@@ -2,6 +2,11 @@ locals {
   create_role = var.create && var.create_role && !var.use_existing_role
   aws_region  = local.create_role && var.aws_region_assume_role == "" ? data.aws_region.current[0].name : var.aws_region_assume_role
 
+  enable_logging = try(var.logging_configuration["level"], "OFF") != "OFF"
+
+  # Normalize ARN by trimming ":*" because data-source has it, but resource does not have it
+  log_group_arn = trimsuffix(element(concat(data.aws_cloudwatch_log_group.sfn.*.arn, aws_cloudwatch_log_group.sfn.*.arn, [""]), 0), ":*")
+
   role_name = local.create_role ? coalesce(var.role_name, var.name) : null
 }
 
@@ -13,6 +18,16 @@ resource "aws_sfn_state_machine" "this" {
   role_arn   = var.use_existing_role ? var.role_arn : aws_iam_role.this[0].arn
   definition = var.definition
 
+  dynamic "logging_configuration" {
+    for_each = local.enable_logging ? [true] : []
+
+    content {
+      log_destination        = lookup(var.logging_configuration, "log_destination", "${local.log_group_arn}:*")
+      include_execution_data = lookup(var.logging_configuration, "include_execution_data", null)
+      level                  = lookup(var.logging_configuration, "level", null)
+    }
+  }
+
   type = upper(var.type)
 
   tags = merge({ Name = var.name }, var.tags)
@@ -217,3 +232,64 @@ resource "aws_iam_policy_attachment" "additional_inline" {
   roles      = [aws_iam_role.this[0].name]
   policy_arn = aws_iam_policy.additional_inline[0].arn
 }
+
+#################################
+# IAM policy for Cloudwatch Logs
+#################################
+
+data "aws_iam_policy_document" "logs" {
+  count = local.create_role && local.enable_logging && var.attach_cloudwatch_logs_policy ? 1 : 0
+
+  # Copied from https://docs.aws.amazon.com/step-functions/latest/dg/cw-logs.html
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogDelivery",
+      "logs:GetLogDelivery",
+      "logs:UpdateLogDelivery",
+      "logs:DeleteLogDelivery",
+      "logs:ListLogDeliveries",
+      "logs:PutResourcePolicy",
+      "logs:DescribeResourcePolicies",
+      "logs:DescribeLogGroups",
+    ]
+
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_policy" "logs" {
+  count = local.create_role && local.enable_logging && var.attach_cloudwatch_logs_policy ? 1 : 0
+
+  name   = "${local.role_name}-logs"
+  policy = data.aws_iam_policy_document.logs[0].json
+}
+
+resource "aws_iam_policy_attachment" "logs" {
+  count = local.create_role && local.enable_logging && var.attach_cloudwatch_logs_policy ? 1 : 0
+
+  name       = "${local.role_name}-logs"
+  roles      = [aws_iam_role.this[0].name]
+  policy_arn = aws_iam_policy.logs[0].arn
+}
+
+##################
+# CloudWatch Logs
+##################
+
+data "aws_cloudwatch_log_group" "sfn" {
+  count = var.create && local.enable_logging && var.use_existing_cloudwatch_log_group ? 1 : 0
+
+  name = var.cloudwatch_log_group_name
+}
+
+resource "aws_cloudwatch_log_group" "sfn" {
+  count = var.create && local.enable_logging && !var.use_existing_cloudwatch_log_group ? 1 : 0
+
+  name              = coalesce(var.cloudwatch_log_group_name, var.name)
+  retention_in_days = var.cloudwatch_log_group_retention_in_days
+  kms_key_id        = var.cloudwatch_log_group_kms_key_id
+
+  tags = merge(var.tags, var.cloudwatch_log_group_tags)
+}

variables.tf

@@ -16,6 +16,12 @@ variable "use_existing_role" {
   default     = false
 }
 
+variable "use_existing_cloudwatch_log_group" {
+  description = "Whether to use an existing CloudWatch log group or create new"
+  type        = bool
+  default     = false
+}
+
 ################
 # Step Function
 ################
@@ -55,6 +61,46 @@ variable "type" {
   }
 }
 
+#################
+# CloudWatch Logs
+#################
+
+variable "logging_configuration" {
+  description = "Defines what execution history events are logged and where they are logged"
+  type        = map(string)
+  default     = {}
+}
+
+variable "cloudwatch_log_group_name" {
+  description = "Name of Cloudwatch Logs group name to use."
+  type        = string
+  default     = null
+}
+
+variable "cloudwatch_log_group_retention_in_days" {
+  description = "Specifies the number of days you want to retain log events in the specified log group. Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653."
+  type        = number
+  default     = null
+}
+
+variable "cloudwatch_log_group_kms_key_id" {
+  description = "The ARN of the KMS Key to use when encrypting log data."
+  type        = string
+  default     = null
+}
+
+variable "cloudwatch_log_group_tags" {
+  description = "A map of tags to assign to the resource."
+  type        = map(string)
+  default     = {}
+}
+
+variable "attach_cloudwatch_logs_policy" {
+  description = "Controls whether CloudWatch Logs policy should be added to IAM role for Lambda Function"
+  type        = bool
+  default     = true
+}
+
 ###########
 # IAM Role
 ###########