Redshift Serverless test infrastructure (#1588)

Author: cnfait

test_infra/app.py

@@ -1,30 +1,44 @@
 #!/usr/bin/env python3
-from aws_cdk import App
+import os
+
+from aws_cdk import App, Environment
 from stacks.base_stack import BaseStack
 from stacks.databases_stack import DatabasesStack
 from stacks.lakeformation_stack import LakeFormationStack
 from stacks.opensearch_stack import OpenSearchStack
 
 app = App()
 
-base = BaseStack(app, "aws-sdk-pandas-base")
+env = {"env": Environment(account=os.environ["CDK_DEFAULT_ACCOUNT"], region=os.environ["CDK_DEFAULT_REGION"])}
+
+base = BaseStack(
+    app,
+    "aws-sdk-pandas-base",
+    **env,
+)
 
 DatabasesStack(
     app,
     "aws-sdk-pandas-databases",
     base.get_vpc,
     base.get_bucket,
     base.get_key,
+    **env,
 )
 
-LakeFormationStack(app, "aws-sdk-pandas-lakeformation")
+LakeFormationStack(
+    app,
+    "aws-sdk-pandas-lakeformation",
+    **env,
+)
 
 OpenSearchStack(
     app,
     "aws-sdk-pandas-opensearch",
     base.get_vpc,
     base.get_bucket,
     base.get_key,
+    **env,
 )
 
 app.synth()

test_infra/poetry.lock

@@ -7,8 +7,8 @@ license = "Apache License 2.0"
 
 [tool.poetry.dependencies]
 python = ">=3.7.1, <3.11"
-"aws-cdk-lib" = "^2.20.0"
-"constructs" = "^10.0.119"
-"aws-cdk.aws-glue-alpha" = "^2.20.0a0"
-"aws-cdk.aws-redshift-alpha" = "^2.20.0a0"
-"aws-cdk.aws-neptune-alpha" = "^2.20.0a0"
+"aws-cdk-lib" = "^2.35.0"
+"constructs" = "^10.1.67"
+"aws-cdk.aws-glue-alpha" = "^2.35.0a0"
+"aws-cdk.aws-redshift-alpha" = "^2.35.0a0"
+"aws-cdk.aws-neptune-alpha" = "^2.35.0a0"

test_infra/pyproject.toml

@@ -96,6 +96,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs: str) -> None:
             value=self.vpc.public_subnets[1].subnet_id,
             export_name="aws-sdk-pandas-base-PublicSubnet2",
         )
+        CfnOutput(
+            self,
+            "PublicSubnet3",
+            value=self.vpc.public_subnets[2].subnet_id,
+            export_name="aws-sdk-pandas-base-PublicSubnet3",
+        )
         CfnOutput(
             self,
             "PrivateSubnet",

test_infra/stacks/base_stack.py

@@ -9,6 +9,7 @@
 from aws_cdk import aws_neptune_alpha as neptune
 from aws_cdk import aws_rds as rds
 from aws_cdk import aws_redshift_alpha as redshift
+from aws_cdk import aws_redshiftserverless as redshiftserverless
 from aws_cdk import aws_s3 as s3
 from aws_cdk import aws_secretsmanager as secrets
 from aws_cdk import aws_ssm as ssm
@@ -42,6 +43,7 @@ def __init__(
         self._set_catalog_encryption()
         if databases_context["redshift"]:
             self._setup_redshift()
+            self._setup_redshift_serverless()
         if databases_context["postgresql"]:
             self._setup_postgresql()
         if databases_context["mysql"]:
@@ -297,6 +299,46 @@ def _setup_redshift(self) -> None:
         CfnOutput(self, "RedshiftSchema", value=schema)
         CfnOutput(self, "RedshiftRole", value=redshift_role.role_arn)
 
+    def _setup_redshift_serverless(self) -> None:
+        database = "test"
+        redshift_cfn_namespace = redshiftserverless.CfnNamespace(
+            self,
+            "aws-sdk-pandas-redshift-serverless-namespace",
+            namespace_name="aws-sdk-pandas-redshift-serverless-namespace",
+            admin_username=self.db_username,
+            admin_user_password=self.db_password,
+            db_name=database,
+        )
+        redshift_cfn_workgroup = redshiftserverless.CfnWorkgroup(
+            self,
+            "aws-sdk-pandas-redshift-serverless-workgroup",
+            workgroup_name="aws-sdk-pandas-redshift-serverless-workgroup",
+            namespace_name=redshift_cfn_namespace.namespace_name,
+            publicly_accessible=True,
+            security_group_ids=[self.db_security_group.security_group_id],
+            subnet_ids=[subnet.subnet_id for subnet in self.vpc.public_subnets],
+        )
+        redshift_cfn_workgroup.node.add_dependency(redshift_cfn_namespace)
+        secret = secrets.Secret(
+            self,
+            "aws-sdk-pandas-redshift-serverless-secret",
+            secret_name="aws-sdk-pandas/redshift-serverless",
+            description="Redshift Serverless credentials",
+            generate_secret_string=secrets.SecretStringGenerator(
+                generate_string_key="dummy",
+                secret_string_template=json.dumps(
+                    {
+                        "username": self.db_username,
+                        "password": self.db_password,
+                        "engine": "redshift-serverless",
+                    }
+                ),
+            ),
+        )
+        CfnOutput(self, "RedshiftServerlessSecretArn", value=secret.secret_arn)
+        CfnOutput(self, "RedshiftServerlessWorkgroup", value=redshift_cfn_workgroup.workgroup_name)
+        CfnOutput(self, "RedshiftServerlessDatabase", value=database)
+
     def _setup_postgresql(self) -> None:
         port = 3306
         database = "postgres"
@@ -483,7 +525,7 @@ def _setup_mysql_serverless(self) -> None:
             ),
             backup_retention=Duration.days(1),
             vpc=self.vpc,
-            vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT),
+            vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS),
             subnet_group=self.rds_subnet_group,
             security_groups=[self.db_security_group],
             enable_data_api=True,