Skip to content

Commit 21852d9

Browse files
vaibhavjainvvaibhavjainv
committed
Revert "Moved schema registry to SAM"
This reverts commit 126917c.
1 parent d6ad9c9 commit 21852d9

File tree

4 files changed

+62
-117
lines changed

4 files changed

+62
-117
lines changed

‎msk-lambda-schema-avro-java-sam/MSKAndKafkaClientEC2.yaml‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,12 @@ Parameters:
3131
ServerlessLandGithubLocation:
3232
Type: String
3333
Default: https://github.com/aws-samples/serverless-patterns/
34-
34+
ContactSchemaName:
35+
Type: String
36+
Default: ContactSchema
37+
GlueSchemaRegistryForMSKName:
38+
Type: String
39+
Default: GlueSchemaRegistryForMSK
3540

3641
Conditions:
3742
CreateProvisionedCluster: !Equals
@@ -271,6 +276,39 @@ Resources:
271276
ToPort: 22
272277
SourceSecurityGroupId: !GetAtt KafkaClientInstanceSecurityGroup.GroupId
273278

279+
MSKGlueRegistry:
280+
Type: AWS::Glue::Registry
281+
Properties:
282+
Name: !Ref GlueSchemaRegistryForMSKName
283+
Description: "Registry for storing schemas related to MSK"
284+
285+
ContactSchema:
286+
Type: AWS::Glue::Schema
287+
Properties:
288+
Name: !Ref ContactSchemaName
289+
Compatibility: BACKWARD
290+
DataFormat: AVRO
291+
Registry:
292+
Arn: !GetAtt MSKGlueRegistry.Arn
293+
SchemaDefinition: |
294+
{
295+
"type": "record",
296+
"name": "Contact",
297+
"fields": [
298+
{"name": "firstname", "type": "string"},
299+
{"name": "lastname", "type": "string"},
300+
{"name": "company", "type": "string"},
301+
{"name": "street", "type": "string"},
302+
{"name": "city", "type": "string"},
303+
{"name": "county", "type": "string"},
304+
{"name": "state", "type": "string"},
305+
{"name": "zip", "type": "string"},
306+
{"name": "homePhone", "type": "string"},
307+
{"name": "cellPhone", "type": "string"},
308+
{"name": "email", "type": "string"},
309+
{"name": "website", "type": "string"}
310+
]
311+
}
274312
275313
KafkaClientEC2InstanceProvisioned:
276314
Condition: CreateProvisionedCluster
@@ -518,6 +556,8 @@ Resources:
518556
cp template_original.yaml template.yaml
519557
sudo chown -R ec2-user .
520558
559+
GLUE_SCHEMA_REGISTRY_NAME=${glue_registry_name}
560+
CONTACT_SCHEMA=${contact_schema_name}
521561
VPC_ID=${vpcid}
522562
LAMBDA_SECURITY_GROUP_ID=${securitygroup}
523563
PRIVATE_SUBNET_1=${privatesubnetone}
@@ -530,6 +570,8 @@ Resources:
530570
sed -i "s/CLUSTER_ID/$CLUSTER_ID/g" template.yaml
531571
sed -i "s/KAFKA_TOPIC/$KAFKA_TOPIC/g" template.yaml
532572
sed -i "s/JAVA_VERSION/$JAVA_VERSION/g" template.yaml
573+
sed -i "s/GLUE_SCHEMA_REGISTRY_NAME/$GLUE_SCHEMA_REGISTRY_NAME/g" template.yaml
574+
sed -i "s/AVRO_SCHEMA/$CONTACT_SCHEMA/g" template.yaml
533575
sed -i "s/VPC_ID/$VPC_ID/g" template.yaml
534576
sed -i "s/LAMBDA_SECURITY_GROUP_ID/$LAMBDA_SECURITY_GROUP_ID/g" template.yaml
535577
sed -i "s/SUBNET_IDS/$SUBNET_IDS/g" template.yaml
@@ -560,6 +602,8 @@ Resources:
560602
privatesubnettwo: !Ref PrivateSubnetMSKTwo
561603
privatesubnetthree: !Ref PrivateSubnetMSKThree
562604
securitygroup: !GetAtt MSKSecurityGroup.GroupId
605+
glue_registry_name: !Ref GlueSchemaRegistryForMSKName
606+
contact_schema_name: !Ref ContactSchemaName
563607

564608

565609
KafkaClientEC2InstanceServerless:
@@ -808,6 +852,8 @@ Resources:
808852
cp template_original.yaml template.yaml
809853
sudo chown -R ec2-user .
810854
855+
GLUE_SCHEMA_REGISTRY_NAME=${glue_registry_name}
856+
CONTACT_SCHEMA=${contact_schema_name}
811857
VPC_ID=${vpcid}
812858
LAMBDA_SECURITY_GROUP_ID=${securitygroup}
813859
PRIVATE_SUBNET_1=${privatesubnetone}
@@ -821,6 +867,8 @@ Resources:
821867
sed -i "s/CLUSTER_ID/$CLUSTER_ID/g" template.yaml
822868
sed -i "s/KAFKA_TOPIC/$KAFKA_TOPIC/g" template.yaml
823869
sed -i "s/JAVA_VERSION/$JAVA_VERSION/g" template.yaml
870+
sed -i "s/GLUE_SCHEMA_REGISTRY_NAME/$GLUE_SCHEMA_REGISTRY_NAME/g" template.yaml
871+
sed -i "s/AVRO_SCHEMA/$CONTACT_SCHEMA/g" template.yaml
824872
sed -i "s/VPC_ID/$VPC_ID/g" template.yaml
825873
sed -i "s/LAMBDA_SECURITY_GROUP_ID/$LAMBDA_SECURITY_GROUP_ID/g" template.yaml
826874
sed -i "s/SUBNET_IDS/$SUBNET_IDS/g" template.yaml
@@ -851,6 +899,8 @@ Resources:
851899
privatesubnettwo: !Ref PrivateSubnetMSKTwo
852900
privatesubnetthree: !Ref PrivateSubnetMSKThree
853901
securitygroup: !GetAtt MSKSecurityGroup.GroupId
902+
glue_registry_name: !Ref GlueSchemaRegistryForMSKName
903+
contact_schema_name: !Ref ContactSchemaName
854904

855905

856906

‎msk-lambda-schema-avro-java-sam/README.md‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@ This pattern is an example of Lambda functions that:
44
1. Consume messages from an Amazon Managed Streaming for Kafka (Amazon MSK) topic
55
2. Produce Avro-formatted messages to an Amazon MSK topic using Schema Registry
66

7-
Both functions use IAM authentication to connect to the MSK Cluster and use AWS Glue Schema Registry for Avro schema management. The Glue Schema Registry and Contact Schema are created as part of the SAM deployment.
7+
Both functions use IAM authentication to connect to the MSK Cluster and use AWS Glue Schema Registry for Avro schema management.
88

99
This project contains source code and supporting files for a serverless application that you can deploy with the SAM CLI. It includes the following files and folders.
1010

1111
- `kafka_event_consumer_function/src/main/java` - Code for the consumer Lambda function.
1212
- `kafka_event_producer_function/src/main/java` - Code for the Avro producer Lambda function.
1313
- `events` - Invocation events that you can use to invoke the functions.
1414
- `kafka_event_consumer_function/src/test/java` - Unit tests for the consumer code.
15-
- `template.yaml` - A template that defines the application's Lambda functions, Glue Schema Registry, and Contact Schema.
15+
- `template.yaml` - A template that defines the application's Lambda functions.
1616
- `template_original.yaml` - The original template with placeholders that get replaced during deployment.
1717
- `MSKAndKafkaClientEC2.yaml` - A CloudFormation template file that can be used to deploy an MSK cluster and also deploy an EC2 machine with all pre-requisites already installed, so you can directly build and deploy the lambda functions and test them out.
1818

@@ -96,15 +96,14 @@ sam deploy --capabilities CAPABILITY_IAM --no-confirm-changeset --no-disable-rol
9696
The `sam deploy` command packages and deploys your application to AWS, with a series of prompts.
9797

9898
> [!NOTE]
99-
> The script retrieves the required parameters from the CloudFormation outputs in the AWS Console after deploying the `MSKAndKafkaClientEC2.yaml` template. These outputs contain the necessary information for deploying the Lambda functions. If you connect to a different Kafka cluster, enter the values manually.
99+
> The script retrieves the required parameters from the CloudFormation outputs in the AWS Console after deploying the `MSKAndKafkaClientEC2.yaml` template. These outputs contain all the necessary information for deploying the Lambda functions. If you connect to a different Kafka cluster, enter the values manually.
100100
101101
* **Stack Name**: The name of the stack to deploy to CloudFormation. This should be unique to your account and region, and a good starting point would be something matching your project name.
102102
* **AWS Region**: The AWS region you want to deploy your app to.
103103
* **Parameter MSKClusterName**: The name of the MSK Cluster. This will be `<stack-name>-cluster` from the CloudFormation template you deployed in the previous step.
104104
* **Parameter MSKClusterId**: The unique ID of the MSK Cluster. This can be found in the MSK console or extracted from the MSK ARN in the CloudFormation outputs.
105105
* **Parameter MSKTopic**: The Kafka topic on which the Lambda functions will produce and consume messages. You can find this in the CloudFormation outputs as `KafkaTopicForLambda`
106-
* **Parameter GlueSchemaRegistryName**: The name of the Glue Schema Registry to be created (default: GlueSchemaRegistryForMSK).
107-
* **Parameter ContactSchemaName**: The name of the Contact Schema to be created (default: ContactSchema).
106+
* **Parameter ContactSchemaName**: The name of the schema to be used for the Avro serialization (default: ContactSchema).
108107
* **Parameter VpcId**: The ID of the VPC where the MSK cluster is deployed. You can find this in the CloudFormation outputs as `VPCId`.
109108
* **Parameter SubnetIds**: Comma-separated list of subnet IDs where the MSK cluster is deployed. You can find these in the CloudFormation outputs as `PrivateSubnetMSKOne`, `PrivateSubnetMSKTwo`, and `PrivateSubnetMSKThree`.
110109
* **Parameter SecurityGroupIds**: Comma-separated list of security group IDs that allow access to the MSK cluster. You can find this in the CloudFormation outputs as `SecurityGroupId`.

‎msk-lambda-schema-avro-java-sam/template.yaml‎

Lines changed: 3 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -11,43 +11,6 @@ Globals:
1111
Timeout: 15
1212

1313
Resources:
14-
# Glue Schema Registry for storing Avro schemas
15-
MSKGlueRegistry:
16-
Type: AWS::Glue::Registry
17-
Properties:
18-
Name: !Ref GlueSchemaRegistryName
19-
Description: "Registry for storing schemas related to MSK"
20-
21-
# Contact Schema for Avro serialization/deserialization
22-
ContactSchema:
23-
Type: AWS::Glue::Schema
24-
Properties:
25-
Name: !Ref ContactSchemaName
26-
Description: "Schema for Contact data in Avro format"
27-
Compatibility: BACKWARD
28-
DataFormat: AVRO
29-
Registry:
30-
Arn: !GetAtt MSKGlueRegistry.Arn
31-
SchemaDefinition: |
32-
{
33-
"type": "record",
34-
"name": "Contact",
35-
"fields": [
36-
{"name": "firstName", "type": "string"},
37-
{"name": "lastName", "type": "string"},
38-
{"name": "company", "type": "string"},
39-
{"name": "address", "type": "string"},
40-
{"name": "city", "type": "string"},
41-
{"name": "county", "type": "string"},
42-
{"name": "state", "type": "string"},
43-
{"name": "zip", "type": "string"},
44-
{"name": "homePhone", "type": "string"},
45-
{"name": "cellPhone", "type": "string"},
46-
{"name": "email", "type": "string"},
47-
{"name": "website", "type": "string"}
48-
]
49-
}
50-
5114
# SQS Queue to use as Dead Letter Queue for the MSK event source mapping
5215
ConsumerDLQ:
5316
Type: AWS::SQS::Queue
@@ -142,9 +105,7 @@ Resources:
142105
- glue:PutSchemaVersionMetadata
143106
- glue:GetSchemaVersionsDiff
144107
- glue:QuerySchemaVersionMetadata
145-
Resource:
146-
- !GetAtt MSKGlueRegistry.Arn
147-
- !GetAtt ContactSchema.Arn
108+
Resource: '*'
148109

149110
- Sid: SQSPermissionsPolicy
150111
Effect: Allow
@@ -219,8 +180,8 @@ Resources:
219180
- glue:CreateSchema
220181
- glue:CreateRegistry
221182
Resource:
222-
- !GetAtt MSKGlueRegistry.Arn
223-
- !GetAtt ContactSchema.Arn
183+
- !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:registry/*"
184+
- !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:schema/*/*"
224185
- VPCAccessPolicy: {}
225186

226187
Parameters:
@@ -270,15 +231,3 @@ Outputs:
270231
ConsumerDLQArn:
271232
Description: "ARN of the Dead Letter Queue for the MSK Consumer"
272233
Value: !GetAtt ConsumerDLQ.Arn
273-
GlueSchemaRegistryArn:
274-
Description: "ARN of the Glue Schema Registry"
275-
Value: !GetAtt MSKGlueRegistry.Arn
276-
GlueSchemaRegistryName:
277-
Description: "Name of the Glue Schema Registry"
278-
Value: !Ref MSKGlueRegistry
279-
ContactSchemaArn:
280-
Description: "ARN of the Contact Schema"
281-
Value: !GetAtt ContactSchema.Arn
282-
ContactSchemaName:
283-
Description: "Name of the Contact Schema"
284-
Value: !Ref ContactSchema

‎msk-lambda-schema-avro-java-sam/template_original.yaml‎

Lines changed: 4 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,43 +11,6 @@ Globals:
1111
Timeout: 15
1212

1313
Resources:
14-
# Glue Schema Registry for storing Avro schemas
15-
MSKGlueRegistry:
16-
Type: AWS::Glue::Registry
17-
Properties:
18-
Name: !Ref GlueSchemaRegistryName
19-
Description: "Registry for storing schemas related to MSK"
20-
21-
# Contact Schema for Avro serialization/deserialization
22-
ContactSchema:
23-
Type: AWS::Glue::Schema
24-
Properties:
25-
Name: !Ref ContactSchemaName
26-
Description: "Schema for Contact data in Avro format"
27-
Compatibility: BACKWARD
28-
DataFormat: AVRO
29-
Registry:
30-
Arn: !GetAtt MSKGlueRegistry.Arn
31-
SchemaDefinition: |
32-
{
33-
"type": "record",
34-
"name": "Contact",
35-
"fields": [
36-
{"name": "firstName", "type": "string"},
37-
{"name": "lastName", "type": "string"},
38-
{"name": "company", "type": "string"},
39-
{"name": "address", "type": "string"},
40-
{"name": "city", "type": "string"},
41-
{"name": "county", "type": "string"},
42-
{"name": "state", "type": "string"},
43-
{"name": "zip", "type": "string"},
44-
{"name": "homePhone", "type": "string"},
45-
{"name": "cellPhone", "type": "string"},
46-
{"name": "email", "type": "string"},
47-
{"name": "website", "type": "string"}
48-
]
49-
}
50-
5114
# SQS Queue to use as Dead Letter Queue for the MSK event source mapping
5215
ConsumerDLQ:
5316
Type: AWS::SQS::Queue
@@ -62,7 +25,7 @@ Resources:
6225
Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
6326
Properties:
6427
CodeUri: kafka_event_consumer_function
65-
Handler: com.amazonaws.services.lambda.samples.events.msk.AvroKafkaHandler::handleRequest
28+
Handler: com.amazonaws.services.lambda.samples.events.msk.HandlerMSK::handleRequest
6629
Runtime: java21
6730
Architectures:
6831
- x86_64
@@ -74,8 +37,6 @@ Resources:
7437
Variables:
7538
PARAM1: VALUE
7639
JAVA_TOOL_OPTIONS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 # More info about tiered compilation https://aws.amazon.com/blogs/compute/optimizing-aws-lambda-function-performance-for-java/
77-
POWERTOOLS_LOG_LEVEL: TRACE
78-
POWERTOOLS_SERVICE_NAME: kafka_consumer
7940
Events:
8041
MSKEvent:
8142
Type: MSK
@@ -151,9 +112,7 @@ Resources:
151112
- glue:PutSchemaVersionMetadata
152113
- glue:GetSchemaVersionsDiff
153114
- glue:QuerySchemaVersionMetadata
154-
Resource:
155-
- !GetAtt MSKGlueRegistry.Arn
156-
- !GetAtt ContactSchema.Arn
115+
Resource: '*'
157116

158117
- Sid: SQSPermissionsPolicy
159118
Effect: Allow
@@ -228,8 +187,8 @@ Resources:
228187
- glue:CreateSchema
229188
- glue:CreateRegistry
230189
Resource:
231-
- !GetAtt MSKGlueRegistry.Arn
232-
- !GetAtt ContactSchema.Arn
190+
- !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:registry/*"
191+
- !Sub "arn:aws:glue:${AWS::Region}:${AWS::AccountId}:schema/*/*"
233192
- VPCAccessPolicy: {}
234193

235194
Parameters:
@@ -279,15 +238,3 @@ Outputs:
279238
ConsumerDLQArn:
280239
Description: "ARN of the Dead Letter Queue for the MSK Consumer"
281240
Value: !GetAtt ConsumerDLQ.Arn
282-
GlueSchemaRegistryArn:
283-
Description: "ARN of the Glue Schema Registry"
284-
Value: !GetAtt MSKGlueRegistry.Arn
285-
GlueSchemaRegistryName:
286-
Description: "Name of the Glue Schema Registry"
287-
Value: !Ref MSKGlueRegistry
288-
ContactSchemaArn:
289-
Description: "ARN of the Contact Schema"
290-
Value: !GetAtt ContactSchema.Arn
291-
ContactSchemaName:
292-
Description: "Name of the Contact Schema"
293-
Value: !Ref ContactSchema

0 commit comments

Comments
 (0)