Skip to content

Commit b79b5e2

Browse files
committed
Upgrade: change cdk ver1 -> cdk ver2
1 parent 93a1048 commit b79b5e2

20 files changed

+2443
-3382
lines changed

cdk.json

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,5 @@
11
{
22
"app": "npx ts-node --prefer-ts-exts infra/app-main",
33
"context": {
4-
"@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
5-
"@aws-cdk/core:enableStackNameDuplicates": "true",
6-
"aws-cdk:enableDiffNoFail": "true",
7-
"@aws-cdk/core:stackRelativeExports": "true",
8-
"@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true,
9-
"@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true,
10-
"@aws-cdk/aws-kms:defaultKeyPolicies": true,
11-
"@aws-cdk/aws-s3:grantWriteWithoutAcl": true,
12-
"@aws-cdk/aws-ecs-patterns:removeDefaultDesiredCount": true,
13-
"@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
14-
"@aws-cdk/aws-efs:defaultEncryptionAtRest": true,
15-
"@aws-cdk/aws-lambda:recognizeVersionProps": true
164
}
175
}

codes/glue/churn-xgboost/script/upload_input.sh

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ PROFILE_NAME=$(cat $APP_CONFIG | jq -r '.Project.Profile')
1313
PROJECT_NAME=$(cat $APP_CONFIG | jq -r '.Project.Name')
1414
PROJECT_STAGE=$(cat $APP_CONFIG | jq -r '.Project.Stage')
1515
PROJECT_PREFIX=$PROJECT_NAME$PROJECT_STAGE
16+
STACK_NAME=ChurnXgboostPipelineStack
17+
BUCKET_SUFFIX=asset
1618

1719
PROJECT_PREFIX_LOWER=$(echo $PROJECT_PREFIX | tr '[:upper:]' '[:lower:]')
18-
ENDPOINT_NAME_LOWER=$(echo $ENDPOINT_NAME | tr '[:upper:]' '[:lower:]')
20+
STACK_NAME_LOWER=$(echo $STACK_NAME | tr '[:upper:]' '[:lower:]')
1921

2022
if [ -z "$PROFILE_NAME" ]; then
21-
aws s3 cp $DATA_PATH/$DATA_FILE s3://"$PROJECT_PREFIX_LOWER"-"$ENDPOINT_NAME_LOWER"-"$REGION"-"${ACCOUNT:0:5}"/input/"${S3_UPLOAD_KEY}"
23+
aws s3 cp $DATA_PATH/$DATA_FILE s3://"$PROJECT_PREFIX_LOWER"-"$STACK_NAME_LOWER"-"$BUCKET_SUFFIX"-"$REGION"-"${ACCOUNT:0:5}"/input/"${S3_UPLOAD_KEY}"
2224
else
23-
aws s3 cp $DATA_PATH/$DATA_FILE s3://"$PROJECT_PREFIX_LOWER"-"$ENDPOINT_NAME_LOWER"-"$REGION"-"${ACCOUNT:0:5}"/input/"${S3_UPLOAD_KEY}" --profile $PROFILE_NAME
25+
aws s3 cp $DATA_PATH/$DATA_FILE s3://"$PROJECT_PREFIX_LOWER"-"$STACK_NAME_LOWER"-"$BUCKET_SUFFIX"-"$REGION"-"${ACCOUNT:0:5}"/input/"${S3_UPLOAD_KEY}" --profile $PROFILE_NAME
2426
fi

infra/app-main.ts

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
11
#!/usr/bin/env node
2-
import { AppContext } from '../lib/template/app-context';
2+
import { AppContext, AppContextError } from '../lib/template/app-context';
33

44
import { MLOpsPipelineStack } from './stack/mlops/mlops-pipeline-stack'
55

66

7-
const appContext = new AppContext({
8-
appConfigEnvName: 'APP_CONFIG',
9-
});
7+
try {
8+
const appContext = new AppContext({
9+
appConfigFileKey: 'APP_CONFIG',
10+
});
1011

11-
if (appContext.stackCommonProps != undefined) {
1212
new MLOpsPipelineStack(appContext, appContext.appConfig.Stack.ChurnXgboostPipeline);
13-
} else {
14-
console.error('[Error] wrong AppConfigFile');
15-
}
13+
} catch (error) {
14+
if (error instanceof AppContextError) {
15+
console.error('[AppContextError]:', error.message);
16+
} else {
17+
console.error('[Error]: not-handled-error', error);
18+
}
19+
}

infra/stack/mlops/mlops-pipeline-stack.ts

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
import * as cdk from '@aws-cdk/core';
2-
import * as iam from '@aws-cdk/aws-iam';
3-
import * as s3 from '@aws-cdk/aws-s3';
4-
import * as ec2 from '@aws-cdk/aws-ec2';
5-
import * as lambda from '@aws-cdk/aws-lambda';
6-
import * as lambda_event from '@aws-cdk/aws-lambda-event-sources';
7-
import * as glue from '@aws-cdk/aws-glue';
8-
import * as sfn from '@aws-cdk/aws-stepfunctions';
9-
import * as tasks from '@aws-cdk/aws-stepfunctions-tasks';
1+
import * as cdk from 'aws-cdk-lib';
2+
import * as iam from 'aws-cdk-lib/aws-iam';
3+
import * as s3 from 'aws-cdk-lib/aws-s3';
4+
import * as ec2 from 'aws-cdk-lib/aws-ec2';
5+
import * as lambda from 'aws-cdk-lib/aws-lambda';
6+
import * as lambda_event from 'aws-cdk-lib/aws-lambda-event-sources';
7+
import * as glue from 'aws-cdk-lib/aws-glue';
8+
import * as sfn from 'aws-cdk-lib/aws-stepfunctions';
9+
import * as tasks from 'aws-cdk-lib/aws-stepfunctions-tasks';
1010

1111
import * as base from '../../../lib/template/stack/base/base-stack';
1212
import { AppContext } from '../../../lib/template/app-context';
1313

14-
import { GlueJobConstruct } from '../../../lib/template/pattern/glue-job-construct';
14+
import { GlueJobConstruct } from '../../../lib/template/construct/pattern/glue-job-construct';
1515

1616
interface MLOpsPipelineConfig {
1717
EndpointName: string;
@@ -67,7 +67,7 @@ export class MLOpsPipelineStack extends base.BaseStack {
6767

6868
const baseName = pipelineConfig.EndpointName;
6969

70-
const bucket = this.createS3Bucket(baseName);
70+
const bucket = this.createS3Bucket('asset');
7171

7272
const stateMachine = this.createStateMachine({
7373
statemachineName: baseName,
@@ -331,8 +331,12 @@ export class MLOpsPipelineStack extends base.BaseStack {
331331
const directory = filePath.replace(`/${fileName}`, '');
332332

333333
return new GlueJobConstruct(this, baseName, {
334-
baseName: baseName,
334+
stackConfig: this.stackConfig,
335+
stackName: this.stackName,
336+
env: this.commonProps.env!,
335337
projectPrefix: this.projectPrefix,
338+
339+
baseName: baseName,
336340
bucket: bucket,
337341
timeoutInMin: timeoutInMin,
338342
etlScriptFileName: fileName,

lib/template/app-config.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: MIT-0
4+
*
5+
* Permission is hereby granted, free of charge, to any person obtaining a copy of this
6+
* software and associated documentation files (the "Software"), to deal in the Software
7+
* without restriction, including without limitation the rights to use, copy, modify,
8+
* merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
9+
* permit persons to whom the Software is furnished to do so.
10+
*
11+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
12+
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
13+
* PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
14+
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
15+
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
16+
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
17+
*/
18+
19+
export interface ProjectConfig {
20+
Name: string,
21+
Stage: string,
22+
Account: string,
23+
Region: string,
24+
Profile: string
25+
}
26+
27+
export interface StackConfig {
28+
Name: string,
29+
UpdateRegionName?: string,
30+
[name: string]: any
31+
}
32+
33+
export interface AppConfig {
34+
Project: ProjectConfig;
35+
Global: any;
36+
Stack: StackConfig;
37+
}

lib/template/app-context.ts

Lines changed: 94 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -18,58 +18,84 @@
1818

1919
const fs = require('fs');
2020
const env = require('env-var');
21-
import * as cdk from '@aws-cdk/core';
21+
import * as cdk from 'aws-cdk-lib';
2222

23+
import { AppConfig } from './app-config';
2324
import { StackCommonProps } from './stack/base/base-stack';
2425

26+
27+
export class AppContextError extends Error {
28+
constructor(message: string) {
29+
super(message);
30+
this.name = "AppConfigFileFailError";
31+
}
32+
}
33+
34+
export enum ProjectPrefixType {
35+
NameStage,
36+
NameHyphenStage,
37+
Name
38+
}
39+
2540
export interface AppContextProps {
26-
appConfigEnvName: string;
27-
projectPrefix?: string;
41+
appConfigFileKey: string;
2842
contextArgs?: string[];
43+
projectPrefixType?: ProjectPrefixType;
2944
}
3045

3146
export class AppContext {
3247
public readonly cdkApp: cdk.App;
33-
public readonly appConfig: any | undefined;
34-
public readonly stackCommonProps: StackCommonProps | undefined;
48+
public readonly appConfig: AppConfig;
49+
public readonly stackCommonProps: StackCommonProps;
3550

36-
private projectPrefix: string|undefined;
37-
private infraConfigPath: string|undefined;
51+
private readonly appContextProps: AppContextProps;
3852

3953
constructor(props: AppContextProps) {
4054
this.cdkApp = new cdk.App();
55+
this.appContextProps = props;
56+
57+
try {
58+
const appConfigFile = this.findAppConfigFile(props.appConfigFileKey);
4159

42-
this.projectPrefix = props.projectPrefix;
60+
this.appConfig = this.loadAppConfigFile(appConfigFile, props.contextArgs);
4361

44-
this.appConfig = this.loadConfig(this.cdkApp, props.appConfigEnvName, props.contextArgs);
62+
if (this.appConfig != undefined) {
63+
this.stackCommonProps = this.createStackCommonProps(appConfigFile);
64+
}
4565

46-
if (this.appConfig != undefined) {
47-
this.stackCommonProps = this.createStackProps();
66+
} catch (e) {
67+
console.error(`==> CDK App-Config File is empty,
68+
set up your environment variable(Usage: export ${props.appConfigFileKey}=config/app-config-xxx.json)
69+
or append inline-argurment(Usage: cdk list --context ${props.appConfigFileKey}=config/app-config-xxx.json)`, e);
70+
throw new AppContextError('Fail to find App-Config json file');
4871
}
4972
}
5073

51-
private createStackProps(): StackCommonProps {
74+
public ready(): boolean {
75+
return this.stackCommonProps ? true : false;
76+
}
77+
78+
private createStackCommonProps(appConfigFile: string): StackCommonProps {
5279
const stackProps: StackCommonProps = {
53-
projectPrefix: this.projectPrefix!,
80+
projectPrefix: this.getProjectPrefix(this.appConfig.Project.Name, this.appConfig.Project.Stage),
5481
appConfig: this.appConfig,
55-
appConfigPath: this.infraConfigPath!,
82+
appConfigPath: appConfigFile,
5683
env: {
5784
account: this.appConfig.Project.Account,
5885
region: this.appConfig.Project.Region
5986
},
60-
variable: {}
87+
variables: {}
6188
}
6289

6390
return stackProps;
6491
}
6592

66-
// must contain infra/config/xxxx.json
67-
private loadConfig(cdkApp: cdk.App, key: string, contextArgs?: string[]): any {
68-
var fromType = 'Input-Parameter';
69-
var configFilePath = cdkApp.node.tryGetContext(key); //'APP_CONFIG'
93+
private findAppConfigFile(appConfigKey: string): string {
94+
let fromType = 'InLine-Argument';
95+
let configFilePath = this.cdkApp.node.tryGetContext(appConfigKey);
7096

7197
if (configFilePath == undefined) {
72-
configFilePath = env.get(key).asString();
98+
configFilePath = env.get(appConfigKey).asString();
7399

74100
if (configFilePath != undefined && configFilePath.length > 0) {
75101
fromType = 'Environment-Variable';
@@ -79,62 +105,68 @@ export class AppContext {
79105
}
80106

81107
if (configFilePath == undefined) {
82-
console.error(`==> CDK App-Config File is empty,
83-
please check your environment variable(Usage: export ${key}=config/app-config-xxx.json)
84-
or input parameter(--context=${key}=config/app-config-xxx.json)`);
85-
return false;
108+
throw new Error('Fail to find App-Config json file')
86109
} else {
87110
console.info(`==> CDK App-Config File is ${configFilePath}, which is from ${fromType}.`);
88-
return this.loadConfigFromFile(configFilePath, cdkApp, contextArgs);
89111
}
112+
113+
return configFilePath;
90114
}
91115

92-
private loadConfigFromFile(filePath: string, app?: cdk.App, contextArgs?: string[]): any {
93-
this.infraConfigPath = filePath;
94-
let config: any = JSON.parse(fs.readFileSync(filePath).toString());
95-
116+
private getProjectPrefix(projectName: string, projectStage: string): string {
117+
let prefix = `${projectName}${projectStage}`;
118+
119+
if (this.appContextProps.projectPrefixType === ProjectPrefixType.NameHyphenStage) {
120+
prefix = `${projectName}-${projectStage}`;
121+
} else if (this.appContextProps.projectPrefixType === ProjectPrefixType.Name) {
122+
prefix = projectName;
123+
}
124+
125+
return prefix;
126+
}
127+
128+
private loadAppConfigFile(filePath: string, contextArgs?: string[]): any {
129+
let appConfig = JSON.parse(fs.readFileSync(filePath).toString());
130+
let projectPrefix = this.getProjectPrefix(appConfig.Project.Name, appConfig.Project.Stage);
131+
96132
if (contextArgs != undefined) {
97-
this.updateContextArgs(config, app!, contextArgs);
133+
this.updateContextArgs(appConfig, contextArgs);
98134
}
99-
100-
this.addPrefixIntoStackName(config);
101135

102-
return config;
136+
this.addPrefixIntoStackName(appConfig, projectPrefix);
137+
138+
return appConfig;
103139
}
104-
105-
private updateContextArgs(config: any, app: cdk.App, contextArgs: string[]) {
106-
for (var key of contextArgs) {
140+
141+
private updateContextArgs(appConfig: any, contextArgs: string[]) {
142+
for (let key of contextArgs) {
107143
const jsonKeys = key.split('.');
108-
let oldValue = '';
109-
const newValue: string = app?.node.tryGetContext(key);
110-
111-
if (jsonKeys.length == 1) {
112-
oldValue = config[jsonKeys[0]];
113-
config[jsonKeys[0]] = newValue;
114-
} else if (jsonKeys.length == 2) {
115-
oldValue = config[jsonKeys[0]][jsonKeys[1]]
116-
config[jsonKeys[0]][jsonKeys[1]] = newValue;
117-
} else if (jsonKeys.length == 3) {
118-
oldValue = config[jsonKeys[0]][jsonKeys[1]][jsonKeys[2]];
119-
config[jsonKeys[0]][jsonKeys[1]][jsonKeys[2]] = newValue;
120-
} else if (jsonKeys.length == 4) {
121-
oldValue = config[jsonKeys[0]][jsonKeys[1]][jsonKeys[2]][jsonKeys[3]];
122-
config[jsonKeys[0]][jsonKeys[1]][jsonKeys[2]][jsonKeys[3]] = newValue;
123-
} else if (jsonKeys.length == 5) {
124-
oldValue = config[jsonKeys[0]][jsonKeys[1]][jsonKeys[2]][jsonKeys[3]][jsonKeys[4]];
125-
config[jsonKeys[0]][jsonKeys[1]][jsonKeys[2]][jsonKeys[3]][jsonKeys[4]] = newValue;
144+
let oldValue = undefined;
145+
const newValue: string = this.cdkApp.node.tryGetContext(key);
146+
147+
if (newValue != undefined && jsonKeys.length > 0) {
148+
try {
149+
oldValue = jsonKeys.reduce((reducer: any, pointer: string) => reducer.hasOwnProperty(pointer) ? reducer[pointer] : undefined, appConfig);
150+
} catch(e) {
151+
console.error(`[ERROR] updateContextArgs: This key[${key}] is an undefined value in Json-Config file.\n`, e);
152+
throw e;
153+
}
154+
155+
jsonKeys.reduce((reducer: any, pointer: string, count: number) => {
156+
if (count == jsonKeys.length - 1) reducer[pointer] = newValue;
157+
return reducer[pointer];
158+
}, appConfig);
159+
160+
console.info(`[INFO] updateContextArgs: Updated ${key} = ${oldValue}-->${newValue}`);
126161
}
127-
128-
console.error(`updateContextArgs: ${key} = ${oldValue}-->${newValue}`);
129162
}
130163
}
131164

132-
private addPrefixIntoStackName(config: any) {
133-
const projectPrefix = this.projectPrefix == undefined ? `${config.Project.Name}${config.Project.Stage}` : this.projectPrefix;
134-
this.projectPrefix = projectPrefix;
135-
136-
for (const key in config.Stack) {
137-
config.Stack[key].Name = `${projectPrefix}-${config.Stack[key].Name}`;
165+
private addPrefixIntoStackName(appConfig: any, projectPrefix: string) {
166+
for (const key in appConfig.Stack) {
167+
const stackOriginalName = appConfig.Stack[key].Name;
168+
appConfig.Stack[key].ShortStackName = stackOriginalName;
169+
appConfig.Stack[key].Name = `${projectPrefix}-${stackOriginalName}`;
138170
}
139171
}
140172
}

0 commit comments

Comments
 (0)