Skip to content

Commit 1778fe7

Browse files
vishalbolluvishalbollu
authored
Generate ami mapping and use it in cluster up (#2010)
1 parent 3f488e5 commit 1778fe7

File tree

5 files changed

+422
-7
lines changed

5 files changed

+422
-7
lines changed

‎build/generate_ami_mapping.go‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
/*
2+
Copyright 2021 Cortex Labs, Inc.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"encoding/json"
21+
"fmt"
22+
"io/ioutil"
23+
"log"
24+
"os"
25+
"sort"
26+
"time"
27+
28+
"github.com/pkg/errors"
29+
30+
"github.com/aws/aws-sdk-go/aws"
31+
"github.com/aws/aws-sdk-go/aws/session"
32+
"github.com/aws/aws-sdk-go/service/ec2"
33+
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
34+
)
35+
36+
// run with `go run build/generate_ami_mapping.go manager/manifests/ami.json`
37+
// copied from https://github.com/weaveworks/eksctl/blob/c211e68d3c8cf3c7f800768bfa0251dda17e011c/pkg/apis/eksctl.io/v1alpha5/types.go
38+
// most of this code can be removed once eksctl can be imported: https://github.com/weaveworks/eksctl/issues/813
39+
const (
40+
eksResourceAccountStandard = "602401143452"
41+
42+
// eksResourceAccountAPEast1 defines the AWS EKS account ID that provides node resources in ap-east-1 region
43+
eksResourceAccountAPEast1 = "800184023465"
44+
45+
// eksResourceAccountMESouth1 defines the AWS EKS account ID that provides node resources in me-south-1 region
46+
eksResourceAccountMESouth1 = "558608220178"
47+
48+
// eksResourceAccountCNNorthWest1 defines the AWS EKS account ID that provides node resources in cn-northwest-1 region
49+
eksResourceAccountCNNorthWest1 = "961992271922"
50+
51+
// eksResourceAccountCNNorth1 defines the AWS EKS account ID that provides node resources in cn-north-1
52+
eksResourceAccountCNNorth1 = "918309763551"
53+
54+
// eksResourceAccountAFSouth1 defines the AWS EKS account ID that provides node resources in af-south-1
55+
eksResourceAccountAFSouth1 = "877085696533"
56+
57+
// eksResourceAccountEUSouth1 defines the AWS EKS account ID that provides node resources in eu-south-1
58+
eksResourceAccountEUSouth1 = "590381155156"
59+
60+
// eksResourceAccountUSGovWest1 defines the AWS EKS account ID that provides node resources in us-gov-west-1
61+
eksResourceAccountUSGovWest1 = "013241004608"
62+
63+
// eksResourceAccountUSGovEast1 defines the AWS EKS account ID that provides node resources in us-gov-east-1
64+
eksResourceAccountUSGovEast1 = "151742754352"
65+
)
66+
67+
// Regions
68+
const (
69+
// RegionUSWest1 represents the US West Region North California
70+
RegionUSWest1 = "us-west-1"
71+
72+
// RegionUSWest2 represents the US West Region Oregon
73+
RegionUSWest2 = "us-west-2"
74+
75+
// RegionUSEast1 represents the US East Region North Virginia
76+
RegionUSEast1 = "us-east-1"
77+
78+
// RegionUSEast2 represents the US East Region Ohio
79+
RegionUSEast2 = "us-east-2"
80+
81+
// RegionCACentral1 represents the Canada Central Region
82+
RegionCACentral1 = "ca-central-1"
83+
84+
// RegionEUWest1 represents the EU West Region Ireland
85+
RegionEUWest1 = "eu-west-1"
86+
87+
// RegionEUWest2 represents the EU West Region London
88+
RegionEUWest2 = "eu-west-2"
89+
90+
// RegionEUWest3 represents the EU West Region Paris
91+
RegionEUWest3 = "eu-west-3"
92+
93+
// RegionEUNorth1 represents the EU North Region Stockholm
94+
RegionEUNorth1 = "eu-north-1"
95+
96+
// RegionEUCentral1 represents the EU Central Region Frankfurt
97+
RegionEUCentral1 = "eu-central-1"
98+
99+
// RegionEUSouth1 represents te Eu South Region Milan
100+
RegionEUSouth1 = "eu-south-1"
101+
102+
// RegionAPNorthEast1 represents the Asia-Pacific North East Region Tokyo
103+
RegionAPNorthEast1 = "ap-northeast-1"
104+
105+
// RegionAPNorthEast2 represents the Asia-Pacific North East Region Seoul
106+
RegionAPNorthEast2 = "ap-northeast-2"
107+
108+
// RegionAPNorthEast3 represents the Asia-Pacific North East region Osaka
109+
RegionAPNorthEast3 = "ap-northeast-3"
110+
111+
// RegionAPSouthEast1 represents the Asia-Pacific South East Region Singapore
112+
RegionAPSouthEast1 = "ap-southeast-1"
113+
114+
// RegionAPSouthEast2 represents the Asia-Pacific South East Region Sydney
115+
RegionAPSouthEast2 = "ap-southeast-2"
116+
117+
// RegionAPSouth1 represents the Asia-Pacific South Region Mumbai
118+
RegionAPSouth1 = "ap-south-1"
119+
120+
// RegionAPEast1 represents the Asia Pacific Region Hong Kong
121+
RegionAPEast1 = "ap-east-1"
122+
123+
// RegionMESouth1 represents the Middle East Region Bahrain
124+
RegionMESouth1 = "me-south-1"
125+
126+
// RegionSAEast1 represents the South America Region Sao Paulo
127+
RegionSAEast1 = "sa-east-1"
128+
129+
// RegionAFSouth1 represents the Africa Region Cape Town
130+
RegionAFSouth1 = "af-south-1"
131+
132+
// RegionCNNorthwest1 represents the China region Ningxia
133+
RegionCNNorthwest1 = "cn-northwest-1"
134+
135+
// RegionCNNorth1 represents the China region Beijing
136+
RegionCNNorth1 = "cn-north-1"
137+
138+
// RegionUSGovWest1 represents the region GovCloud (US-West)
139+
RegionUSGovWest1 = "us-gov-west-1"
140+
141+
// RegionUSGovEast1 represents the region GovCloud (US-East)
142+
RegionUSGovEast1 = "us-gov-east-1"
143+
144+
// DefaultRegion defines the default region, where to deploy the EKS cluster
145+
DefaultRegion = RegionUSWest2
146+
)
147+
148+
// SupportedRegions are the regions where EKS is available
149+
func SupportedRegions() []string {
150+
return []string{
151+
RegionUSWest1,
152+
RegionUSWest2,
153+
RegionUSEast1,
154+
RegionUSEast2,
155+
RegionCACentral1,
156+
RegionEUWest1,
157+
RegionEUWest2,
158+
RegionEUWest3,
159+
RegionEUNorth1,
160+
RegionEUCentral1,
161+
RegionEUSouth1,
162+
RegionAPNorthEast1,
163+
RegionAPNorthEast2,
164+
RegionAPNorthEast3,
165+
RegionAPSouthEast1,
166+
RegionAPSouthEast2,
167+
RegionAPSouth1,
168+
RegionAPEast1,
169+
RegionMESouth1,
170+
RegionSAEast1,
171+
RegionAFSouth1,
172+
// RegionCNNorthwest1,
173+
// RegionCNNorth1,
174+
// RegionUSGovWest1,
175+
// RegionUSGovEast1,
176+
}
177+
}
178+
179+
func EKSResourceAccountID(region string) string {
180+
switch region {
181+
case RegionAPEast1:
182+
return eksResourceAccountAPEast1
183+
case RegionMESouth1:
184+
return eksResourceAccountMESouth1
185+
case RegionCNNorthwest1:
186+
return eksResourceAccountCNNorthWest1
187+
case RegionCNNorth1:
188+
return eksResourceAccountCNNorth1
189+
case RegionUSGovWest1:
190+
return eksResourceAccountUSGovWest1
191+
case RegionUSGovEast1:
192+
return eksResourceAccountUSGovEast1
193+
case RegionAFSouth1:
194+
return eksResourceAccountAFSouth1
195+
case RegionEUSouth1:
196+
return eksResourceAccountEUSouth1
197+
default:
198+
return eksResourceAccountStandard
199+
}
200+
}
201+
202+
func main() {
203+
destFile := mustExtractArg()
204+
k8sVersion := "1.18"
205+
regions := map[string]map[string]string{}
206+
for _, region := range SupportedRegions() {
207+
fmt.Print(region)
208+
sess := session.New(&aws.Config{Region: aws.String(region)})
209+
svc := ec2.New(sess)
210+
cpuAMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-node-%s-v*", k8sVersion))
211+
if err != nil {
212+
log.Fatal(err.Error())
213+
}
214+
acceleratedAMI, err := FindImage(svc, EKSResourceAccountID(region), fmt.Sprintf("amazon-eks-gpu-node-%s-v*", k8sVersion))
215+
if err != nil {
216+
log.Fatal(err.Error())
217+
}
218+
regions[region] = map[string]string{
219+
"cpu": cpuAMI,
220+
"accelerated": acceleratedAMI,
221+
}
222+
fmt.Println(" ✓")
223+
}
224+
225+
k8sVersionMap := map[string]interface{}{}
226+
k8sVersionMap[k8sVersion] = regions
227+
marshalledBytes, err := json.MarshalIndent(k8sVersionMap, "", "\t")
228+
if err != nil {
229+
log.Fatal(err.Error())
230+
}
231+
232+
marshalledBytes = append(marshalledBytes, []byte("\n")...)
233+
234+
err = ioutil.WriteFile(destFile, marshalledBytes, 0664)
235+
if err != nil {
236+
log.Fatal(err.Error())
237+
}
238+
}
239+
240+
func FindImage(ec2api ec2iface.EC2API, ownerAccount, namePattern string) (string, error) {
241+
input := &ec2.DescribeImagesInput{
242+
Owners: []*string{&ownerAccount},
243+
Filters: []*ec2.Filter{
244+
{
245+
Name: aws.String("name"),
246+
Values: []*string{&namePattern},
247+
},
248+
{
249+
Name: aws.String("virtualization-type"),
250+
Values: []*string{aws.String("hvm")},
251+
},
252+
{
253+
Name: aws.String("root-device-type"),
254+
Values: []*string{aws.String("ebs")},
255+
},
256+
{
257+
Name: aws.String("is-public"),
258+
Values: []*string{aws.String("true")},
259+
},
260+
{
261+
Name: aws.String("state"),
262+
Values: []*string{aws.String("available")},
263+
},
264+
},
265+
}
266+
267+
output, err := ec2api.DescribeImages(input)
268+
if err != nil {
269+
return "", errors.Wrapf(err, "error querying AWS for images")
270+
}
271+
272+
if len(output.Images) < 1 {
273+
return "", nil
274+
}
275+
276+
if len(output.Images) == 1 {
277+
return *output.Images[0].ImageId, nil
278+
}
279+
280+
// Sort images so newest is first
281+
sort.Slice(output.Images, func(i, j int) bool {
282+
//nolint:gosec
283+
creationLeft, _ := time.Parse(time.RFC3339, *output.Images[i].CreationDate)
284+
//nolint:gosec
285+
creationRight, _ := time.Parse(time.RFC3339, *output.Images[j].CreationDate)
286+
return creationLeft.After(creationRight)
287+
})
288+
289+
return *output.Images[0].ImageId, nil
290+
}
291+
292+
func mustExtractArg() string {
293+
if len(os.Args) != 2 {
294+
fmt.Println("usage: go run generate_ami_mapping.go <abs_dest_path>")
295+
os.Exit(1)
296+
}
297+
298+
return os.Args[1]
299+
}

‎build/lint.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ output=$(cd "$ROOT" && find . -type f \
8585
! -path "**/.idea/*" \
8686
! -path "**/.history/*" \
8787
! -path "**/__pycache__/*" \
88+
! -path "**/.pytest_cache/*" \
89+
! -path "**.egg-info/*" \
8890
! -path "./test/*" \
8991
! -path "./dev/config/*" \
9092
! -path "./bin/*" \
@@ -94,6 +96,7 @@ output=$(cd "$ROOT" && find . -type f \
9496
! -name "*requirements.txt" \
9597
! -name "go.*" \
9698
! -name "*.md" \
99+
! -name "*.json" \
97100
! -name ".*" \
98101
! -name "*.bin" \
99102
! -name "Dockerfile" \
@@ -113,6 +116,8 @@ if [ "$is_release_branch" = "true" ]; then
113116
! -path "**/.idea/*" \
114117
! -path "**/.history/*" \
115118
! -path "**/__pycache__/*" \
119+
! -path "**/.pytest_cache/*" \
120+
! -path "**.egg-info/*" \
116121
! -path "./dev/config/*" \
117122
! -path "./bin/*" \
118123
! -path "./.git/*" \
@@ -134,6 +139,8 @@ output=$(cd "$ROOT" && find . -type f \
134139
! -path "**/.history/*" \
135140
! -path "**/.vscode/*" \
136141
! -path "**/__pycache__/*" \
142+
! -path "**/.pytest_cache/*" \
143+
! -path "**.egg-info/*" \
137144
! -path "./dev/config/*" \
138145
! -path "./bin/*" \
139146
! -path "./.git/*" \
@@ -154,6 +161,8 @@ output=$(cd "$ROOT" && find . -type f \
154161
! -path "**/.history/*" \
155162
! -path "**/.vscode/*" \
156163
! -path "**/__pycache__/*" \
164+
! -path "**/.pytest_cache/*" \
165+
! -path "**.egg-info/*" \
157166
! -path "./dev/config/*" \
158167
! -path "./bin/*" \
159168
! -path "./.git/*" \
@@ -174,6 +183,8 @@ output=$(cd "$ROOT" && find . -type f \
174183
! -path "**/.idea/*" \
175184
! -path "**/.history/*" \
176185
! -path "**/__pycache__/*" \
186+
! -path "**/.pytest_cache/*" \
187+
! -path "**.egg-info/*" \
177188
! -path "./dev/config/*" \
178189
! -path "./bin/*" \
179190
! -path "./.git/*" \
@@ -194,6 +205,8 @@ output=$(cd "$ROOT" && find . -type f \
194205
! -path "**/.history/*" \
195206
! -path "**/.vscode/*" \
196207
! -path "**/__pycache__/*" \
208+
! -path "**/.pytest_cache/*" \
209+
! -path "**.egg-info/*" \
197210
! -path "./dev/config/*" \
198211
! -path "./bin/*" \
199212
! -path "./.git/*" \

0 commit comments

Comments
 (0)