Skip to content

Commit 0fafdbe

Browse files
authored
Add support for private docker image registries (#1460)
1 parent 72624c9 commit 0fafdbe

File tree

10 files changed

+375
-25
lines changed

10 files changed

+375
-25
lines changed

cli/local/validations.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ func ValidateLocalAPIs(apis []userconfig.API, projectFiles ProjectFiles, awsClie
118118
for i := range apis {
119119
api := &apis[i]
120120

121-
if err := spec.ValidateAPI(api, projectFiles, types.LocalProviderType, awsClient); err != nil {
121+
if err := spec.ValidateAPI(api, projectFiles, types.LocalProviderType, awsClient, nil); err != nil {
122122
return errors.Wrap(err, api.Identify())
123123
}
124124

docs/guides/private-docker.md

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Private docker registry
2+
3+
_WARNING: you are on the master branch, please refer to the docs on the branch that matches your `cortex version`_
4+
5+
Until [#1459](https://github.com/cortexlabs/cortex/issues/1459) is addressed, you can use a private docker registry for your Predictor images by following this guide.
6+
7+
## Local
8+
9+
When running Cortex locally, you can use private Docker images by running `docker login` and then `docker pull <your_image>`. The Docker image will be present on your machine, and will be accessible by Cortex the next time you run `cortex deploy`.
10+
11+
## Cluster
12+
13+
### Step 1
14+
15+
Install and configure kubectl ([instructions](kubectl-setup.md)).
16+
17+
### Step 2
18+
19+
Set the following environment variables, replacing the placeholders with your docker username and password:
20+
21+
```bash
22+
DOCKER_USERNAME=***
23+
DOCKER_PASSWORD=***
24+
```
25+
26+
Run the following commands:
27+
28+
```bash
29+
kubectl create secret docker-registry registry-credentials \
30+
--namespace default \
31+
--docker-username=$DOCKER_USERNAME \
32+
--docker-password=$DOCKER_PASSWORD
33+
34+
kubectl patch serviceaccount default \
35+
--namespace default \
36+
-p "{\"imagePullSecrets\": [{\"name\": \"registry-credentials\"}]}"
37+
```
38+
39+
### Updating your credentials
40+
41+
To remove your docker credentials from the cluster, run this command:
42+
43+
```bash
44+
kubectl delete secret --namespace default registry-credentials
45+
```
46+
47+
Then repeat step 2 above with your updated credentials.
48+
49+
### Removing your credentials
50+
51+
To remove your docker credentials from the cluster, run the following commands:
52+
53+
```bash
54+
kubectl delete secret --namespace default registry-credentials
55+
56+
kubectl patch serviceaccount default \
57+
--namespace default \
58+
-p "{\"imagePullSecrets\": []}"
59+
```

docs/summary.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
* [SSH into worker instance](guides/ssh-instance.md)
7272
* [Single node deployment](guides/single-node-deployment.md)
7373
* [Set up kubectl](guides/kubectl-setup.md)
74+
* [Private docker registry](guides/private-docker.md)
7475

7576
## Contributing
7677

pkg/lib/docker/docker.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
"github.com/cortexlabs/cortex/pkg/lib/parallel"
3838
"github.com/cortexlabs/cortex/pkg/lib/print"
3939
"github.com/cortexlabs/cortex/pkg/lib/slices"
40+
"github.com/cortexlabs/cortex/pkg/types"
4041
dockertypes "github.com/docker/docker/api/types"
4142
dockerclient "github.com/docker/docker/client"
4243
"github.com/docker/docker/pkg/jsonmessage"
@@ -47,15 +48,15 @@ var NoAuth string
4748

4849
var _cachedClient *Client
4950

51+
func init() {
52+
NoAuth, _ = EncodeAuthConfig(dockertypes.AuthConfig{})
53+
}
54+
5055
type Client struct {
5156
*dockerclient.Client
5257
Info dockertypes.Info
5358
}
5459

55-
func init() {
56-
NoAuth, _ = EncodeAuthConfig(dockertypes.AuthConfig{})
57-
}
58-
5960
func GetDockerClient() (*Client, error) {
6061
if _cachedClient != nil {
6162
return _cachedClient, nil
@@ -146,7 +147,7 @@ func PullImage(image string, encodedAuthConfig string, pullVerbosity PullVerbosi
146147
return false, err
147148
}
148149

149-
if err := CheckLocalImageAccessible(dockerClient, image); err == nil {
150+
if err := CheckImageExistsLocally(dockerClient, image); err == nil {
150151
return false, nil
151152
}
152153

@@ -314,14 +315,14 @@ func EncodeAuthConfig(authConfig dockertypes.AuthConfig) (string, error) {
314315
return registryAuth, nil
315316
}
316317

317-
func CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string) error {
318+
func CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string, providerType types.ProviderType) error {
318319
if _, err := dockerClient.DistributionInspect(context.Background(), dockerImage, authConfig); err != nil {
319-
return ErrorImageInaccessible(dockerImage, err)
320+
return ErrorImageInaccessible(dockerImage, providerType, err)
320321
}
321322
return nil
322323
}
323324

324-
func CheckLocalImageAccessible(dockerClient *Client, dockerImage string) error {
325+
func CheckImageExistsLocally(dockerClient *Client, dockerImage string) error {
325326
images, err := dockerClient.ImageList(context.Background(), dockertypes.ImageListOptions{})
326327
if err != nil {
327328
return WrapDockerError(err)
@@ -337,7 +338,8 @@ func CheckLocalImageAccessible(dockerClient *Client, dockerImage string) error {
337338
return nil
338339
}
339340
}
340-
return ErrorImageInaccessible(dockerImage, nil)
341+
342+
return ErrorImageDoesntExistLocally(dockerImage)
341343
}
342344

343345
func ExtractImageTag(dockerImage string) string {

pkg/lib/docker/errors.go

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@ import (
2121
"runtime"
2222
"strings"
2323

24+
"github.com/cortexlabs/cortex/pkg/consts"
2425
"github.com/cortexlabs/cortex/pkg/lib/errors"
26+
"github.com/cortexlabs/cortex/pkg/types"
2527
)
2628

2729
const (
28-
ErrConnectToDockerDaemon = "docker.connect_to_docker_daemon"
29-
ErrDockerPermissions = "docker.docker_permissions"
30-
ErrImageInaccessible = "docker.image_inaccessible"
30+
ErrConnectToDockerDaemon = "docker.connect_to_docker_daemon"
31+
ErrDockerPermissions = "docker.docker_permissions"
32+
ErrImageDoesntExistLocally = "docker.image_doesnt_exist_locally"
33+
ErrImageInaccessible = "docker.image_inaccessible"
3134
)
3235

3336
func ErrorConnectToDockerDaemon() error {
@@ -56,10 +59,29 @@ func ErrorDockerPermissions(err error) error {
5659
})
5760
}
5861

59-
func ErrorImageInaccessible(image string, cause error) error {
62+
func ErrorImageDoesntExistLocally(image string) error {
63+
return errors.WithStack(&errors.Error{
64+
Kind: ErrImageDoesntExistLocally,
65+
Message: fmt.Sprintf("%s does not exist locally; download it with `docker pull %s` (if your registry is private, run `docker login` first)", image, image),
66+
})
67+
}
68+
69+
func ErrorImageInaccessible(image string, providerType types.ProviderType, cause error) error {
6070
message := fmt.Sprintf("%s is not accessible", image)
71+
6172
if cause != nil {
62-
message += "\n" + errors.Message(cause) // add \n because docker client errors are
73+
message += "\n" + errors.Message(cause) // add \n because docker client errors are verbose but useful
74+
}
75+
76+
if providerType == types.LocalProviderType {
77+
message += fmt.Sprintf("\n\nyou can download your image with `docker pull %s` and try this command again", image)
78+
if strings.Contains(cause.Error(), "authorized") || strings.Contains(cause.Error(), "authentication") {
79+
message += " (if your registry is private, run `docker login` first)"
80+
}
81+
} else if providerType == types.AWSProviderType {
82+
if strings.Contains(cause.Error(), "authorized") || strings.Contains(cause.Error(), "authentication") {
83+
message += fmt.Sprintf("\n\nif you would like to use a private docker registry, see https://docs.cortex.dev/v/%s/guides/private-docker", consts.CortexVersionMinor)
84+
}
6385
}
6486

6587
return errors.WithStack(&errors.Error{

pkg/lib/k8s/k8s.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ type Client struct {
5656
nodeClient kclientcore.NodeInterface
5757
serviceClient kclientcore.ServiceInterface
5858
configMapClient kclientcore.ConfigMapInterface
59+
secretClient kclientcore.SecretInterface
5960
deploymentClient kclientapps.DeploymentInterface
6061
jobClient kclientbatch.JobInterface
6162
ingressClient kclientextensions.IngressInterface
@@ -100,6 +101,7 @@ func New(namespace string, inCluster bool) (*Client, error) {
100101
client.nodeClient = client.clientset.CoreV1().Nodes()
101102
client.serviceClient = client.clientset.CoreV1().Services(namespace)
102103
client.configMapClient = client.clientset.CoreV1().ConfigMaps(namespace)
104+
client.secretClient = client.clientset.CoreV1().Secrets(namespace)
103105
client.deploymentClient = client.clientset.AppsV1().Deployments(namespace)
104106
client.jobClient = client.clientset.BatchV1().Jobs(namespace)
105107
client.ingressClient = client.clientset.ExtensionsV1beta1().Ingresses(namespace)

pkg/lib/k8s/secret.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
/*
2+
Copyright 2020 Cortex Labs, Inc.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package k8s
18+
19+
import (
20+
"context"
21+
22+
"github.com/cortexlabs/cortex/pkg/lib/errors"
23+
kcore "k8s.io/api/core/v1"
24+
kerrors "k8s.io/apimachinery/pkg/api/errors"
25+
kmeta "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
klabels "k8s.io/apimachinery/pkg/labels"
27+
)
28+
29+
var _secretTypeMeta = kmeta.TypeMeta{
30+
APIVersion: "v1",
31+
Kind: "Secret",
32+
}
33+
34+
type SecretSpec struct {
35+
Name string
36+
Data map[string][]byte
37+
Labels map[string]string
38+
Annotations map[string]string
39+
}
40+
41+
func Secret(spec *SecretSpec) *kcore.Secret {
42+
secret := &kcore.Secret{
43+
TypeMeta: _secretTypeMeta,
44+
ObjectMeta: kmeta.ObjectMeta{
45+
Name: spec.Name,
46+
Labels: spec.Labels,
47+
Annotations: spec.Annotations,
48+
},
49+
Data: spec.Data,
50+
}
51+
return secret
52+
}
53+
54+
func (c *Client) CreateSecret(secret *kcore.Secret) (*kcore.Secret, error) {
55+
secret.TypeMeta = _secretTypeMeta
56+
secret, err := c.secretClient.Create(context.Background(), secret, kmeta.CreateOptions{})
57+
if err != nil {
58+
return nil, errors.WithStack(err)
59+
}
60+
return secret, nil
61+
}
62+
63+
func (c *Client) UpdateSecret(secret *kcore.Secret) (*kcore.Secret, error) {
64+
secret.TypeMeta = _secretTypeMeta
65+
secret, err := c.secretClient.Update(context.Background(), secret, kmeta.UpdateOptions{})
66+
if err != nil {
67+
return nil, errors.WithStack(err)
68+
}
69+
return secret, nil
70+
}
71+
72+
func (c *Client) ApplySecret(secret *kcore.Secret) (*kcore.Secret, error) {
73+
existing, err := c.GetSecret(secret.Name)
74+
if err != nil {
75+
return nil, err
76+
}
77+
if existing == nil {
78+
return c.CreateSecret(secret)
79+
}
80+
return c.UpdateSecret(secret)
81+
}
82+
83+
func (c *Client) GetSecret(name string) (*kcore.Secret, error) {
84+
secret, err := c.secretClient.Get(context.Background(), name, kmeta.GetOptions{})
85+
if kerrors.IsNotFound(err) {
86+
return nil, nil
87+
}
88+
if err != nil {
89+
return nil, errors.WithStack(err)
90+
}
91+
secret.TypeMeta = _secretTypeMeta
92+
return secret, nil
93+
}
94+
95+
func (c *Client) GetSecretData(name string) (map[string][]byte, error) {
96+
secret, err := c.GetSecret(name)
97+
if err != nil {
98+
return nil, err
99+
}
100+
if secret == nil {
101+
return nil, nil
102+
}
103+
return secret.Data, nil
104+
}
105+
106+
func (c *Client) DeleteSecret(name string) (bool, error) {
107+
err := c.secretClient.Delete(context.Background(), name, _deleteOpts)
108+
if kerrors.IsNotFound(err) {
109+
return false, nil
110+
}
111+
if err != nil {
112+
return false, errors.WithStack(err)
113+
}
114+
return true, nil
115+
}
116+
117+
func (c *Client) ListSecrets(opts *kmeta.ListOptions) ([]kcore.Secret, error) {
118+
if opts == nil {
119+
opts = &kmeta.ListOptions{}
120+
}
121+
secretList, err := c.secretClient.List(context.Background(), *opts)
122+
if err != nil {
123+
return nil, errors.WithStack(err)
124+
}
125+
for i := range secretList.Items {
126+
secretList.Items[i].TypeMeta = _secretTypeMeta
127+
}
128+
return secretList.Items, nil
129+
}
130+
131+
func (c *Client) ListSecretsByLabels(labels map[string]string) ([]kcore.Secret, error) {
132+
opts := &kmeta.ListOptions{
133+
LabelSelector: klabels.SelectorFromSet(labels).String(),
134+
}
135+
return c.ListSecrets(opts)
136+
}
137+
138+
func (c *Client) ListSecretsByLabel(labelKey string, labelValue string) ([]kcore.Secret, error) {
139+
return c.ListSecretsByLabels(map[string]string{labelKey: labelValue})
140+
}
141+
142+
func (c *Client) ListSecretsWithLabelKeys(labelKeys ...string) ([]kcore.Secret, error) {
143+
opts := &kmeta.ListOptions{
144+
LabelSelector: LabelExistsSelector(labelKeys...),
145+
}
146+
return c.ListSecrets(opts)
147+
}
148+
149+
func SecretMap(secrets []kcore.Secret) map[string]kcore.Secret {
150+
secretMap := map[string]kcore.Secret{}
151+
for _, secret := range secrets {
152+
secretMap[secret.Name] = secret
153+
}
154+
return secretMap
155+
}

pkg/operator/resources/validations.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func ValidateClusterAPIs(apis []userconfig.API, projectFiles spec.ProjectFiles)
102102
for i := range apis {
103103
api := &apis[i]
104104
if api.Kind == userconfig.RealtimeAPIKind || api.Kind == userconfig.BatchAPIKind {
105-
if err := spec.ValidateAPI(api, projectFiles, types.AWSProviderType, config.AWS); err != nil {
105+
if err := spec.ValidateAPI(api, projectFiles, types.AWSProviderType, config.AWS, config.K8s); err != nil {
106106
return errors.Wrap(err, api.Identify())
107107
}
108108
if err := validateK8s(api, virtualServices, maxMem); err != nil {

0 commit comments

Comments
 (0)