diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml index efc6a62..1b53366 100644 --- a/.github/workflows/docker-release.yaml +++ b/.github/workflows/docker-release.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/github-release.yaml b/.github/workflows/github-release.yaml index 7f91c7a..286617b 100644 --- a/.github/workflows/github-release.yaml +++ b/.github/workflows/github-release.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v3 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0128692..0a8dc82 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v3 diff --git a/README.md b/README.md index 52ba7b0..3c7e39f 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ Summary: User can put AWS secret ARN as environment variable value. The `secrets-init` will resolve any environment value, using specified ARN, to referenced secret value. If the secret is saved as a Key/Value pair, all the keys are applied to as environment variables and passed. The environment variable passed is ignored unless it is inside the key/value pair. + +#### Simple Key/Value Secrets + ```sh # environment variable passed to `secrets-init` MY_DB_PASSWORD=arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT_ID:secret:mydbpassword-cdma3 @@ -40,6 +43,48 @@ MY_DB_PASSWORD=arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT_ID:secret:mydbpas MY_DB_PASSWORD=very-secret-password ``` +#### JSON Secrets with Nested Key Extraction + +For JSON secrets, you can extract specific nested values using the `$` syntax followed by [gjson](https://github.com/tidwall/gjson) path expressions: + +> **Note:** If your secret name contains multiple `$` characters, only the first `$` is used to split the secret ARN from the nested key path. For example, `arn:aws:secretsmanager:mysecret$level$key` will extract the key `level$key` from the secret named `arn:aws:secretsmanager:mysecret`. + +```sh +# Extract a top-level key from JSON +MY_DB_PASSWORD=arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT_ID:secret:mydbpassword-cdma3$password + +# Extract a nested key from JSON +MY_REDIS_PASSWORD=arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT_ID:secret:mydbpassword-cdma3$redis.password + +# Extract from array elements +MY_API_KEY=arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT_ID:secret:apikeys-cdma3$keys.0.value + +# If the nested key doesn't exist, the original ARN is preserved +MY_INEXISTENT_KEY=arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT_ID:secret:mydbpassword-cdma3$inexistent.key +``` + +**Example JSON Secret:** +```json +{ + "password": "secret-password", + "redis": { + "password": "secret-redis-password" + }, + "keys": [ + {"value": "api-key-1"}, + {"value": "api-key-2"} + ] +} +``` + +**Resulting Environment Variables:** +```sh +MY_DB_PASSWORD=secret-password +MY_REDIS_PASSWORD=secret-redis-password +MY_API_KEY=api-key-1 +MY_INEXISTENT_KEY=arn:aws:secretsmanager:us-east-1:123456789012:secret:mydbpassword-cdma3$inexistent.key +``` + ### Integration with AWS Systems Manager Parameter Store It is possible to use AWS Systems Manager Parameter Store to store application parameters and secrets. diff --git a/go.mod b/go.mod index 03e25b2..fe1c93d 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,9 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.15.0 // indirect diff --git a/go.sum b/go.sum index f5de4ec..126c917 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/urfave/cli/v2 v2.23.0 h1:pkly7gKIeYv3olPAeNajNpLjeJrmTPYCoZWaV+2VfvE= github.com/urfave/cli/v2 v2.23.0/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= diff --git a/main.go b/main.go index 57cdbef..76d6714 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" + cli "github.com/urfave/cli/v2" "golang.org/x/sys/unix" //nolint:gci ) diff --git a/pkg/secrets/aws/secrets.go b/pkg/secrets/aws/secrets.go index 28cb6c6..3184376 100644 --- a/pkg/secrets/aws/secrets.go +++ b/pkg/secrets/aws/secrets.go @@ -14,6 +14,8 @@ import ( "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/pkg/errors" //nolint:gci + + "github.com/tidwall/gjson" ) const ( @@ -53,24 +55,35 @@ func (sp *SecretsProvider) ResolveSecrets(_ context.Context, vars []string) ([]s kv := strings.Split(env, "=") key, value := kv[0], kv[1] if strings.HasPrefix(value, "arn:aws:secretsmanager") || strings.HasPrefix(value, "arn:aws-cn:secretsmanager") { + secretKey, nestedKey, _ := strings.Cut(value, "$") + // get secret value - secret, err := sp.sm.GetSecretValue(&secretsmanager.GetSecretValueInput{SecretId: &value}) + secret, err := sp.sm.GetSecretValue(&secretsmanager.GetSecretValueInput{SecretId: &secretKey}) if err != nil { return vars, errors.Wrap(err, "failed to get secret from AWS Secrets Manager") } + if IsJSON(secret.SecretString) { - var keyValueSecret map[string]string - err = json.Unmarshal([]byte(*secret.SecretString), &keyValueSecret) - if err != nil { - return vars, errors.Wrap(err, "failed to decode key/value secret") - } - for key, value := range keyValueSecret { - e := key + "=" + value - envs = append(envs, e) + if nestedKey != "" { + jsonValue := gjson.Get(*secret.SecretString, nestedKey) + if jsonValue.Exists() { + env = key + "=" + jsonValue.String() + } + } else { + var keyValueSecret map[string]string + err = json.Unmarshal([]byte(*secret.SecretString), &keyValueSecret) + if err != nil { + return vars, errors.Wrap(err, "failed to decode key/value secret") + } + for key, value := range keyValueSecret { + e := key + "=" + value + envs = append(envs, e) + } + continue // We continue to not add this ENV variable but only the environment variables that exists in the JSON } - continue // We continue to not add this ENV variable but only the environment variables that exists in the JSON + } else { + env = key + "=" + *secret.SecretString } - env = key + "=" + *secret.SecretString } else if (strings.HasPrefix(value, "arn:aws:ssm") || strings.HasPrefix(value, "arn:aws-cn:ssm")) && strings.Contains(value, ":parameter/") { tokens := strings.Split(value, ":") // valid parameter ARN arn:aws:ssm:REGION:ACCOUNT:parameter/PATH diff --git a/pkg/secrets/aws/secrets_test.go b/pkg/secrets/aws/secrets_test.go index 1a60bd3..b452d0f 100644 --- a/pkg/secrets/aws/secrets_test.go +++ b/pkg/secrets/aws/secrets_test.go @@ -92,6 +92,62 @@ func TestSecretsProvider_ResolveSecrets(t *testing.T) { return &sp }, }, + { + name: "get secret from from Secrets Manager json with nested key", + vars: []string{ + "test-secret-1=arn:aws:secretsmanager:12345678-json-nested$password", + "test-secret-2=arn:aws:secretsmanager:12345678-json-nested$redis.password", + "test-secret-3=arn:aws:secretsmanager:12345678-json-nested$inexistent", + }, + want: []string{ + "test-secret-1=secret-password", + "test-secret-2=secret-redis-password", + "test-secret-3=arn:aws:secretsmanager:12345678-json-nested$inexistent", + }, + mockServiceProvider: func(mockSM *mocks.SecretsManagerAPI, mockSSM *mocks.SSMAPI) secrets.Provider { + sp := SecretsProvider{sm: mockSM, ssm: mockSSM} + vars := map[string]string{ + "arn:aws:secretsmanager:12345678-json-nested": "{\n \"TEST_1\": \"test-secret-value-1\",\n \"TEST_2\": \"test-secret-value-2\"\n, \"password\": \"secret-password\", \"redis\": {\"password\": \"secret-redis-password\"}\n}", + } + for n, v := range vars { + name := n + value := v + valueInput := secretsmanager.GetSecretValueInput{SecretId: &name} + valueOutput := secretsmanager.GetSecretValueOutput{SecretString: &value} + mockSM.On("GetSecretValue", &valueInput).Return(&valueOutput, nil) + } + return &sp + }, + }, + { + name: "get secret from Secrets Manager json with array elements", + vars: []string{ + "api-key-1=arn:aws:secretsmanager:12345678-json-array$keys.0.value", + "api-key-2=arn:aws:secretsmanager:12345678-json-array$keys.1.value", + "api-key-3=arn:aws:secretsmanager:12345678-json-array$keys.2.value", + "non-existent=arn:aws:secretsmanager:12345678-json-array$keys.5.value", + }, + want: []string{ + "api-key-1=api-key-1-value", + "api-key-2=api-key-2-value", + "api-key-3=api-key-3-value", + "non-existent=arn:aws:secretsmanager:12345678-json-array$keys.5.value", + }, + mockServiceProvider: func(mockSM *mocks.SecretsManagerAPI, mockSSM *mocks.SSMAPI) secrets.Provider { + sp := SecretsProvider{sm: mockSM, ssm: mockSSM} + vars := map[string]string{ + "arn:aws:secretsmanager:12345678-json-array": "{\n \"keys\": [\n {\"value\": \"api-key-1-value\"},\n {\"value\": \"api-key-2-value\"},\n {\"value\": \"api-key-3-value\"}\n ]\n}", + } + for n, v := range vars { + name := n + value := v + valueInput := secretsmanager.GetSecretValueInput{SecretId: &name} + valueOutput := secretsmanager.GetSecretValueOutput{SecretString: &value} + mockSM.On("GetSecretValue", &valueInput).Return(&valueOutput, nil) + } + return &sp + }, + }, { name: "no secrets", vars: []string{ @@ -164,6 +220,24 @@ func TestSecretsProvider_ResolveSecrets(t *testing.T) { return &sp }, }, + { + name: "only first occurrence of separator is used", + vars: []string{ + "test-secret=arn:aws:secretsmanager:multi$level$key", + }, + want: []string{ + "test-secret=the-value-for-level$key", + }, + mockServiceProvider: func(mockSM *mocks.SecretsManagerAPI, mockSSM *mocks.SSMAPI) secrets.Provider { + sp := SecretsProvider{sm: mockSM, ssm: mockSSM} + secretName := "arn:aws:secretsmanager:multi" + secretValue := "{\"level$key\": \"the-value-for-level$key\"}" + valueInput := secretsmanager.GetSecretValueInput{SecretId: &secretName} + valueOutput := secretsmanager.GetSecretValueOutput{SecretString: &secretValue} + mockSM.On("GetSecretValue", &valueInput).Return(&valueOutput, nil) + return &sp + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/secrets/google/interface.go b/pkg/secrets/google/interface.go index 6e40f36..97d9a5b 100644 --- a/pkg/secrets/google/interface.go +++ b/pkg/secrets/google/interface.go @@ -3,7 +3,7 @@ package google import ( "context" - "github.com/googleapis/gax-go/v2" + gax "github.com/googleapis/gax-go/v2" secretspb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" )