diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebccfcd..e0aae4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,9 +14,10 @@ permissions: jobs: set-version: + name: Set Version runs-on: ubuntu-latest container: - image: mcr.microsoft.com/dotnet/sdk:6.0 + image: mcr.microsoft.com/dotnet/sdk:10.0 outputs: semVer: ${{ steps.gitversion.outputs.semVer }} steps: @@ -30,11 +31,11 @@ jobs: git config user.email ${{ github.actor }}-ci@gha.org git config user.name ${{ github.actor }} - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v3.0.0 + uses: gittools/actions/gitversion/setup@v4.1.0 with: - versionSpec: "5.x" + versionSpec: "6.x" - name: Set SemVer Version - uses: gittools/actions/gitversion/execute@v3.0.0 + uses: gittools/actions/gitversion/execute@v4.1.0 id: gitversion - name: echo VERSIONS @@ -44,6 +45,7 @@ jobs: test: runs-on: ubuntu-latest + name: Run Tests needs: set-version env: SEMVER: ${{ needs.set-version.outputs.semVer }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ddc903..405c112 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,10 +14,11 @@ permissions: jobs: set-version: + name: Set Version runs-on: ubuntu-latest if: ${{ github.event.workflow_run.head_branch == 'master' && github.event.workflow_run.conclusion == 'success' }} container: - image: mcr.microsoft.com/dotnet/sdk:6.0 + image: mcr.microsoft.com/dotnet/sdk:10.0 outputs: semVer: ${{ steps.gitversion.outputs.semVer }} steps: @@ -31,14 +32,15 @@ jobs: git config user.email ${{ github.actor }}-ci@gha.org git config user.name ${{ github.actor }} - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v3.0.0 + uses: gittools/actions/gitversion/setup@v4.1.0 with: - versionSpec: '5.x' + versionSpec: '6.x' - name: Set SemVer Version - uses: gittools/actions/gitversion/execute@v3.0.0 + uses: gittools/actions/gitversion/execute@v4.1.0 id: gitversion release: + name: Release runs-on: ubuntu-latest needs: set-version env: @@ -66,7 +68,7 @@ jobs: generate_release_notes: true token: ${{ secrets.GITHUB_TOKEN }} files: ./dist/* - prerelease: false + prerelease: true - name: release library run: | diff --git a/Dockerfile b/Dockerfile index 8dd4e1a..5871513 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app COPY ./ /app RUN CGO_ENABLED=0 go build -mod=readonly -buildvcs=false \ - -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \ + -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \ -o bin/configmanager cmd/main.go FROM docker.io/alpine:3 diff --git a/README.md b/README.md index 635e376..5b73d7a 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Where `configVar` can be either a parseable string `'som3#!S$CRet'` or a number - Kubernetes - Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS#/$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases. + Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS:///$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases. - VMs @@ -86,7 +86,7 @@ The token is made up of the following parts: _An example token would look like this_ -#### `AWSSECRETS#/path/to/my/key|lookup.Inside.Object[meta=data]` +#### `AWSSECRETS:///path/to/my/key|lookup.Inside.Object[meta=data]` ### Implementation indicator @@ -156,7 +156,9 @@ See [examples of working with files](docs/examples.md#working-with-files) for mo The `[meta=data]` from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) - is the optional metadata about the target in the backing provider -IT must have this format `[key=value]` - IT IS OPTIONAL + +> IT must have this format `[key=value]` - IT IS OPTIONAL +> IT must be specified last - either after a path lookup or if there is no key look up path specified then after the full path The `key` and `value` would be provider specific. Meaning that different providers support different config, these values _CAN_ be safely omitted configmanager would just use the defaults where applicable or not specify the additional @@ -220,7 +222,7 @@ All the usual token rules apply e.g. of `keySeparator` For HashicorpVault the first part of the token needs to be the name of the mountpath. In Dev Vaults this is `"secret"`, e.g.: `VAULT://secret___demo/configmanager|test` -or if the secrets are at another location: `VAULT://another/mount/path__config/app1/db` +or if the secrets are at another location: `VAULT://another/mount/path___config/app1/db` The hardcoded separator cannot be modified and you must separate your `mountPath` with `___` (3x `_`) followed by the key to the secret. diff --git a/cmd/configmanager/configmanager.go b/cmd/configmanager/configmanager.go index 01af646..1eed6cb 100644 --- a/cmd/configmanager/configmanager.go +++ b/cmd/configmanager/configmanager.go @@ -5,11 +5,11 @@ import ( "fmt" "io" - "github.com/DevLabFoundry/configmanager/v2" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/spf13/cobra" ) @@ -46,8 +46,8 @@ func NewRootCmd(logger log.ILogger) *Root { //channelOut, channelErr io.Writer } rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.verbose, "verbose", "v", false, "Verbosity level") - rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "#", "Separator to use to mark concrete store and the key within it") - rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS#/token/map|key1") + rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "://", "Separator to use to mark concrete store and the key within it") + rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS:///token/map|key1") rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.enableEnvSubst, "enable-envsubst", "e", false, "Enable envsubst on input. This will fail on any unset or empty variables") addSubCmds(rc) return rc diff --git a/cmd/configmanager/configmanager_test.go b/cmd/configmanager/configmanager_test.go index fce284a..5089ad3 100644 --- a/cmd/configmanager/configmanager_test.go +++ b/cmd/configmanager/configmanager_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - cmd "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + cmd "github.com/DevLabFoundry/configmanager/v3/cmd/configmanager" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) type cmdTestInput struct { diff --git a/cmd/configmanager/fromfileinput.go b/cmd/configmanager/fromfileinput.go index 4b8e915..f531033 100644 --- a/cmd/configmanager/fromfileinput.go +++ b/cmd/configmanager/fromfileinput.go @@ -3,7 +3,7 @@ package cmd import ( "fmt" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" "github.com/spf13/cobra" ) diff --git a/cmd/main.go b/cmd/main.go index ee4999a..4f10c03 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,8 +4,8 @@ import ( "context" "os" - cfgmgr "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + cfgmgr "github.com/DevLabFoundry/configmanager/v3/cmd/configmanager" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) func main() { diff --git a/configmanager.go b/configmanager.go index a422dfb..f3f6134 100644 --- a/configmanager.go +++ b/configmanager.go @@ -2,50 +2,60 @@ package configmanager import ( "context" - "encoding/json" "errors" "fmt" - "regexp" + "io" + "slices" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/a8m/envsubst" - "gopkg.in/yaml.v3" ) const ( - TERMINATING_CHAR string = `[^\'\"\s\n\\\,]` + TERMINATING_CHAR string = `[^\'\"\s\n\\\,]` // :\@\?\/ ) // generateAPI type generateAPI interface { - Generate(tokens []string) (generator.ParsedMap, error) + Generate(tokens []string) (generator.ReplacedToken, error) } type ConfigManager struct { Config *config.GenVarsConfig generator generateAPI + logger log.ILogger } // New returns an initialised instance of ConfigManager // Uses default config for: // -// ``` -// outputPath = "" -// keySeparator = "|" -// tokenSeparator = "://" -// ``` +// outputPath = "" +// keySeparator = "|" +// tokenSeparator = "://" +// +// # Calling cm.Config.WithXXX() will overwrite the generator config +// +// Default logger will log to io.Discard +// Attach your own if you need via // -// Calling cm.Config.WithXXX() will overwrite the generator config +// WithLogger(l log.ILogger) *ConfigManager func New(ctx context.Context) *ConfigManager { cm := &ConfigManager{} cm.Config = config.NewConfig() cm.generator = generator.NewGenerator(ctx).WithConfig(cm.Config) + cm.logger = log.New(io.Discard) return cm } +func (c *ConfigManager) WithLogger(l log.ILogger) *ConfigManager { + c.logger = l + return c +} + // GeneratorConfig // Returns the gettable generator config func (c *ConfigManager) GeneratorConfig() *config.GenVarsConfig { @@ -60,25 +70,15 @@ func (c *ConfigManager) WithGenerator(generator generateAPI) *ConfigManager { // Retrieve gets a rawMap from a set implementation // will be empty if no matches found -func (c *ConfigManager) Retrieve(tokens []string) (generator.ParsedMap, error) { - return c.retrieve(tokens) -} - -func (c *ConfigManager) retrieve(tokens []string) (generator.ParsedMap, error) { +func (c *ConfigManager) Retrieve(tokens []string) (generator.ReplacedToken, error) { return c.generator.Generate(tokens) } var ErrEnvSubst = errors.New("envsubst enabled and errored on") -// RetrieveWithInputReplaced parses given input against all possible token strings -// using regex to grab a list of found tokens in the given string and returns the replaced string -func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) { +// RetrieveReplacedString parses given input against all possible token strings +func (c *ConfigManager) RetrieveReplacedString(input string) (string, error) { // replaces all env vars using strict mode of no unset and no empty - // - // NOTE: this happens before the FindTokens is called - // currently it uses a regex, and envsubst uses a more robust lexer => parser mechanism - // - // NOTE: configmanager needs an own lexer => parser to allow for easier modification extension in the future if c.GeneratorConfig().EnvSubstEnabled() { var err error input, err = envsubst.StringRestrictedNoDigit(input, true, true, false) @@ -86,7 +86,9 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) return "", fmt.Errorf("%w\n%v", ErrEnvSubst, err) } } - m, err := c.retrieve(FindTokens(input)) + + // calling the same Generate method with the input as single item in a slice + m, err := c.generator.Generate([]string{input}) if err != nil { return "", err @@ -95,19 +97,14 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) return replaceString(m, input), nil } -// FindTokens extracts all replaceable tokens -// from a given input string -func FindTokens(input string) []string { - tokens := []string{} - for k := range config.VarPrefix { - matches := regexp.MustCompile(regexp.QuoteMeta(string(k))+`.(`+TERMINATING_CHAR+`+)`).FindAllString(input, -1) - tokens = append(tokens, matches...) - } - return tokens +// RetrieveReplacedBytes is functionally identical RetrieveReplacedString +func (c *ConfigManager) RetrieveReplacedBytes(input []byte) ([]byte, error) { + r, err := c.RetrieveReplacedString(string(input)) + return []byte(r), err } // replaceString fills tokens in a provided input with their actual secret/config values -func replaceString(inputMap generator.ParsedMap, inputString string) string { +func replaceString(inputMap generator.ReplacedToken, inputString string) string { oldNew := []string(nil) // ordered values by index @@ -118,7 +115,7 @@ func replaceString(inputMap generator.ParsedMap, inputString string) string { return replacer.Replace(inputString) } -func orderedKeysList(inputMap generator.ParsedMap) []string { +func orderedKeysList(inputMap generator.ReplacedToken) []string { mkeys := inputMap.MapKeys() // order map by keys length so that when passed to the // replacer it will replace the longest first @@ -128,85 +125,3 @@ func orderedKeysList(inputMap generator.ParsedMap) []string { slices.Sort(mkeys) return mkeys } - -// RetrieveMarshalledJson -// -// It marshalls an input pointer value of a type with appropriate struct tags in JSON -// marshalls it into a string and runs the appropriate token replacement. -// and fills the same pointer value with the replaced fields. -// -// This is useful for when you have another tool or framework already passing you a known type. -// e.g. a CRD Spec in kubernetes - where you POSTed the json/yaml spec with tokens in it -// but now want to use them with tokens replaced for values in a stateless way. -// -// Enables you to store secrets in CRD Specs and other metadata your controller can use -func (cm *ConfigManager) RetrieveMarshalledJson(input any) error { - - // marshall type into a []byte - // with tokens in a string like object - rawBytes, err := json.Marshal(input) - if err != nil { - return err - } - // run the replacement of tokens for values - replacedString, err := cm.RetrieveWithInputReplaced(string(rawBytes)) - if err != nil { - return err - } - // replace the original pointer value with replaced tokens - if err := json.Unmarshal([]byte(replacedString), input); err != nil { - return err - } - return nil -} - -// RetrieveUnmarshalledFromJson -// It accepts an already marshalled byte slice and pointer to the value type. -// It fills the type with the replaced -func (c *ConfigManager) RetrieveUnmarshalledFromJson(input []byte, output any) error { - replaced, err := c.RetrieveWithInputReplaced(string(input)) - if err != nil { - return err - } - if err := json.Unmarshal([]byte(replaced), output); err != nil { - return err - } - return nil -} - -// RetrieveMarshalledYaml -// -// Same as RetrieveMarshalledJson -func (cm *ConfigManager) RetrieveMarshalledYaml(input any) error { - - // marshall type into a []byte - // with tokens in a string like object - rawBytes, err := yaml.Marshal(input) - if err != nil { - return err - } - // run the replacement of tokens for values - replacedString, err := cm.RetrieveWithInputReplaced(string(rawBytes)) - if err != nil { - return err - } - // replace the original pointer value with replaced tokens - if err := yaml.Unmarshal([]byte(replacedString), input); err != nil { - return err - } - return nil -} - -// RetrieveUnmarshalledFromYaml -// -// Same as RetrieveUnmarshalledFromJson -func (c *ConfigManager) RetrieveUnmarshalledFromYaml(input []byte, output any) error { - replaced, err := c.RetrieveWithInputReplaced(string(input)) - if err != nil { - return err - } - if err := yaml.Unmarshal([]byte(replaced), output); err != nil { - return err - } - return nil -} diff --git a/configmanager_test.go b/configmanager_test.go index 43e0ef8..1321232 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -2,28 +2,29 @@ package configmanager_test import ( "context" + "encoding/json" "fmt" "os" "reflect" - "sort" "testing" - "github.com/DevLabFoundry/configmanager/v2" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/go-test/deep" + "gopkg.in/yaml.v3" ) type mockGenerator struct { - generate func(tokens []string) (generator.ParsedMap, error) + generate func(tokens []string) (generator.ReplacedToken, error) } -func (m *mockGenerator) Generate(tokens []string) (generator.ParsedMap, error) { +func (m *mockGenerator) Generate(tokens []string) (generator.ReplacedToken, error) { if m.generate != nil { return m.generate(tokens) } - pm := generator.ParsedMap{} + pm := generator.ReplacedToken{} pm["FOO#/test"] = "val1" pm["ANOTHER://bar/quz"] = "fux" pm["ZODTHER://bar/quz"] = "xuf" @@ -63,15 +64,15 @@ func Test_Retrieve_from_token_list(t *testing.T) { } } -func Test_retrieveWithInputReplaced(t *testing.T) { +func Test_retrieveReplacedBytes(t *testing.T) { tests := map[string]struct { name string - input string + input []byte genvar *mockGenerator expect string }{ "strYaml": { - input: ` + input: []byte(` space: preserved indents: preserved arr: [ "FOO#/test" ] @@ -79,7 +80,7 @@ space: preserved arr: - "FOO#/test" - ANOTHER://bar/quz -`, +`), genvar: &mockGenerator{}, expect: ` space: preserved @@ -92,11 +93,11 @@ space: preserved `, }, "strToml": { - input: ` + input: []byte(` // TOML [[somestuff]] key = "FOO#/test" -`, +`), genvar: &mockGenerator{}, expect: ` // TOML @@ -105,14 +106,14 @@ key = "val1" `, }, "strTomlWithoutQuotes": { - input: ` + input: []byte(` // TOML [[somestuff]] key = FOO#/test,FOO#/test-FOO#/test key2 = FOO#/test key3 = FOO#/test key4 = FOO#/test -`, +`), genvar: &mockGenerator{}, expect: ` // TOML @@ -124,7 +125,7 @@ key4 = val1 `, }, "strTomlWithoutMultiline": { - input: ` + input: []byte(` export FOO='FOO#/test' export FOO1=FOO#/test export FOO2="FOO#/test" @@ -134,7 +135,7 @@ export FOO4=FOO#/test [[section]] foo23 = FOO#/test -`, +`), genvar: &mockGenerator{}, expect: ` export FOO='val1' @@ -149,7 +150,7 @@ foo23 = val1 `, }, "escaped input": { - input: `"{\"patchPayloadTemplate\":\"{\\\"password\\\":\\\"FOO#/test\\\",\\\"passwordConfirm\\\":\\\"FOO#/test\\\"}\\n\"}"`, + input: []byte(`"{\"patchPayloadTemplate\":\"{\\\"password\\\":\\\"FOO#/test\\\",\\\"passwordConfirm\\\":\\\"FOO#/test\\\"}\\n\"}"`), genvar: &mockGenerator{}, expect: `"{\"patchPayloadTemplate\":\"{\\\"password\\\":\\\"val1\\\",\\\"passwordConfirm\\\":\\\"val1\\\"}\\n\"}"`, }, @@ -159,11 +160,11 @@ foo23 = val1 t.Run(tt.name, func(t *testing.T) { cm := configmanager.New(context.TODO()) cm.WithGenerator(tt.genvar) - got, err := cm.RetrieveWithInputReplaced(tt.input) + got, err := cm.RetrieveReplacedBytes([]byte(tt.input)) if err != nil { t.Errorf("failed with %v", err) } - if got != tt.expect { + if string(got) != string(tt.expect) { t.Errorf(testutils.TestPhrase, got, tt.expect) } }) @@ -171,7 +172,6 @@ foo23 = val1 } func Test_replaceString_with_envsubst(t *testing.T) { - t.Parallel() ttests := map[string]struct { expect string setup func() func() @@ -198,7 +198,7 @@ func Test_replaceString_with_envsubst(t *testing.T) { cm := configmanager.New(context.TODO()) cm.WithGenerator(tt.genvar) cm.Config.WithEnvSubst(true) - got, err := cm.RetrieveWithInputReplaced(tt.input) + got, err := cm.RetrieveReplacedString(tt.input) if err != nil { t.Errorf("failed with %v", err) } @@ -264,8 +264,8 @@ var marshallTests = map[string]struct { }, generator: func(t *testing.T) *mockGenerator { m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) + m.generate = func(tokens []string) (generator.ReplacedToken, error) { + pm := make(generator.ReplacedToken) pm[testTokenAWS] = "baz" return pm, nil } @@ -284,8 +284,8 @@ var marshallTests = map[string]struct { }, generator: func(t *testing.T) *mockGenerator { m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) + m.generate = func(tokens []string) (generator.ReplacedToken, error) { + pm := make(generator.ReplacedToken) pm[testTokenAWS] = "baz" return pm, nil } @@ -294,30 +294,44 @@ var marshallTests = map[string]struct { }, } -func Test_RetrieveMarshalledJson(t *testing.T) { +func Test_RetrieveBytes_MarshalledJson(t *testing.T) { for name, tt := range marshallTests { t.Run(name, func(t *testing.T) { c := configmanager.New(context.TODO()) c.Config.WithTokenSeparator("://") c.WithGenerator(tt.generator(t)) - input := &tt.testType - err := c.RetrieveMarshalledJson(input) - MarhsalledHelper(t, err, input, &tt.expect) + b, err := json.Marshal(tt.testType) + if err != nil { + t.Fatal(err) + } + got, err := c.RetrieveReplacedBytes(b) + output := testNestedStruct{} + json.Unmarshal(got, &output) + MarhsalledHelper(t, err, &output, &tt.expect) }) } } -func Test_YamlRetrieveMarshalled(t *testing.T) { +// func Example_RetrieveReplacedBytesMarshalledJSON(t *testing.T) { +// return +// } + +func Test_RetrieveBytes_MarshalledYaml(t *testing.T) { for name, tt := range marshallTests { t.Run(name, func(t *testing.T) { c := configmanager.New(context.TODO()) c.Config.WithTokenSeparator("://") c.WithGenerator(tt.generator(t)) - input := &tt.testType - err := c.RetrieveMarshalledYaml(input) - MarhsalledHelper(t, err, input, &tt.expect) + b, err := yaml.Marshal(tt.testType) + if err != nil { + t.Fatal(err) + } + got, err := c.RetrieveReplacedBytes(b) + output := testNestedStruct{} + yaml.Unmarshal(got, &output) + MarhsalledHelper(t, err, &output, &tt.expect) }) } } @@ -332,261 +346,6 @@ func MarhsalledHelper(t *testing.T, err error, input, expectOut any) { } } -func Test_YamlRetrieveUnmarshalled(t *testing.T) { - ttests := map[string]struct { - input []byte - expect testNestedStruct - generator func(t *testing.T) *mockGenerator - }{ - "happy path complex struct complete": { - input: []byte(`foo: AWSSECRETS:///bar/foo -bar: quz -lol: - bla: booo - another: - number: 1235 - float: 123.09`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{ - Bla: "booo", - Another: testAnotherNEst{ - Number: 1235, - Float: 123.09, - }, - }, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - "complex struct - missing fields": { - input: []byte(`foo: AWSSECRETS:///bar/foo -bar: quz`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{}, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(tt.generator(t)) - output := &testNestedStruct{} - err := c.RetrieveUnmarshalledFromYaml(tt.input, output) - MarhsalledHelper(t, err, output, &tt.expect) - }) - } -} - -func Test_JsonRetrieveUnmarshalled(t *testing.T) { - tests := map[string]struct { - input []byte - expect testNestedStruct - generator func(t *testing.T) *mockGenerator - }{ - "happy path complex struct complete": { - input: []byte(`{"foo":"AWSSECRETS:///bar/foo","bar":"quz","lol":{"bla":"booo","another":{"number":1235,"float":123.09}}}`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{ - Bla: "booo", - Another: testAnotherNEst{ - Number: 1235, - Float: 123.09, - }, - }, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - "complex struct - missing fields": { - input: []byte(`{"foo":"AWSSECRETS:///bar/foo","bar":"quz"}`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{}, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(tt.generator(t)) - output := &testNestedStruct{} - err := c.RetrieveUnmarshalledFromJson(tt.input, output) - MarhsalledHelper(t, err, output, &tt.expect) - }) - } -} - -func TestFindTokens(t *testing.T) { - ttests := map[string]struct { - input string - expect []string - }{ - "extract from text correctly": { - `Where does it come from? - Contrary to popular belief, - Lorem Ipsum is AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj <= in middle of sentencenot simply random text. - It has roots in a piece of classical Latin literature from 45 - BC, making it over 2000 years old. Richard McClintock, a Latin professor at - Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, c - onsectetur, from a Lorem Ipsum passage , at the end of line => AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - and going through the cites of the word in c - lassical literature, discovered the undoubtable source. Lorem Ipsum comes from secti - ons in singles =>'AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj'1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) - in doubles => "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj" - by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular - during the :=> embedded in text RenaissanceAWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[] embedded in text <=: - The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.`, - []string{ - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[]"}, - }, - "unknown implementation not picked up": { - `foo: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - bar: AWSPARAMSTR#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123] - unknown: GCPPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, - []string{ - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123]"}, - }, - "all implementations": { - `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - secretsmgr: AWSSECRETS#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123] - gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[] - som othere strufsd - azkv: AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, - []string{ - "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSSECRETS#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123]", - "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[]"}, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - got := configmanager.FindTokens(tt.input) - sort.Strings(got) - sort.Strings(tt.expect) - - if !reflect.DeepEqual(got, tt.expect) { - t.Errorf("input=(%q)\n\ngot: %v\n\nwant: %v", tt.input, got, tt.expect) - } - }) - } -} - -func Test_YamlRetrieveMarshalled_errored_in_generator(t *testing.T) { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return nil, fmt.Errorf("failed to generate a parsedMap") - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - input := &testNestedStruct{} - err := c.RetrieveMarshalledYaml(input) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - -func Test_YamlRetrieveMarshalled_errored_in_marshal(t *testing.T) { - t.Skip() - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return generator.ParsedMap{}, nil - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - err := c.RetrieveMarshalledYaml(&struct { - A int - B map[string]int `yaml:",inline"` - }{1, map[string]int{"a": 2}}) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - -func Test_JsonRetrieveMarshalled_errored_in_generator(t *testing.T) { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return nil, fmt.Errorf("failed to generate a parsedMap") - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - input := &testNestedStruct{} - err := c.RetrieveMarshalledJson(input) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - -func Test_JsonRetrieveMarshalled_errored_in_marshal(t *testing.T) { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return generator.ParsedMap{}, nil - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - // input := &testNestedStruct{} - err := c.RetrieveMarshalledJson(nil) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - // config tests func Test_Generator_Config_(t *testing.T) { ttests := map[string]struct { diff --git a/docs/examples.md b/docs/examples.md index 73313c8..07634fa 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -84,8 +84,8 @@ import ( "context" "fmt" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" - "github.com/DevLabFoundry/configmanager/v2" + "github.com/DevLabFoundry/configmanager/v3/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3" ) func main() { @@ -128,8 +128,8 @@ import ( "log" "os" - "github.com/DevLabFoundry/configmanager/v2" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/pkg/generator" ) var ( diff --git a/docs/v3-updates-migrations.md b/docs/v3-updates-migrations.md new file mode 100644 index 0000000..3f05690 --- /dev/null +++ b/docs/v3-updates-migrations.md @@ -0,0 +1,49 @@ +# V3 Changes and updates + +As part of the V3 update we are aiming to streamline and improve the following high level areas. + +- Network call optimisation +- Backing store plugin architecture + +## Network Call optimisation + +There are many cases when an input string/file or array of `--token` can point to the same underlying token. + +e.g. +```yaml +db_user: AWSSECRETS:///app1/db|user +db_password: AWSSECRETS:///app1/db|password +db_port: AWSSECRETS:///app1/db|port +db_host: AWSSECRETS:///app1/db|host +``` + +Given the above input passed into the CLI i.e. `configmanager fromstr -i above-config.yaml` + +This would result in 4 network calls to the underlying service, in this case the AWS Secrets Manager. + +The V3 update would fan in these 4 tokens into a single network call and then fan out back to a full map with the individual values for each of the look up keys. + +> NB: any token using a metadata annotation on any token would guarantee a unique call to the underlying service + +e.g.: + +```yaml +db_user: AWSSECRETS:///app1/db|user +db_password: AWSSECRETS:///app1/db|password +db_port: AWSSECRETS:///app1/db|port +db_host: AWSSECRETS:///app1/db|host +db_host_2: AWSSECRETS:///app1/db|host[version=2] +``` + +Even though `AWSSECRETS:///app1/db|host[version=2]` and `AWSSECRETS:///app1/db|host` are technically the same AWS Secrets Manager item, specifying the version requires two separate network calls. + +## Backing Store plugin architecture + +The current implementation of the backing stores is defined entirely within the configmanager source code which becomes part of the final staticly linked binary. In order to avoid the bigger size binary and **more importantly** avoid security alerts for libraries that are nothing to do with a backing store provider which is not used! + +Most probably, and most commonly, one would only use single or a combination of providers within the same Cloud for example. + +### Plugin Architecture + +There are a few options to choose from - terraform style provider plugins using gRPC. + diff --git a/eirctl.yaml b/eirctl.yaml index fd5349c..f9e4f9c 100644 --- a/eirctl.yaml +++ b/eirctl.yaml @@ -12,17 +12,20 @@ contexts: name: mirror.gcr.io/bash:5.0.18-alpine3.22 pipelines: - gha:unit:test: + unit:test: - pipeline: test:unit env: ROOT_PKG_NAME: github.com/DevLabFoundry + + gha:unit:test: + - pipeline: unit:test - task: sonar:coverage:prep - depends_on: test:unit + depends_on: unit:test show_coverage: - - pipeline: test:unit + - pipeline: unit:test - task: show:coverage - depends_on: test:unit + depends_on: unit:test build:bin: - task: clean @@ -53,7 +56,7 @@ tasks: - | mkdir -p .deps unset GOTOOLCHAIN - ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${VERSION}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${REVISION}\" -extldflags -static" + ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Version=${VERSION}\" -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Revision=${REVISION}\" -extldflags -static" GOPATH=/eirctl/.deps GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} CGO_ENABLED=0 go build -mod=readonly -buildvcs=false -ldflags="$ldflags" \ -o ./dist/configmanager-${BUILD_GOOS}-${BUILD_GOARCH}${BUILD_SUFFIX} ./cmd echo "---" diff --git a/examples/examples.go b/examples/examples.go index b15016a..379a84d 100644 --- a/examples/examples.go +++ b/examples/examples.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "github.com/DevLabFoundry/configmanager/v2" + "github.com/DevLabFoundry/configmanager/v3" ) const DO_STUFF_WITH_VALS_HERE = "connstring:user@%v:host=%s/someschema..." @@ -46,7 +46,7 @@ spec: secret_val: AWSSECRETS#/customfoo/secret-val owner: test_10016@example.com ` - pm, err := cm.RetrieveWithInputReplaced(exampleK8sCrdMarshalled) + pm, err := cm.RetrieveReplacedString(exampleK8sCrdMarshalled) if err != nil { panic(err) @@ -69,7 +69,7 @@ func SpecConfigTokenReplace[T any](inputType T) (*T, error) { // use custom token separator cm.Config.WithTokenSeparator("://") - replaced, err := cm.RetrieveWithInputReplaced(string(rawBytes)) + replaced, err := cm.RetrieveReplacedBytes(rawBytes) if err != nil { return nil, err } @@ -79,57 +79,29 @@ func SpecConfigTokenReplace[T any](inputType T) (*T, error) { return outType, nil } -// Example -func exampleRetrieveYamlUnmarshalled() { - - type config struct { - DbHost string `yaml:"dbhost"` - Username string `yaml:"user"` - Password string `yaml:"pass"` - } - configMarshalled := ` -user: AWSPARAMSTR:///int-test/pocketbase/config|user -pass: AWSPARAMSTR:///int-test/pocketbase/config|pwd -dbhost: AWSPARAMSTR:///int-test/pocketbase/config|host -` - - appConf := &config{} - cm := configmanager.New(context.TODO()) - // use custom token separator inline with future releases - cm.Config.WithTokenSeparator("://") - err := cm.RetrieveUnmarshalledFromYaml([]byte(configMarshalled), appConf) - if err != nil { - panic(err) - } - fmt.Println(appConf.DbHost) - fmt.Println(appConf.Username) - fmt.Println(appConf.Password) -} - -// ### exampleRetrieveYamlMarshalled -func exampleRetrieveYamlMarshalled() { - type config struct { - DbHost string `yaml:"dbhost"` - Username string `yaml:"user"` - Password string `yaml:"pass"` - } - - appConf := &config{ - DbHost: "AWSPARAMSTR:///int-test/pocketbase/config|host", - Username: "AWSPARAMSTR:///int-test/pocketbase/config|user", - Password: "AWSPARAMSTR:///int-test/pocketbase/config|pwd", - } - - cm := configmanager.New(context.TODO()) - cm.Config.WithTokenSeparator("://") - err := cm.RetrieveMarshalledYaml(appConf) - if err != nil { - panic(err) - } - if appConf.DbHost == "AWSPARAMSTR:///int-test/pocketbase/config|host" { - panic(fmt.Errorf("value of DbHost should have been replaced with a value from token")) - } - fmt.Println(appConf.DbHost) - fmt.Println(appConf.Username) - fmt.Println(appConf.Password) -} +// // Example +// func exampleRetrieveYamlUnmarshalled() { + +// type config struct { +// DbHost string `yaml:"dbhost"` +// Username string `yaml:"user"` +// Password string `yaml:"pass"` +// } +// configMarshalled := ` +// user: AWSPARAMSTR:///int-test/pocketbase/config|user +// pass: AWSPARAMSTR:///int-test/pocketbase/config|pwd +// dbhost: AWSPARAMSTR:///int-test/pocketbase/config|host +// ` + +// appConf := &config{} +// cm := configmanager.New(context.TODO()) +// // use custom token separator inline with future releases +// cm.Config.WithTokenSeparator("://") +// err := cm.RetrieveUnmarshalledFromYaml([]byte(configMarshalled), appConf) +// if err != nil { +// panic(err) +// } +// fmt.Println(appConf.DbHost) +// fmt.Println(appConf.Username) +// fmt.Println(appConf.Password) +// } diff --git a/generator/generator.go b/generator/generator.go new file mode 100644 index 0000000..237c6a6 --- /dev/null +++ b/generator/generator.go @@ -0,0 +1,258 @@ +package generator + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/parser" + "github.com/DevLabFoundry/configmanager/v3/internal/strategy" +) + +// GenVars is the main struct holding the +// strategy patterns iface +// any initialised config if overridded with withers +// as well as the final outString and the initial rawMap +// which wil be passed in a loop into a goroutine to perform the +// relevant strategy network calls to the config store implementations +type GenVars struct { + Logger log.ILogger + strategy strategy.StrategyFuncMap + ctx context.Context + config config.GenVarsConfig +} + +type Opts func(*GenVars) + +// NewGenerator returns a new instance of Generator +// with a default strategy pattern wil be overwritten +// during the first run of a found tokens map +func NewGenerator(ctx context.Context, opts ...Opts) *GenVars { + // defaultStrategy := NewDefatultStrategy() + return newGenVars(ctx, opts...) +} + +func newGenVars(ctx context.Context, opts ...Opts) *GenVars { + conf := config.NewConfig() + g := &GenVars{ + Logger: log.New(io.Discard), + ctx: ctx, + // return using default config + config: *conf, + } + g.strategy = nil + + // now apply additional opts + for _, o := range opts { + o(g) + } + + return g +} + +// WithStrategyMap +// +// Adds addtional funcs for storageRetrieval used for testing only +func (c *GenVars) WithStrategyMap(sm strategy.StrategyFuncMap) *GenVars { + c.strategy = sm + return c +} + +// WithConfig uses custom config +func (c *GenVars) WithConfig(cfg *config.GenVarsConfig) *GenVars { + // backwards compatibility + if cfg != nil { + c.config = *cfg + } + return c +} + +// WithContext uses caller passed context +func (c *GenVars) WithContext(ctx context.Context) *GenVars { + c.ctx = ctx + return c +} + +// Config gets Config on the GenVars +func (c *GenVars) Config() *config.GenVarsConfig { + return &c.config +} + +// Generate generates a k/v map of the tokens with their corresponding secret/paramstore values +// the standard pattern of a token should follow a path like string +// +// Called only from a slice of tokens +func (c *GenVars) Generate(tokens []string) (ReplacedToken, error) { + + ntm, err := c.DiscoverTokens(strings.Join(tokens, "\n")) + if err != nil { + return nil, err + } + + // pass in default initialised retrieveStrategy + // input should be + rt, err := c.generate(ntm) + if err != nil { + return nil, err + } + return rt, nil +} + +var ErrTokenDiscovery = errors.New("failed to discover tokens") + +// DiscoverToken generates a k/v map of the tokens with their corresponding secret/paramstore values +// the standard pattern of a token should follow a path like string +// +// Called only from a slice of tokens +func (c *GenVars) DiscoverTokens(text string) (NormalizedTokenSafe, error) { + + rtm := NewRawTokenConfig() + + lexerSource := lexer.Source{FileName: text[0:min(len(text), 20)], FullPath: "", Input: text} + l := lexer.New(lexerSource, c.config) + p := parser.New(l, &c.config).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + return NormalizedTokenSafe{}, fmt.Errorf("%w in input (%s) with errors: %q", ErrTokenDiscovery, text[0:min(len(text), 25)], errs) + } + for _, prsdToken := range parsed { + rtm.AddToken(prsdToken.ParsedToken.String(), &prsdToken.ParsedToken) + } + return c.NormalizeRawToken(rtm), nil +} + +// IsParsed will try to parse the return found string into +// map[string]string +// If found it will convert that to a map with all keys uppercased +// and any characters +func IsParsed(v any, trm ReplacedToken) bool { + str := fmt.Sprint(v) + err := json.Unmarshal([]byte(str), &trm) + return err == nil +} + +// generate initiates waitGroup to handle 1 or more normalized network calls concurrently to the underlying stores +// +// Captures the response/error in TokenResponse struct +// It then denormalizes the NormalizedTokenSafe back to a ReplacedToken map +// which stores the values for each token to be returned to the caller +func (c *GenVars) generate(ntm NormalizedTokenSafe) (ReplacedToken, error) { + if len(ntm.normalizedTokenMap) < 1 { + c.Logger.Debug("no replaceable tokens found in input") + return nil, nil + } + + wg := &sync.WaitGroup{} + + s := strategy.New(c.config, c.Logger, strategy.WithStrategyFuncMap(c.strategy)) + + // safe read of normalized token map + // this will ensure that we are minimizing + // the number of network calls to each underlying store + for _, prsdTkn := range ntm.GetMap() { + if len(prsdTkn.parsedTokens) == 0 { + // TODO: err type this + return nil, fmt.Errorf("no tokens assigned to parsedTokens slice") + } + token := prsdTkn.parsedTokens[0] + wg.Go(func() { + prsdTkn.resp = &strategy.TokenResponse{} + storeStrategy, err := s.GetImplementation(c.ctx, token) + if err != nil { + prsdTkn.resp.Err = err + return + } + prsdTkn.resp = strategy.ExchangeToken(storeStrategy, token) + }) + } + + wg.Wait() + + // now we fan out the normalized value to ReplacedToken map + // this will ensure all found tokens will have a value assigned to them + replacedToken := make(ReplacedToken) + for _, r := range ntm.GetMap() { + if r == nil { + // defensive as this shouldn't happen + continue + } + if r.resp.Err != nil { + c.Logger.Debug("cr.err %v, for token: %s", r.resp.Err, r.resp.Key().String()) + continue + } + for _, originalToken := range r.parsedTokens { + replacedToken[originalToken.String()] = keySeparatorLookup(originalToken, r.resp.Value()) + } + } + return replacedToken, nil +} + +// NormalizedToken represents the struct after all the possible tokens +// were merged into the lowest commmon denominator. +// The idea is to minimize the number of networks calls to the underlying `store` Implementations +// +// The merging is based on the implemenentation and sanitized token being the same, +// if the token contains metadata then it must be +// +// # Merging strategy +// +// Same Prefix + Same SanitisedToken && No Metadata +type NormalizedToken struct { + // all the tokens that can be used to do a replacement + parsedTokens []*config.ParsedTokenConfig + // will be assigned post generate + resp *strategy.TokenResponse + // // configToken is the last assigned full config in the loop if multip + // configToken *config.ParsedTokenConfig +} + +func (n *NormalizedToken) WithParsedToken(v *config.ParsedTokenConfig) *NormalizedToken { + n.parsedTokens = append(n.parsedTokens, v) + return n +} + +// NormalizedTokenSafe is the map of lowest common denominators +// by token.Keypathless or token.String (full token) if metadata is included +type NormalizedTokenSafe struct { + mu *sync.Mutex + normalizedTokenMap map[string]*NormalizedToken +} + +func (n NormalizedTokenSafe) GetMap() map[string]*NormalizedToken { + n.mu.Lock() + defer n.mu.Unlock() + return n.normalizedTokenMap +} + +func (c *GenVars) NormalizeRawToken(rtm *RawTokenConfig) NormalizedTokenSafe { + ntm := NormalizedTokenSafe{mu: &sync.Mutex{}, normalizedTokenMap: make(map[string]*NormalizedToken)} + + for _, r := range rtm.RawTokenMap() { + // if a string contains we need to store it uniquely + // future improvements might group all the metadata values together + if len(r.Metadata()) > 0 { + if n, found := ntm.normalizedTokenMap[r.String()]; found { + n.WithParsedToken(r) + continue + } + ntm.normalizedTokenMap[r.String()] = (&NormalizedToken{}).WithParsedToken(r) + continue + } + + if n, found := ntm.normalizedTokenMap[r.Keypathless()]; found { + n.WithParsedToken(r) + continue + } + ntm.normalizedTokenMap[r.Keypathless()] = (&NormalizedToken{}).WithParsedToken(r) + continue + } + return ntm +} diff --git a/generator/generator_test.go b/generator/generator_test.go new file mode 100644 index 0000000..e48e546 --- /dev/null +++ b/generator/generator_test.go @@ -0,0 +1,370 @@ +package generator_test + +import ( + "bytes" + "context" + "fmt" + "slices" + "testing" + + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/strategy" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" +) + +type mockGenerate struct { + inToken, value string + err error +} + +func (m *mockGenerate) SetToken(s *config.ParsedTokenConfig) { +} +func (m *mockGenerate) Value() (s string, e error) { + return m.value, m.err +} + +func TestGenerate(t *testing.T) { + + t.Run("succeeds with funcMap", func(t *testing.T) { + var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"AWSPARAMSTR://mountPath/token", "bar", nil} + return m, nil + } + + g := generator.NewGenerator(context.TODO(), func(gv *generator.GenVars) { + gv.Logger = log.New(&bytes.Buffer{}) + }) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: custFunc}) + got, err := g.Generate([]string{"AWSPARAMSTR://mountPath/token"}) + + if err != nil { + t.Fatal("errored on generate") + } + if len(got) != 1 { + t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 1) + } + }) + + t.Run("errors in retrieval and logs it out", func(t *testing.T) { + var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"AWSPARAMSTR://mountPath/token", "bar", fmt.Errorf("failed to get value")} + return m, nil + } + + g := generator.NewGenerator(context.TODO()) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: custFunc}) + got, err := g.Generate([]string{"AWSPARAMSTR://mountPath/token"}) + + if err != nil { + t.Fatal("errored on generate") + } + if len(got) != 0 { + t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) + } + }) + + t.Run("retrieves values correctly from a keylookup inside", func(t *testing.T) { + var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"token-unused", `{"foo":"bar","key1":{"key2":"val"}}`, nil} + return m, nil + } + + g := generator.NewGenerator(context.TODO()) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: custFunc}) + got, err := g.Generate([]string{"AWSPARAMSTR://mountPath/token|key1.key2"}) + + if err != nil { + t.Fatal("errored on generate") + } + if len(got) != 1 { + t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) + } + if got["AWSPARAMSTR://mountPath/token|key1.key2"] != "val" { + t.Errorf(testutils.TestPhraseWithContext, "incorrect value returned in parsedMap", got["AWSPARAMSTR://mountPath/token|key1.key2"], "val") + } + }) +} + +func TestGenerate_withKeys_lookup(t *testing.T) { + ttests := map[string]struct { + custFunc strategy.StrategyFunc + token string + expectVal string + }{ + "retrieves string value correctly from a keylookup inside": { + custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":"val"}}`, nil} + return m, nil + }, + token: "AWSPARAMSTR://mountPath/token|key1.key2", + expectVal: "val", + }, + "retrieves number value correctly from a keylookup inside": { + custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":123}}`, nil} + return m, nil + }, + token: "AWSPARAMSTR://mountPath/token|key1.key2", + expectVal: "123", + }, + "retrieves nothing as keylookup is incorrect": { + custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":123}}`, nil} + return m, nil + }, + token: "AWSPARAMSTR://mountPath/token|noprop", + expectVal: "", + }, + "retrieves value as is due to incorrectly stored json in backing store": { + custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"token", `foo":"bar","key1":{"key2":123}}`, nil} + return m, nil + }, + token: "AWSPARAMSTR://mountPath/token|noprop", + expectVal: `foo":"bar","key1":{"key2":123}}`, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + g := generator.NewGenerator(context.TODO()) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: tt.custFunc}) + got, err := g.Generate([]string{tt.token}) + + if err != nil { + t.Fatal("errored on generate") + } + if len(got) != 1 { + t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) + } + if got[tt.token] != tt.expectVal { + t.Errorf(testutils.TestPhraseWithContext, "incorrect value returned in parsedMap", got[tt.token], tt.expectVal) + } + }) + } +} + +func Test_IsParsed(t *testing.T) { + ttests := map[string]struct { + val any + isParsed bool + }{ + "not parseable": { + `notparseable`, false, + }, + "one level parseable": { + `{"parseable":"foo"}`, true, + }, + "incorrect JSON": { + `parseable":"foo"}`, false, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + typ := generator.ReplacedToken{} + got := generator.IsParsed(tt.val, typ) + if got != tt.isParsed { + t.Errorf(testutils.TestPhraseWithContext, "unexpected IsParsed", got, tt.isParsed) + } + }) + } +} + +func TestGenVars_NormalizeRawToken(t *testing.T) { + + t.Run("multiple tokens", func(t *testing.T) { + g := generator.NewGenerator(context.TODO()) + + input := `GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|a + GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|b + GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|c + AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key1 + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key2 + AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj` + want := []string{"GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj"} + got, err := g.DiscoverTokens(input) + if err != nil { + t.Fatal(err) + } + if len(got.GetMap()) != len(want) { + t.Errorf("got %v wanted %d", len(got.GetMap()), len(want)) + } + for key := range got.GetMap() { + if !slices.Contains(want, key) { + t.Errorf("got %s, wanted to be included in %v", key, want) + } + } + }) +} + +func Test_ConfigManager_DiscoverTokens(t *testing.T) { + ttests := map[string]struct { + input string + separator string + expect []string + }{ + "multiple tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR:///path/config|foo.user:AWSPARAMSTR:///path/config|password@AWSPARAMSTR:///path/config|foo.endpoint:AWSPARAMSTR:///path/config|foo.port/?someQ=AWSPARAMSTR:///path/queryparam|p1[version=123]&anotherQ=false`, + "://", + []string{ + "AWSPARAMSTR:///path/config", + // "AWSPARAMSTR:///path/config|password", + // "AWSPARAMSTR:///path/config|foo.endpoint", + // "AWSPARAMSTR:///path/config|foo.port", + "AWSPARAMSTR:///path/queryparam|p1[version=123]"}, + }, + "# tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR#/path/config|foo.user:AWSPARAMSTR#/path/config|password@AWSPARAMSTR#/path/config|foo.endpoint:AWSPARAMSTR#/path/config|foo.port/?someQ=AWSPARAMSTR#/path/queryparam|p1[version=123]&anotherQ=false`, + "#", + []string{ + "AWSPARAMSTR#/path/config", + // "AWSPARAMSTR#/path/config|password", + // "AWSPARAMSTR#/path/config|foo.endpoint", + // "AWSPARAMSTR#/path/config|foo.port", + "AWSPARAMSTR#/path/queryparam|p1[version=123]"}, + }, + "without leading slash and path like name # tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR#path_config|foo.user:AWSPARAMSTR#path_config|password@AWSPARAMSTR#path_config|foo.endpoint:AWSPARAMSTR#path_config|foo.port/?someQ=AWSPARAMSTR#path_queryparam|p1[version=123]&anotherQ=false`, + "#", + []string{ + "AWSPARAMSTR#path_config", + // "AWSPARAMSTR#path_config|password", + // "AWSPARAMSTR#path_config|foo.endpoint", + // "AWSPARAMSTR#path_config|foo.port", + "AWSPARAMSTR#path_queryparam|p1[version=123]"}, + }, + // Ensures all previous test cases pass as well + "extract from text correctly": { + `Where does it come from? + Contrary to popular belief, + Lorem Ipsum is AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1 <= in middle of sentencenot simply random text. + It has roots in a piece of classical Latin literature from 45 + BC, making it over 2000 years old. Richard McClintock, a Latin professor at + Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, c + onsectetur, from a Lorem Ipsum passage , at the end of line => AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4 + and going through the cites of the word in c + lassical literature, discovered the undoubtable source. Lorem Ipsum comes from secti + ons in singles =>'AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2'1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) + in doubles => "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3" + by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular + during the :=> embedded in text RenaissanceAWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5 embedded in text <=: + The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.`, + "://", + []string{ + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5", + }, + }, + "unknown implementation not picked up": { + `foo: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + bar: AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + unknown: GCPPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + unknown: GCPSECRETS#/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, + "://", + []string{ + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]"}, + }, + "all implementations": { + `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + secretsmgr: AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + som othere strufsd + azkv: AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, + "://", + []string{ + "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]", + "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj"}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + config.VarPrefix = map[config.ImplementationPrefix]bool{"AWSPARAMSTR": true} + g := generator.NewGenerator(context.TODO()) + g.Config().WithTokenSeparator(tt.separator) + gdt, err := g.DiscoverTokens(tt.input) + if err != nil { + t.Fatal(err) + } + got := gdt.GetMap() + + if len(got) != len(tt.expect) { + t.Errorf("wrong nmber of tokens resolved\ngot (%d) want (%d)", len(got), len(tt.expect)) + } + // for _, v := range got { + // if !slices.Contains(tt.expect, v.String()) { + // t.Errorf("got (%s) not found in expected slice (%v)", v, tt.expect) + // } + // } + }) + } +} + +func Test_Generate_EnsureRaceFree(t *testing.T) { + g := generator.NewGenerator(context.TODO()) + + input := ` +fg +dfg gdfgfdGCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj +GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|a +GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|b +GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|c +ddsffds AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + 'AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]' + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key1 + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key2 + AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj gdf gdfgdf + dfg gdf gdf gdf + fdg dgf dgf + VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj . dfg dfgdf dfg fddf` + + g.WithStrategyMap(strategy.StrategyFuncMap{ + config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"a":"bar","b":{"key2":"val"},"c":123}`, nil} + return m, nil + }, + config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"a":"bar","b":{"key2":"val"},"c":123}`, nil} + return m, nil + }, + config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"key1":"bar","key2":"val","c":123}`, nil} + return m, nil + }, + config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"key1":"bar","key2":"val","c":123}`, nil} + return m, nil + }, + config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"key1":"bar","key2":"val","c":123}`, nil} + return m, nil + }, + }) + + got, err := g.Generate([]string{input}) + if err != nil { + t.Fatal(err) + } + if len(got) != 10 { + t.Errorf("got %v wanted %d", len(got), 10) + } + +} diff --git a/generator/generatorvars.go b/generator/generatorvars.go new file mode 100644 index 0000000..79a56ae --- /dev/null +++ b/generator/generatorvars.go @@ -0,0 +1,94 @@ +package generator + +import ( + "fmt" + "strconv" + "sync" + + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/spyzhov/ajson" +) + +// ReplacedToken is the internal working object definition and +// the return type if results are not flushed to file +type ReplacedToken map[string]any + +func (pm ReplacedToken) MapKeys() (keys []string) { + for k := range pm { + keys = append(keys, k) + } + return +} + +// RawTokenConfig represents the map of +// discovered tokens via the lexer/parser +type RawTokenConfig struct { + mu *sync.Mutex + tokenMap map[string]*config.ParsedTokenConfig +} + +func NewRawTokenConfig() *RawTokenConfig { + return &RawTokenConfig{mu: &sync.Mutex{}, tokenMap: map[string]*config.ParsedTokenConfig{}} +} + +func (rtm *RawTokenConfig) AddToken(name string, parsedToken *config.ParsedTokenConfig) { + rtm.mu.Lock() + defer rtm.mu.Unlock() + rtm.tokenMap[name] = parsedToken +} + +func (rtm *RawTokenConfig) RawTokenMap() map[string]*config.ParsedTokenConfig { + rtm.mu.Lock() + defer rtm.mu.Unlock() + return rtm.tokenMap +} + +// type tokenMapSafe struct { +// mu *sync.Mutex +// tokenMap ReplacedToken +// } + +// func (tms *tokenMapSafe) getTokenMap() ReplacedToken { +// tms.mu.Lock() +// defer tms.mu.Unlock() +// return tms.tokenMap +// } + +// func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { +// tms.mu.Lock() +// defer tms.mu.Unlock() +// // NOTE: still use the metadata in the key +// // there could be different versions / labels for the same token and hence different values +// // However the JSONpath look up +// tms.tokenMap[key.String()] = keySeparatorLookup(key, val) +// } + +// keySeparatorLookup checks if the key contains +// keySeparator character +// If it does contain one then it tries to parse +func keySeparatorLookup(token *config.ParsedTokenConfig, val string) string { + k := token.LookupKeys() + if k == "" { + return val + } + + keys, err := ajson.JSONPath([]byte(val), fmt.Sprintf("$..%s", k)) + if err != nil { + return val + } + + if len(keys) == 1 { + v := keys[0] + if v.Type() == ajson.String { + str, err := strconv.Unquote(fmt.Sprintf("%v", v)) + if err != nil { + return fmt.Sprintf("%v", v) + } + return str + } + + return fmt.Sprintf("%v", v) + } + + return "" +} diff --git a/go.mod b/go.mod index 422785f..e0c3290 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ -module github.com/DevLabFoundry/configmanager/v2 +module github.com/DevLabFoundry/configmanager/v3 -go 1.25.1 +go 1.25.3 require ( cloud.google.com/go/secretmanager v1.15.1 diff --git a/internal/cmdutils/cmdutils.go b/internal/cmdutils/cmdutils.go index f1f71dc..b86b292 100644 --- a/internal/cmdutils/cmdutils.go +++ b/internal/cmdutils/cmdutils.go @@ -11,15 +11,16 @@ import ( "os" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/spf13/cobra" ) type configManagerIface interface { - RetrieveWithInputReplaced(input string) (string, error) - Retrieve(tokens []string) (generator.ParsedMap, error) + RetrieveReplacedBytes(input []byte) ([]byte, error) + RetrieveReplacedString(input string) (string, error) + Retrieve(tokens []string) (generator.ReplacedToken, error) GeneratorConfig() *config.GenVarsConfig } @@ -111,13 +112,13 @@ func (c *CmdUtils) generateStrOutFromInput(input io.Reader, writer io.Writer) er return err } - str, err := c.configManager.RetrieveWithInputReplaced(string(b)) + replacedBytes, err := c.configManager.RetrieveReplacedBytes(b) if err != nil { return err } pp := &PostProcessor{} - return pp.StrToFile(writer, str) + return pp.StrToFile(writer, string(replacedBytes)) } type WriterCloserWrapper struct { diff --git a/internal/cmdutils/cmdutils_test.go b/internal/cmdutils/cmdutils_test.go index f8fad0f..7ec1daf 100644 --- a/internal/cmdutils/cmdutils_test.go +++ b/internal/cmdutils/cmdutils_test.go @@ -8,16 +8,16 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - log "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + log "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/spf13/cobra" ) type mockCfgMgr struct { - parsedMap generator.ParsedMap + parsedMap generator.ReplacedToken err error parsedString string config *config.GenVarsConfig @@ -27,7 +27,13 @@ func (m mockCfgMgr) RetrieveWithInputReplaced(input string) (string, error) { return m.parsedString, m.err } -func (m mockCfgMgr) Retrieve(tokens []string) (generator.ParsedMap, error) { +func (m mockCfgMgr) RetrieveReplacedBytes(input []byte) ([]byte, error) { + return []byte(m.parsedString), m.err +} +func (m mockCfgMgr) RetrieveReplacedString(input string) (string, error) { + return m.parsedString, m.err +} +func (m mockCfgMgr) Retrieve(tokens []string) (generator.ReplacedToken, error) { return m.parsedMap, m.err } @@ -61,15 +67,13 @@ func cmdTestHelper(t *testing.T, err error, got []byte, expect []string) { } func Test_GenerateFromCmd(t *testing.T) { - t.Parallel() - ttests := map[string]struct { - mockMap generator.ParsedMap + mockMap generator.ReplacedToken tokens []string expect []string }{ "succeeds with 3 tokens": { - generator.ParsedMap{"FOO://bar/qusx": "aksujg", "FOO://bar/lorem": "", "FOO://bar/ducks": "sdhbjk0293"}, + generator.ReplacedToken{"FOO://bar/qusx": "aksujg", "FOO://bar/lorem": "", "FOO://bar/ducks": "sdhbjk0293"}, []string{"FOO://bar/qusx", "FOO://bar/lorem", "FOO://bar/ducks"}, []string{"export QUSX='aksujg'", "export LOREM=''", "export DUCKS='sdhbjk0293'"}, }, @@ -106,8 +110,6 @@ func (m *mockWriter) Write(in []byte) (int, error) { } func Test_GenerateStrOut(t *testing.T) { - t.Parallel() - inputStr := `FOO://bar/qusx FOO://bar/lorem FOO://bar/ducks` mockParsedStr := `aksujg fooLorem Mighty` expect := []string{"aksujg", "fooLorem", "Mighty"} @@ -210,7 +212,7 @@ func Test_CmdUtils_Errors_on(t *testing.T) { t.Run("REtrieve from tokens in fetching ANY of the tokens", func(t *testing.T) { m := &mockCfgMgr{ config: config.NewConfig(), - parsedMap: generator.ParsedMap{}, + parsedMap: generator.ReplacedToken{}, err: fmt.Errorf("err in fetching tokens"), } @@ -225,7 +227,7 @@ func Test_CmdUtils_Errors_on(t *testing.T) { t.Run("REtrieve from tokens in fetching SOME of the tokens", func(t *testing.T) { m := &mockCfgMgr{ config: config.NewConfig(), - parsedMap: generator.ParsedMap{"IMNP://foo": "bar"}, + parsedMap: generator.ReplacedToken{"IMNP://foo": "bar"}, err: fmt.Errorf("err in fetching tokens"), } @@ -239,7 +241,7 @@ func Test_CmdUtils_Errors_on(t *testing.T) { t.Run("REtrieve from string in fetching SOME of the tokens", func(t *testing.T) { m := &mockCfgMgr{ config: config.NewConfig().WithOutputPath("stdout"), - parsedMap: generator.ParsedMap{"IMNP://foo": "bar"}, + parsedMap: generator.ReplacedToken{"IMNP://foo": "bar"}, parsedString: `bar `, err: fmt.Errorf("err in fetching tokens"), } diff --git a/internal/cmdutils/postprocessor.go b/internal/cmdutils/postprocessor.go index 8419cc7..3b4a33b 100644 --- a/internal/cmdutils/postprocessor.go +++ b/internal/cmdutils/postprocessor.go @@ -5,15 +5,15 @@ import ( "io" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" ) // PostProcessor // processes the rawMap and outputs the result // depending on cmdline options type PostProcessor struct { - ProcessedMap generator.ParsedMap + ProcessedMap generator.ReplacedToken Config *config.GenVarsConfig outString []string } @@ -24,7 +24,7 @@ func (p *PostProcessor) ConvertToExportVar() []string { for k, v := range p.ProcessedMap { rawKeyToken := strings.Split(k, "/") // assumes a path like token was used topLevelKey := rawKeyToken[len(rawKeyToken)-1] - trm := generator.ParsedMap{} + trm := generator.ReplacedToken{} if parsedOk := generator.IsParsed(v, trm); parsedOk { // if is a map // try look up on key if separator defined @@ -32,21 +32,21 @@ func (p *PostProcessor) ConvertToExportVar() []string { p.exportVars(normMap) continue } - p.exportVars(generator.ParsedMap{topLevelKey: v}) + p.exportVars(generator.ReplacedToken{topLevelKey: v}) } return p.outString } // envVarNormalize -func (p *PostProcessor) envVarNormalize(pmap generator.ParsedMap) generator.ParsedMap { - normalizedMap := make(generator.ParsedMap) +func (p *PostProcessor) envVarNormalize(pmap generator.ReplacedToken) generator.ReplacedToken { + normalizedMap := make(generator.ReplacedToken) for k, v := range pmap { normalizedMap[p.normalizeKey(k)] = v } return normalizedMap } -func (p *PostProcessor) exportVars(exportMap generator.ParsedMap) { +func (p *PostProcessor) exportVars(exportMap generator.ReplacedToken) { for k, v := range exportMap { // NOTE: \n line ending is not totally cross platform diff --git a/internal/cmdutils/postprocessor_test.go b/internal/cmdutils/postprocessor_test.go index 001ea68..5c18e23 100644 --- a/internal/cmdutils/postprocessor_test.go +++ b/internal/cmdutils/postprocessor_test.go @@ -5,10 +5,10 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func postprocessorHelper(t *testing.T) { @@ -17,14 +17,14 @@ func postprocessorHelper(t *testing.T) { } func Test_ConvertToExportVars(t *testing.T) { tests := map[string]struct { - rawMap generator.ParsedMap + rawMap generator.ReplacedToken expectStr string expectLength int }{ - "number included": {generator.ParsedMap{"foo": "BAR", "num": 123}, `export FOO='BAR'`, 2}, - "strings only": {generator.ParsedMap{"foo": "BAR", "num": "a123"}, `export FOO='BAR'`, 2}, - "numbers only": {generator.ParsedMap{"foo": 123, "num": 456}, `export FOO=123`, 2}, - "map inside response": {generator.ParsedMap{"map": `{"foo":"bar","baz":"qux"}`, "num": 123}, `export FOO='bar'`, 3}, + "number included": {generator.ReplacedToken{"foo": "BAR", "num": 123}, `export FOO='BAR'`, 2}, + "strings only": {generator.ReplacedToken{"foo": "BAR", "num": "a123"}, `export FOO='BAR'`, 2}, + "numbers only": {generator.ReplacedToken{"foo": 123, "num": 456}, `export FOO=123`, 2}, + "map inside response": {generator.ReplacedToken{"map": `{"foo":"bar","baz":"qux"}`, "num": 123}, `export FOO='bar'`, 3}, } for name, tt := range tests { diff --git a/internal/config/config.go b/internal/config/config.go index 50aecce..d1d05ed 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -49,6 +49,7 @@ var ( GcpSecretsPrefix: true, HashicorpVaultPrefix: true, AzTableStorePrefix: true, AzAppConfigPrefix: true, UnknownPrefix: true, } + ErrConfigValidation = errors.New("config validation failed") ) // GenVarsConfig defines the input config object to be passed @@ -60,7 +61,9 @@ type GenVarsConfig struct { // parseAdditionalVars func(token string) TokenConfigVars } -// NewConfig +// NewConfig returns a new GenVarsConfig with default values +// +// keySeparator should be only a single character func NewConfig() *GenVarsConfig { return &GenVarsConfig{ tokenSeparator: tokenSeparator, @@ -120,49 +123,51 @@ func (c *GenVarsConfig) Config() GenVarsConfig { return cc } -// Parsed token config section +// Config returns the derefed value +func (c *GenVarsConfig) Validate() error { + if len(c.keySeparator) > 1 { + return fmt.Errorf("%w, keyseparator can only be 1 character", ErrConfigValidation) + } + return nil +} +// Parsed token config section var ErrInvalidTokenPrefix = errors.New("token prefix has no implementation") type ParsedTokenConfig struct { - prefix ImplementationPrefix + prefix ImplementationPrefix + // cofig values keySeparator, tokenSeparator string - prefixLessToken, fullToken string - metadataStr, keysPath string - storeToken, metadataLess string + // tokenb parts + metadataStr string + keysPath string + sanitizedToken string } -// NewParsedTokenConfig returns a pointer to a new TokenConfig struct -// returns nil if current prefix does not correspond to an Implementation -// -// The caller needs to make sure it is not nil -// TODO: a custom parser would be best here -func NewParsedTokenConfig(token string, config GenVarsConfig) (*ParsedTokenConfig, error) { - ptc := &ParsedTokenConfig{} - prfx := strings.Split(token, config.TokenSeparator())[0] - - // This should already only be a list of properly supported tokens but just in case - if found := VarPrefix[ImplementationPrefix(prfx)]; !found { - return nil, fmt.Errorf("prefix: %s\n%w", prfx, ErrInvalidTokenPrefix) +// NewToken initialises a *ParsedTokenConfig +func NewToken(prefix ImplementationPrefix, config GenVarsConfig) (*ParsedTokenConfig, error) { + tokenConf := &ParsedTokenConfig{} + if err := config.Validate(); err != nil { + return nil, err } + tokenConf.keySeparator = config.keySeparator + tokenConf.tokenSeparator = config.tokenSeparator + + tokenConf.prefix = prefix + + return tokenConf, nil +} - ptc.keySeparator = config.keySeparator - ptc.tokenSeparator = config.tokenSeparator - ptc.prefix = ImplementationPrefix(prfx) - ptc.fullToken = token - return ptc.new(), nil +func (ptc *ParsedTokenConfig) WithKeyPath(kp string) { + ptc.keysPath = kp } -func (ptc *ParsedTokenConfig) new() *ParsedTokenConfig { - // order must be respected here - // - ptc.prefixLessToken = strings.Replace(ptc.fullToken, fmt.Sprintf("%s%s", ptc.prefix, ptc.tokenSeparator), "", 1) +func (ptc *ParsedTokenConfig) WithMetadata(md string) { + ptc.metadataStr = md +} - // token without metadata and the string itself - ptc.extractMetadataStr() - // token without keys - ptc.keysLookup() - return ptc +func (ptc *ParsedTokenConfig) WithSanitizedToken(v string) { + ptc.sanitizedToken = v } func (t *ParsedTokenConfig) ParseMetadata(metadataTyp any) error { @@ -189,86 +194,52 @@ func (t *ParsedTokenConfig) ParseMetadata(metadataTyp any) error { return nil } -func (t *ParsedTokenConfig) StripPrefix() string { - return t.prefixLessToken -} - -// StripMetadata returns the fullToken without the -// metadata -func (t *ParsedTokenConfig) StripMetadata() string { - return t.metadataLess -} - -// Strip -// -// returns the only the store indicator string -// without any of the configmanager token enrichment: -// -// - metadata -// -// - keySeparator -// -// - keys -// -// - prefix +// StoreToken returns the sanitized token without: +// - metadata +// - keySeparator +// - keys +// - prefix func (t *ParsedTokenConfig) StoreToken() string { - return t.storeToken + return t.sanitizedToken } // Full returns the full Token path. // Including key separator and metadata values func (t *ParsedTokenConfig) String() string { - return t.fullToken + token := t.Metadaless() + if len(t.metadataStr) > 0 { + token += fmt.Sprintf("[%s]", t.metadataStr) + } + return token } -func (t *ParsedTokenConfig) LookupKeys() string { - return t.keysPath +// Keypathless returns the token without the key and metadata attributes +// Token will include the ImplementationPrefix + token separator + path to item +func (t *ParsedTokenConfig) Keypathless() string { + token := fmt.Sprintf("%s%s%s", t.prefix, t.tokenSeparator, t.sanitizedToken) + return token } -func (t *ParsedTokenConfig) Prefix() ImplementationPrefix { - return t.prefix +func (t *ParsedTokenConfig) Metadaless() string { + token := fmt.Sprintf("%s%s%s", t.prefix, t.tokenSeparator, t.sanitizedToken) + if len(t.keysPath) > 0 { + token += t.keySeparator + t.keysPath + } + return token } -const ( - startMetaStr string = `[` - endMetaStr string = `]` -) +func (t *ParsedTokenConfig) LookupKeys() string { + return t.keysPath +} -// extractMetadataStr returns anything between the start and end -// metadata markers in the token string itself -// returns the token without meta -func (t *ParsedTokenConfig) extractMetadataStr() { - token := t.prefixLessToken - t.metadataLess = token - startIndex := strings.Index(token, startMetaStr) - // token has no startMetaStr - if startIndex == -1 { - return - } - newS := token[startIndex+len(startMetaStr):] +func (t *ParsedTokenConfig) Metadata() string { + return t.metadataStr +} - endIndex := strings.Index(newS, endMetaStr) - // token has no meta end - if endIndex == -1 { - return - } - // metastring extracted - // complete [key=value] has been found - metaString := newS[:endIndex] - t.metadataStr = metaString - // Set Metadataless token - t.metadataLess = strings.ReplaceAll(token, startMetaStr+metaString+endMetaStr, "") +func (t *ParsedTokenConfig) Prefix() ImplementationPrefix { + return t.prefix } -// keysLookup returns the keysLookup path and the string without it -// -// NOTE: metadata was already stripped at this point -func (t *ParsedTokenConfig) keysLookup() { - keysIndex := strings.Index(t.metadataLess, t.keySeparator) - if keysIndex >= 0 { - t.keysPath = t.metadataLess[keysIndex+len(t.keySeparator):] - t.storeToken = t.metadataLess[:keysIndex] - return - } - t.storeToken = t.metadataLess +func (t *ParsedTokenConfig) TokenSeparator() string { + return t.tokenSeparator } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6bc6803..8fc33f0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,8 +3,8 @@ package config_test import ( "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func Test_SelfName(t *testing.T) { @@ -30,69 +30,77 @@ func Test_MarshalMetadata_with_label_struct_succeeds(t *testing.T) { } ttests := map[string]struct { - config *config.GenVarsConfig - rawToken string + token func() *config.ParsedTokenConfig wantLabel string wantMetaStrippedToken string }{ "when provider expects label on token and label exists": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88[label=dev]`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithMetadata("label=dev") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "dev", "basjh/dskjuds/123", }, "when provider expects label on token and label does not exist": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88[someother=dev]`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithMetadata("someother=dev") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "", "basjh/dskjuds/123", }, "no metadata found": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "", "basjh/dskjuds/123", }, "no metadata found incorrect marker placement": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88]asdas=bar[`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88]asdas=bar[") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "", "basjh/dskjuds/123", }, "no metadata found incorrect marker placement and no key separator": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123]asdas=bar[`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithSanitizedToken("basjh/dskjuds/123]asdas=bar[") + return tkn + }, "", "basjh/dskjuds/123]asdas=bar[", }, - "no end found incorrect marker placement and no key separator": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123[asdas=bar`, - "", - "basjh/dskjuds/123[asdas=bar", - }, "no start found incorrect marker placement and no key separator": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123]asdas=bar]`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithMetadata("someother=dev") + tkn.WithSanitizedToken("basjh/dskjuds/123]asdas=bar]") + return tkn + }, "", "basjh/dskjuds/123]asdas=bar]", }, - "metadata is in the middle of path lookup": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123[label=bar]|lookup`, - "bar", - "basjh/dskjuds/123", - }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { inputTyp := &labelMeta{} - got, err := config.NewParsedTokenConfig(tt.rawToken, *tt.config) - - if err != nil { - t.Fatalf("got an error on NewParsedTokenconfig (%s)\n", tt.rawToken) - } - + got := tt.token() if got == nil { t.Errorf(testutils.TestPhraseWithContext, "Unable to parse token", nil, config.ParsedTokenConfig{}) } @@ -100,7 +108,7 @@ func Test_MarshalMetadata_with_label_struct_succeeds(t *testing.T) { got.ParseMetadata(inputTyp) if got.StoreToken() != tt.wantMetaStrippedToken { - t.Errorf(testutils.TestPhraseWithContext, "Token does not match", got.StripMetadata(), tt.wantMetaStrippedToken) + t.Errorf(testutils.TestPhraseWithContext, "Token does not match", got.StoreToken(), tt.wantMetaStrippedToken) } if inputTyp.Label != tt.wantLabel { @@ -115,23 +123,27 @@ func Test_TokenParser_config(t *testing.T) { Version string `json:"version"` } ttests := map[string]struct { - input string - expPrefix config.ImplementationPrefix - expLookupKeys string - expStoreToken string - expString string // fullToken - expMetadataVersion string + rawToken, keyPath, metadataStr string + expPrefix config.ImplementationPrefix + expLookupKeys string + expStoreToken string // sanitised + expString string // fullToken + expMetadataVersion string }{ - "bare": {"AWSSECRETS://foo/bar", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar", ""}, - "with metadata version": {"AWSSECRETS://foo/bar[version=123]", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar[version=123]", "123"}, - "with keys lookup and label": {"AWSSECRETS://foo/bar|key1.key2[version=123]", config.SecretMgrPrefix, "key1.key2", "foo/bar", "AWSSECRETS://foo/bar|key1.key2[version=123]", "123"}, - "with keys lookup and longer token": {"AWSSECRETS://foo/bar|key1.key2]version=123]", config.SecretMgrPrefix, "key1.key2]version=123]", "foo/bar", "AWSSECRETS://foo/bar|key1.key2]version=123]", ""}, - "with keys lookup but no keys": {"AWSSECRETS://foo/bar/sdf/sddd.90dsfsd|[version=123]", config.SecretMgrPrefix, "", "foo/bar/sdf/sddd.90dsfsd", "AWSSECRETS://foo/bar/sdf/sddd.90dsfsd|[version=123]", "123"}, + "bare": {"foo/bar", "", "", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar", ""}, + "with metadata version": {"foo/bar", "", "version=123", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar[version=123]", "123"}, + "with keys lookup and label": {"foo/bar", "key1.key2", "version=123", config.SecretMgrPrefix, "key1.key2", "foo/bar", "AWSSECRETS://foo/bar|key1.key2[version=123]", "123"}, + "with keys lookup and longer token": {"foo/bar", "key1.key2]version=123]", "", config.SecretMgrPrefix, "key1.key2]version=123]", "foo/bar", "AWSSECRETS://foo/bar|key1.key2]version=123]", ""}, + "with keys lookup but no keys": {"foo/bar/sdf/sddd.90dsfsd", "", "version=123", config.SecretMgrPrefix, "", "foo/bar/sdf/sddd.90dsfsd", "AWSSECRETS://foo/bar/sdf/sddd.90dsfsd[version=123]", "123"}, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { conf := &mockConfAwsSecrMgr{} - got, _ := config.NewParsedTokenConfig(tt.input, *config.NewConfig()) + got, _ := config.NewToken(tt.expPrefix, *config.NewConfig()) + got.WithSanitizedToken(tt.rawToken) + got.WithKeyPath(tt.keyPath) + got.WithMetadata(tt.metadataStr) + got.ParseMetadata(conf) if got.LookupKeys() != tt.expLookupKeys { @@ -152,3 +164,21 @@ func Test_TokenParser_config(t *testing.T) { }) } } + +func TestLookupIdent(t *testing.T) { + ttests := map[string]struct { + char string + expect config.TokenType + }{ + "new line": {"\n", config.NEW_LINE}, + "dash": {"-", config.TEXT}, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + got := config.LookupIdent(tt.char) + if got != tt.expect { + t.Errorf("got %v wanted %v", got, tt.expect) + } + }) + } +} diff --git a/internal/config/token.go b/internal/config/token.go new file mode 100644 index 0000000..00b13a6 --- /dev/null +++ b/internal/config/token.go @@ -0,0 +1,80 @@ +package config + +// TokenType is the lexer parsed TokenType +type TokenType string + +const ( + ILLEGAL TokenType = "ILLEGAL" + EOF TokenType = "EOF" + + SPACE TokenType = "SPACE" // ' ' + TAB TokenType = "TAB" // '\t' + NEW_LINE TokenType = "NEW_LINE" // '\n' + CARRIAGE_RETURN TokenType = "CARRIAGE_RETURN" // '\r' + CONTROL TokenType = "CONTROL" + + // Identifiers + literals + TEXT TokenType = "TEXT" + + EXCLAMATION TokenType = "!" + DOUBLE_QUOTE TokenType = "\"" + SINGLE_QUOTE TokenType = "'" + // other separators + AT_SIGN TokenType = "AT_SIGN" // `@` + PIPE TokenType = "PIPE" // `|` + COLON TokenType = "COLON" // `:` + EQUALS TokenType = "EQUALS" // `=` + DOT TokenType = "DOT" // `.` + COMMA TokenType = "COMMA" // `,` + QUESTION_MARK TokenType = "QUESTION_MARK" // `?` + BACK_SLASH TokenType = "BACK_SLASH" // `\` + FORWARD_SLASH TokenType = "FORWARD_SLASH" // `/` + SLASH_QUESTION_MARK TokenType = "SLASH_QUESTION_MARK" // `/?` + + // Comment Tokens + DOUBLE_FORWARD_SLASH TokenType = "DOUBLE_FORWARD_SLASH" // `//` + HASH TokenType = "HASH" // `#` + + // CONFIGMANAGER_TOKEN Keywords + // CONFIGMANAGER_TOKEN_SEPARATOR TokenType = "CONFIGMANAGER_TOKEN_SEPARATOR" // Dynamically set + BEGIN_CONFIGMANAGER_TOKEN TokenType = "BEGIN_CONFIGMANAGER_TOKEN" // Dynamically set + CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR TokenType = "CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR" // Dynamically set + BEGIN_META_CONFIGMANAGER_TOKEN TokenType = "BEGIN_META_CONFIGMANAGER_TOKEN" // `[` + END_META_CONFIGMANAGER_TOKEN TokenType = "END_META_CONFIGMANAGER_TOKEN" // `]` + // This may not possible + END_CONFIGMANAGER_TOKEN TokenType = "END_CONFIGMANAGER_TOKEN" + + // Parsed "expressions" + CONFIGMANAGER_TOKEN_CONTENT TokenType = "CONFIGMANAGER_TOKEN_CONTENT" + UNUSED_TEXT TokenType = "UNUSED_TEXT" +) + +type Source struct { + File string `json:"file"` + Path string `json:"path"` +} + +// Token is the basic structure of the captured token +type Token struct { + Type TokenType + Literal string + ImpPrefix ImplementationPrefix + Line int + Column int + Source Source +} + +var keywords = map[string]TokenType{ + " ": SPACE, + "\n": NEW_LINE, + "\r": CARRIAGE_RETURN, + "\t": TAB, + "\f": CONTROL, +} + +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[ident]; ok { + return tok + } + return TEXT +} diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go new file mode 100644 index 0000000..a8f5d5a --- /dev/null +++ b/internal/lexer/lexer.go @@ -0,0 +1,241 @@ +// Package lexer +// +// Performs lexical analysis on the source files and emits tokens. +package lexer + +import ( + "github.com/DevLabFoundry/configmanager/v3/internal/config" +) + +// nonText characters captures all character sets that are _not_ assignable to TEXT +var nonText = map[string]bool{ + // separators + " ": true, "\n": true, "\r": true, "\t": true, + "=": true, ".": true, ",": true, "|": true, "?": true, "/": true, "@": true, ":": true, + "]": true, "[": true, "'": true, "\"": true, + // initial chars of potential identifiers + // this forces the lexer to not treat at as TEXT + // and enter the switch statement of the state machine + // NOTE: when a new implementation is added we should add it here + // AWS|AZure + "A": true, + // VAULT (HashiCorp) + "V": true, + // GCP + "G": true, +} + +type Source struct { + Input string + FileName string + FullPath string +} + +// Lexer +type Lexer struct { + config config.GenVarsConfig + keySeparator byte + length int + source Source + position int // current position in input (points to current char) + readPosition int // current reading position in input (after current char) + ch byte // current char under examination + line int // current line - start at 1 + column int // column of text - gets set to 0 on every new line - start at 0 +} + +// New returns a Lexer pointer allocation +func New(source Source, config config.GenVarsConfig) *Lexer { + l := &Lexer{ + source: source, + line: 1, + column: 0, + length: len(source.Input), + config: config, + keySeparator: config.KeySeparator()[0], + } + l.readChar() + return l +} + +// NextToken advances through the source returning a found token +func (l *Lexer) NextToken() config.Token { + var tok config.Token + + switch l.ch { + // identify the dynamically selected key separator + case l.keySeparator: + tok = config.Token{Type: config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, Literal: string(l.ch)} + // Specific cases for BEGIN_CONFIGMANAGER_TOKEN possibilities + case 'A': + if l.peekChar() == 'W' { + // AWS store types + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.SecretMgrPrefix, config.ParamStorePrefix}, "AW"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AW as text + tok = config.Token{Type: config.TEXT, Literal: "AW"} + } + } else if l.peekChar() == 'Z' { + // Azure Store Types + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.AzKeyVaultSecretsPrefix, config.AzTableStorePrefix, config.AzAppConfigPrefix}, "AZ"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AZ as text + tok = config.Token{Type: config.TEXT, Literal: "AZ"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "A"} + } + case 'G': + // GCP TOKENS + if l.peekChar() == 'C' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.GcpSecretsPrefix}, "GC"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker - GC literal as text + tok = config.Token{Type: config.TEXT, Literal: "GC"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "G"} + } + case 'V': + // HASHI VAULT Tokens + if l.peekChar() == 'A' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.HashicorpVaultPrefix}, "VA"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker VA as text + tok = config.Token{Type: config.TEXT, Literal: "VA"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "V"} + } + case '=': + tok = config.Token{Type: config.EQUALS, Literal: "="} + case '.': + tok = config.Token{Type: config.DOT, Literal: "."} + case ',': + tok = config.Token{Type: config.COMMA, Literal: ","} + case '/': + if l.peekChar() == '?' { + l.readChar() + tok = config.Token{Type: config.SLASH_QUESTION_MARK, Literal: "/?"} + } else { + tok = config.Token{Type: config.FORWARD_SLASH, Literal: "/"} + } + case '\\': + tok = config.Token{Type: config.BACK_SLASH, Literal: "\\"} + case '?': + tok = config.Token{Type: config.QUESTION_MARK, Literal: "?"} + case ']': + tok = config.Token{Type: config.END_META_CONFIGMANAGER_TOKEN, Literal: "]"} + case '[': + tok = config.Token{Type: config.BEGIN_META_CONFIGMANAGER_TOKEN, Literal: "["} + case '|': + tok = config.Token{Type: config.PIPE, Literal: "|"} + case '@': + tok = config.Token{Type: config.AT_SIGN, Literal: "@"} + case ':': + tok = config.Token{Type: config.COLON, Literal: ":"} + case '"': + tok = config.Token{Type: config.DOUBLE_QUOTE, Literal: "\""} + case '\'': + tok = config.Token{Type: config.SINGLE_QUOTE, Literal: "'"} + case '\n': + l.line = l.line + 1 + l.column = 0 // reset column count + tok = l.setTextSeparatorToken() + // want to preserve all indentations and punctuation + case ' ', '\r', '\t', '\f': + tok = l.setTextSeparatorToken() + case 0: + tok.Literal = "" + tok.Type = config.EOF + default: + if isText(l.ch) { + tok.Literal = l.readText() + tok.Type = config.TEXT + return tok + } + tok = newToken(config.ILLEGAL, l.ch) + } + // add general properties to each token + tok.Line = l.line + tok.Column = l.column + tok.Source = config.Source{Path: l.source.FullPath, File: l.source.FileName} + l.readChar() + return tok +} + +// readChar moves cursor along +func (l *Lexer) readChar() { + if l.readPosition >= l.length { + l.ch = 0 + } else { + l.ch = l.source.Input[l.readPosition] + } + l.position = l.readPosition + l.readPosition += 1 + l.column += 1 +} + +// peekChar reveals next char withouh advancing the cursor along +func (l *Lexer) peekChar() byte { + if l.readPosition >= l.length { + return 0 + } else { + return l.source.Input[l.readPosition] + } +} + +func (l *Lexer) readText() string { + position := l.position + for isText(l.ch) && l.readPosition <= l.length { + l.readChar() + } + return l.source.Input[position:l.position] +} + +func (l *Lexer) setTextSeparatorToken() config.Token { + tok := newToken(config.LookupIdent(string(l.ch)), l.ch) + return tok +} + +// peekIsBeginOfToken attempts to identify the possible token +func (l *Lexer) peekIsBeginOfToken(possibleBeginToken []config.ImplementationPrefix, charsRead string) (bool, string, config.ImplementationPrefix) { + for _, pbt := range possibleBeginToken { + configToken := "" + pbtWithTokenSep := string(pbt[len(charsRead):]) + l.config.TokenSeparator() + for i := 0; i < len(pbtWithTokenSep); i++ { + configToken += string(l.peekChar()) + l.readChar() + } + + if configToken == pbtWithTokenSep { + return true, charsRead + configToken, pbt + } + l.resetAfterPeek(len(pbtWithTokenSep)) + } + return false, "", "" +} + +// resetAfterPeek will go back specified amount on the cursor +func (l *Lexer) resetAfterPeek(back int) { + l.position = l.position - back + l.readPosition = l.readPosition - back +} + +// isText only deals with any text characters defined as +// outside of the capture group +func isText(ch byte) bool { + return !nonText[string(ch)] +} + +func newToken(tokenType config.TokenType, ch byte) config.Token { + return config.Token{Type: tokenType, Literal: string(ch)} +} diff --git a/internal/lexer/lexer_test.go b/internal/lexer/lexer_test.go new file mode 100644 index 0000000..97f6ba0 --- /dev/null +++ b/internal/lexer/lexer_test.go @@ -0,0 +1,99 @@ +package lexer_test + +import ( + "testing" + + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" +) + +func Test_Lexer_NextToken(t *testing.T) { + input := `foo stuyfsdfsf +foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo +META_INCLUDED=VAULT://baz/bar/123|key1.prop2[role=arn:aws:iam::1111111:role,version=1082313] +` + ttests := []struct { + expectedType config.TokenType + expectedLiteral string + }{ + {config.TEXT, "foo"}, + {config.SPACE, " "}, + {config.TEXT, "stuyfsdfsf"}, + {config.NEW_LINE, "\n"}, + {config.TEXT, "foo"}, + {config.EQUALS, "="}, + {config.BEGIN_CONFIGMANAGER_TOKEN, "AWSPARAMSTR://"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "path"}, + {config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {config.TEXT, "key"}, + {config.BEGIN_CONFIGMANAGER_TOKEN, "AWSSECRETS://"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "foo"}, + {config.NEW_LINE, "\n"}, + {config.TEXT, "MET"}, + {config.TEXT, "A"}, + {config.TEXT, "_INCLUDED"}, + // {config.TEXT, "U"}, + // {config.TEXT, "DED"}, + {config.EQUALS, "="}, + {config.BEGIN_CONFIGMANAGER_TOKEN, "VAULT://"}, + {config.TEXT, "baz"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "bar"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "123"}, + {config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {config.TEXT, "key1"}, + {config.DOT, "."}, + {config.TEXT, "prop2"}, + {config.BEGIN_META_CONFIGMANAGER_TOKEN, "["}, + {config.TEXT, "role"}, + {config.EQUALS, "="}, + {config.TEXT, "arn"}, + {config.COLON, ":"}, + {config.TEXT, "aws"}, + {config.COLON, ":"}, + {config.TEXT, "iam"}, + {config.COLON, ":"}, + {config.COLON, ":"}, + {config.TEXT, "1111111"}, + {config.COLON, ":"}, + {config.TEXT, "role"}, + {config.COMMA, ","}, + {config.TEXT, "version"}, + {config.EQUALS, "="}, + {config.TEXT, "1082313"}, + {config.END_META_CONFIGMANAGER_TOKEN, "]"}, + {config.NEW_LINE, "\n"}, + {config.EOF, ""}, + } + + l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig()) + + for i, tt := range ttests { + + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. got=%q, expected=%q", + i, tok.Type, tt.expectedType) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - literal wrong. got=%q, expected=%q", + i, tok.Literal, tt.expectedLiteral) + } + if tok.Type == config.BEGIN_CONFIGMANAGER_TOKEN { + + } + } +} + +func Test_empty_file(t *testing.T) { + input := `` + l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig()) + tok := l.NextToken() + if tok.Type != config.EOF { + t.Fatal("expected EOF") + } +} diff --git a/internal/log/log_test.go b/internal/log/log_test.go index be24a28..10922fa 100644 --- a/internal/log/log_test.go +++ b/internal/log/log_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func Test_LogInfo(t *testing.T) { diff --git a/internal/parser/doc.go b/internal/parser/doc.go new file mode 100644 index 0000000..c33ca52 --- /dev/null +++ b/internal/parser/doc.go @@ -0,0 +1,5 @@ +// Package parser +// Analyses a given string of text and extracts any configmanager tokens +// +// It builds any additiona metadata as part of the analysis +package parser diff --git a/internal/parser/parser.go b/internal/parser/parser.go new file mode 100644 index 0000000..b785dc2 --- /dev/null +++ b/internal/parser/parser.go @@ -0,0 +1,266 @@ +package parser + +import ( + "errors" + "fmt" + "os" + + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" +) + +func wrapErr(incompleteToken *config.ParsedTokenConfig, sanitized string, line, position int, etyp error) error { + return fmt.Errorf("\n- token: (%s%s%s) on line: %d column: %d] %w", incompleteToken.Prefix(), incompleteToken.TokenSeparator(), sanitized, line, position, etyp) +} + +var ( + ErrNoEndTagFound = errors.New("no corresponding end tag found") + ErrUnableToReplaceVarPlaceholder = errors.New("variable specified in the content was not found in the environment") +) + +type ConfigManagerTokenBlock struct { + BeginToken config.Token + ParsedToken config.ParsedTokenConfig + EndToken config.Token +} + +type Parser struct { + l *lexer.Lexer + errors []error + log log.ILogger + currentToken config.Token + peekToken config.Token + config *config.GenVarsConfig + environ []string +} + +func New(l *lexer.Lexer, c *config.GenVarsConfig) *Parser { + p := &Parser{ + l: l, + log: log.New(os.Stderr), + errors: []error{}, + config: c, + environ: os.Environ(), + } + + // Read two tokens, so curToken and peekToken are both set + // first one sets the curToken to the value of peekToken - + // which at this point is just the first upcoming token + p.nextToken() + // second one sets the curToken to the actual value of the first upcoming + // token and peekToken is the actual second upcoming token + p.nextToken() + + return p +} + +func (p *Parser) WithLogger(logger log.ILogger) *Parser { + p.log = nil //speed up GC + p.log = logger + return p +} + +// Parse creates a flat list of ConfigManagerTokenBlock +// In the order they were declared in the source text +// +// The parser does not do a second pass and interprets the source from top to bottom +func (p *Parser) Parse() ([]ConfigManagerTokenBlock, []error) { + stmts := []ConfigManagerTokenBlock{} + + for !p.currentTokenIs(config.EOF) { + if p.currentTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { + // continues to read the tokens until it hits an end token or errors + configManagerToken, err := config.NewToken(p.currentToken.ImpPrefix, *p.config) + if err != nil { + return nil, []error{err} + } + if stmt := p.buildConfigManagerTokenFromBlocks(configManagerToken); stmt != nil { + stmts = append(stmts, *stmt) + } + } + p.nextToken() + } + + return stmts, p.errors +} + +func (p *Parser) nextToken() { + p.currentToken = p.peekToken + p.peekToken = p.l.NextToken() +} + +func (p *Parser) currentTokenIs(t config.TokenType) bool { + return p.currentToken.Type == t +} + +func (p *Parser) peekTokenIs(t config.TokenType) bool { + return p.peekToken.Type == t +} + +func (p *Parser) peekTokenIsEnd() bool { + endTokens := map[config.TokenType]bool{ + config.AT_SIGN: true, config.QUESTION_MARK: true, config.COLON: true, + config.SLASH_QUESTION_MARK: true, config.EOF: true, + // traditional ends of tokens + config.DOUBLE_QUOTE: true, config.SINGLE_QUOTE: true, config.SPACE: true, + config.NEW_LINE: true, + } + return endTokens[p.peekToken.Type] +} + +// buildConfigManagerTokenFromBlocks +func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.ParsedTokenConfig) *ConfigManagerTokenBlock { + currentToken := p.currentToken + stmt := &ConfigManagerTokenBlock{BeginToken: currentToken} + + // move past current token + p.nextToken() + + // built as part of the below parser + sanitizedToken := "" + + // stop on end of file + for !p.peekTokenIs(config.EOF) { + // // This is the target state when there is an optional token wrapping + // // e.g. `{{ IMP://path }}` + // // currently this is untestable + // if p.peekTokenIs(config.END_CONFIGMANAGER_TOKEN) { + // notFoundEnd = false + // fullToken += p.curToken.Literal + // sanitizedToken += p.curToken.Literal + // stmt.EndToken = p.curToken + // break + // } + + // when next token is another token + // i.e. the tokens are adjacent + if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { + sanitizedToken += p.currentToken.Literal + stmt.EndToken = p.currentToken + break + } + + // reached the end of token + if p.peekTokenIsEnd() { + sanitizedToken += p.currentToken.Literal + stmt.EndToken = p.currentToken + break + } + + //sample token will be consumed like this + // AWSSECRETS:///path/to/my/key|lookup.Inside.Object[meta=data] + // + // everything is token path until (if any key separator exists) + // check key separator this marks the end of a normal token path + // + // keyLookup and Metadata are optional - is always specified in that order + if p.currentTokenIs(config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR) { + if err := p.buildKeyPathSeparator(configManagerToken); err != nil { + p.errors = append(p.errors, wrapErr(configManagerToken, sanitizedToken, currentToken.Line, currentToken.Column, err)) + return nil + } + // keyPath would have built the keyPath and metadata if any + break + } + + // optionally at the end of the path without key separator + // check metadata there can be a metadata bracket `[key=val,k1=v2]` + if p.currentTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { + if err := p.buildMetadata(configManagerToken); err != nil { + p.errors = append(p.errors, wrapErr(configManagerToken, sanitizedToken, currentToken.Line, currentToken.Column, err)) + return nil + } + break + } + + sanitizedToken += p.currentToken.Literal + + // when the next token is EOF + // we want set the current token + // else it would be lost once the parser is advanced below + p.nextToken() + if p.peekTokenIs(config.EOF) { + sanitizedToken += p.currentToken.Literal + stmt.EndToken = p.currentToken + break + } + } + + configManagerToken.WithSanitizedToken(sanitizedToken) + stmt.ParsedToken = *configManagerToken + + return stmt +} + +// buildKeyPathSeparator already advanced to the first token +func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenConfig) error { + // advance to next token i.e. post the path separator + p.nextToken() + keyPath := "" + if p.peekTokenIs(config.EOF) { + // if the next token EOF we set the path as current token and exit + // otherwise we would never hit the below loop + configManagerToken.WithKeyPath(p.currentToken.Literal) + return nil + } + for !p.peekTokenIs(config.EOF) { + if p.peekTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { + // add current token to the keysPath and move onto the metadata + keyPath += p.currentToken.Literal + p.nextToken() + if err := p.buildMetadata(configManagerToken); err != nil { + return err + } + break + } + // touching another token or end of token + if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() { + keyPath += p.currentToken.Literal + break + } + keyPath += p.currentToken.Literal + p.nextToken() + if p.peekTokenIs(config.EOF) { + // check if the next token is EOF once advanced + // if it is we want to consume current token else it will be skipped + keyPath += p.currentToken.Literal + break + } + } + configManagerToken.WithKeyPath(keyPath) + return nil +} + +var ErrMetadataEmpty = errors.New("emtpy metadata") + +// buildMetadata adds metadata to the ParsedTokenConfig +func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) error { + metadata := "" + found := false + if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { + return fmt.Errorf("%w, metadata brackets must include at least one set of key=value pairs", ErrMetadataEmpty) + } + p.nextToken() + for !p.peekTokenIs(config.EOF) { + if p.peekTokenIsEnd() { + // next token is an end of token but no closing `]` found + return fmt.Errorf("%w, metadata (%s) string has no closing", ErrNoEndTagFound, metadata) + } + if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { + metadata += p.currentToken.Literal + found = true + p.nextToken() + break + } + metadata += p.currentToken.Literal + p.nextToken() + } + configManagerToken.WithMetadata(metadata) + + if !found { + // hit the end of file and no end tag found + return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) + } + return nil +} diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go new file mode 100644 index 0000000..2a97c04 --- /dev/null +++ b/internal/parser/parser_test.go @@ -0,0 +1,304 @@ +package parser_test + +import ( + "errors" + "os" + "testing" + + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/parser" + "github.com/DevLabFoundry/configmanager/v3/internal/store" +) + +var lexerSource = lexer.Source{FileName: "bar", FullPath: "/foo/bar"} + +func Test_ParserBlocks(t *testing.T) { + ttests := map[string]struct { + input string + // prefix,path,keyLookup + expected [][3]string + }{ + "tokens touching each other in source after key path": { + `foo stuyfsdfsf + foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo + other text her + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", "key"}, + {string(config.SecretMgrPrefix), "/foo", ""}, + }}, + "full URL of tokens": { + `foo stuyfsdfsf + foo=proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2 + # some comment + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/config", "user"}, + {string(config.SecretMgrPrefix), "/creds", "password"}, + {string(config.ParamStorePrefix), "/config", "endpoint"}, + {string(config.ParamStorePrefix), "/config", "port"}, + {string(config.ParamStorePrefix), "/config", "qp2"}, + }, + }, + "tokens touching each other in source after metadata": { + `foo stuyfsdfsf + foo=AWSPARAMSTR:///path|key[meta=val]AWSSECRETS:///foo + other text her + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", "key"}, + {string(config.SecretMgrPrefix), "/foo", ""}, + }, + }, + "tokens touching each other in source": { + `foo stuyfsdfsf GCFOO VAbarAWbuX AZmore + foo=AWSPARAMSTR:///pathAWSSECRETS:///foo + other text her + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", ""}, + {string(config.SecretMgrPrefix), "/foo", ""}, + }, + }, + "touching EOF single token": { + `AWSPARAMSTR:///config|qp2`, + [][3]string{ + {string(config.ParamStorePrefix), "/config", "qp2"}, + }, + }, + "touching EOF multi token": { + `proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2`, + [][3]string{ + {string(config.ParamStorePrefix), "/config", "user"}, + {string(config.SecretMgrPrefix), "/creds", "password"}, + {string(config.ParamStorePrefix), "/config", "endpoint"}, + {string(config.ParamStorePrefix), "/config", "port"}, + {string(config.ParamStorePrefix), "/config", "qp2"}, + }, + }, + } + + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + l := lexer.New(lexerSource, *config.NewConfig()) + p := parser.New(l, config.NewConfig()).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("parser had errors, expected \nerror: %v", errs) + } + + if len(parsed) != len(tt.expected) { + t.Fatalf("parsed statements count does not match\ngot=%d want=%d\nparsed %q", + len(parsed), + len(tt.expected), + parsed) + } + + for idx, stmt := range parsed { + if !testHelperGenDocBlock(t, stmt, config.ImplementationPrefix(tt.expected[idx][0]), tt.expected[idx][1], tt.expected[idx][2]) { + return + } + } + }) + } +} + +func Test_Parse_should_fail_on_metadata(t *testing.T) { + ttests := map[string]struct { + input string + errTyp error + }{ + "when _end_tag_found without keysPath": { + `AWSSECRETS:///foo[version=1.2.3`, + parser.ErrNoEndTagFound, + }, + "when _end_tag_found with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3`, + parser.ErrNoEndTagFound, + }, + "when _end_tag_found with keysPath in the middle": { + `AWSSECRETS:///foo|path.one[version=1.2.3 + more content here +`, + parser.ErrNoEndTagFound, + }, + "when no metadata has been supplied": { + `AWSSECRETS:///foo|path.one[]`, + parser.ErrMetadataEmpty, + }, + "when no metadata has been supplied - without key path": { + `AWSSECRETS:///foo[]`, + parser.ErrMetadataEmpty, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + _, errs := p.Parse() + if len(errs) != 1 { + t.Fatalf("unexpected number of errors\n got: %v, wanted: 1", errs) + } + if !errors.Is(errs[0], tt.errTyp) { + t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, parser.ErrNoEndTagFound) + } + }) + } +} + +func Test_Parse_should_pass_with_metadata_end_tag(t *testing.T) { + ttests := map[string]struct { + input string + metdataStr string + }{ + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3]`, + `version=1.2.3`, + }, + "without keysPath in the middle of content": { + `AWSSECRETS:///foo[version=1.2.3] +`, + `version=1.2.3`, + }, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3]`, + `version=1.2.3`, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("unexpected number of errors\n got: %v, wanted: 0", errs) + } + for _, prsd := range parsed { + prsd.ParsedToken.LookupKeys() + + } + }) + } +} + +func Test_Parse_ParseMetadata(t *testing.T) { + + ttests := map[string]struct { + input string + typ *store.SecretsMgrConfig + }{ + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3]`, + &store.SecretsMgrConfig{}, + }, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3]`, + &store.SecretsMgrConfig{}, + }, + "nestled in text": { + `someQ=AWSPARAMSTR:///path/queryparam|p1[version=1.2.3]&anotherQ`, + &store.SecretsMgrConfig{}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("%v", errs) + } + + for _, p := range parsed { + if err := p.ParsedToken.ParseMetadata(tt.typ); err != nil { + t.Fatal(err) + } + if tt.typ.Version != "1.2.3" { + t.Errorf("got %v wanted 1.2.3", tt.typ.Version) + } + } + }) + } +} + +func Test_Parse_Path_Keys_WithParsedMetadat(t *testing.T) { + + ttests := map[string]struct { + input string + typ *store.SecretsMgrConfig + wantSanitizedPath string + wantKeyPath string + }{ + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3]`, + &store.SecretsMgrConfig{}, + "/foo", "", + }, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3]`, + &store.SecretsMgrConfig{}, + "/foo", "path.one", + }, + "nestled in text": { + `someQ=AWSPARAMSTR:///path/queryparam|p1[version=1.2.3]&anotherQ`, + &store.SecretsMgrConfig{}, + "/path/queryparam", "p1", + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("%v", errs) + } + + for _, p := range parsed { + if p.ParsedToken.StoreToken() != tt.wantSanitizedPath { + t.Errorf("got %s want %s", p.ParsedToken.StoreToken(), tt.wantSanitizedPath) + } + if p.ParsedToken.LookupKeys() != tt.wantKeyPath { + t.Errorf("got %s want %s", p.ParsedToken.LookupKeys(), tt.wantKeyPath) + } + if err := p.ParsedToken.ParseMetadata(tt.typ); err != nil { + t.Fatal(err) + } + if tt.typ.Version != "1.2.3" { + t.Errorf("got %v wanted 1.2.3", tt.typ.Version) + } + } + }) + } +} + +func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBlock, tokenType config.ImplementationPrefix, tokenValue, keysLookupPath string) bool { + t.Helper() + if stmtBlock.ParsedToken.Prefix() != tokenType { + t.Errorf("got=%q, wanted stmtBlock.ImpPrefix = '%v'.", stmtBlock.ParsedToken.Prefix(), tokenType) + return false + } + + if stmtBlock.ParsedToken.StoreToken() != tokenValue { + t.Errorf("token StoreToken got=%s, wanted=%s", stmtBlock.ParsedToken.StoreToken(), tokenValue) + return false + } + + if stmtBlock.ParsedToken.LookupKeys() != keysLookupPath { + t.Errorf("token LookupKeys. got=%s, wanted=%s", stmtBlock.ParsedToken.LookupKeys(), keysLookupPath) + return false + } + + return true +} diff --git a/internal/store/azappconf.go b/internal/store/azappconf.go index a37cc8a..c35f538 100644 --- a/internal/store/azappconf.go +++ b/internal/store/azappconf.go @@ -11,8 +11,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) // appConfApi @@ -51,8 +51,8 @@ func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger l token: token, logger: logger, } - srvInit := azServiceFromToken(token.StoreToken(), "https://%s.azconfig.io", 1) - backingStore.strippedToken = srvInit.token + srvInit := AzServiceFromToken(token.StoreToken(), "https://%s.azconfig.io", 1) + backingStore.strippedToken = srvInit.Token cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -60,7 +60,7 @@ func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger l return nil, err } - c, err := azappconfig.NewClient(srvInit.serviceUri, cred, nil) + c, err := azappconfig.NewClient(srvInit.ServiceUri, cred, nil) if err != nil { logger.Error("failed to init the client: %v", err) return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) @@ -71,6 +71,10 @@ func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger l } +func (s *AzAppConf) WithSvc(svc appConfApi) { + s.svc = svc +} + // setTokenVal sets the token func (implmt *AzAppConf) SetToken(token *config.ParsedTokenConfig) {} @@ -78,7 +82,7 @@ func (implmt *AzAppConf) SetToken(token *config.ParsedTokenConfig) {} // label can be specified // From this point then normal rules of configmanager apply, // including keySeperator and lookup. -func (imp *AzAppConf) Token() (string, error) { +func (imp *AzAppConf) Value() (string, error) { imp.logger.Info("Concrete implementation AzAppConf") imp.logger.Info("AzAppConf Token: %s", imp.token.String()) diff --git a/internal/store/azappconf_test.go b/internal/store/azappconf_test.go index 82ed7e9..17da526 100644 --- a/internal/store/azappconf_test.go +++ b/internal/store/azappconf_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "bytes" @@ -8,9 +8,10 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - logger "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + logger "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func azAppConfCommonChecker(t *testing.T, key string, expectedKey string, expectLabel string, opts *azappconfig.GetSettingOptions) { @@ -36,20 +37,25 @@ func (m mockAzAppConfApi) GetSetting(ctx context.Context, key string, options *a } func Test_AzAppConf_Success(t *testing.T) { - t.Parallel() tsuccessParam := "somecvla" logr := logger.New(&bytes.Buffer{}) tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) appConfApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzAppConfApi }{ "successVal": { - "AZAPPCONF#/test-app-config-instance/table//token/1", + func() *config.ParsedTokenConfig { + // "AZAPPCONF#/test-app-config-instance/table//token/1", + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/table//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, tsuccessParam, - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "table//token/1", "", options) resp := azappconfig.GetSettingResponse{} @@ -57,12 +63,18 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), }, "successVal with :// token Separator": { - "AZAPPCONF:///test-app-config-instance/conf_key[label=dev]", + func() *config.ParsedTokenConfig { + // "AZAPPCONF:///test-app-config-instance/conf_key[label=dev]", + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://")) + tkn.WithSanitizedToken("/test-app-config-instance/conf_key") + tkn.WithKeyPath("") + tkn.WithMetadata("label=dev") + return tkn + }, tsuccessParam, - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "conf_key", "dev", options) resp := azappconfig.GetSettingResponse{} @@ -70,12 +82,17 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), }, "successVal with :// token Separator and etag specified": { - "AZAPPCONF:///test-app-config-instance/conf_key[label=dev,etag=sometifdsssdsfdi_string01209222]", + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/conf_key") + tkn.WithKeyPath("") + tkn.WithMetadata("label=dev,etag=sometifdsssdsfdi_string01209222") + return tkn + }, tsuccessParam, - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "conf_key", "dev", options) if !options.OnlyIfChanged.Equals("sometifdsssdsfdi_string01209222") { @@ -86,12 +103,17 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), }, "successVal with keyseparator but no val returned": { - "AZAPPCONF#/test-app-config-instance/try_to_find|key_separator.lookup", + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/try_to_find") + tkn.WithKeyPath("key_separator.lookup") + tkn.WithMetadata("") + return tkn + }, "", - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "try_to_find", "", options) resp := azappconfig.GetSettingResponse{} @@ -99,21 +121,18 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - - impl, err := NewAzAppConf(context.TODO(), token, logr) + impl, err := store.NewAzAppConf(context.TODO(), tt.token(), logr) if err != nil { t.Errorf("failed to init AZAPPCONF") } - impl.svc = tt.mockClient(t) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) @@ -129,39 +148,41 @@ func Test_AzAppConf_Success(t *testing.T) { } func Test_AzAppConf_Error(t *testing.T) { - t.Parallel() - logr := logger.New(&bytes.Buffer{}) tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect error - mockClient func(t *testing.T) appConfApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzAppConfApi }{ "errored on service method call": { - "AZAPPCONF#/test-app-config-instance/table/token/ok", - ErrRetrieveFailed, - func(t *testing.T) appConfApi { + func() *config.ParsedTokenConfig { + // "AZAPPCONF#/test-app-config-instance/table/token/ok", + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/table/token/ok") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + store.ErrRetrieveFailed, + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { t.Helper() resp := azappconfig.GetSettingResponse{} return resp, fmt.Errorf("network error") }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - impl, err := NewAzAppConf(context.TODO(), token, logr) + impl, err := store.NewAzAppConf(context.TODO(), tt.token(), logr) if err != nil { t.Fatal("failed to init AZAPPCONF") } - impl.svc = tt.mockClient(t) - if _, err := impl.Token(); !errors.Is(err, tt.expect) { + impl.WithSvc(tt.mockClient(t)) + if _, err := impl.Value(); !errors.Is(err, tt.expect) { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) } }) @@ -169,19 +190,19 @@ func Test_AzAppConf_Error(t *testing.T) { } func Test_fail_AzAppConf_Client_init(t *testing.T) { - t.Parallel() logr := logger.New(&bytes.Buffer{}) // this is basically a wrap around test for the url.Parse method in the stdlib // as that is what the client uses under the hood - token, _ := config.NewParsedTokenConfig("AZAPPCONF:///%25%65%6e%301-._~/") } - if !errors.Is(err, ErrClientInitialization) { - t.Fatalf(testutils.TestPhraseWithContext, "azappconf client init", err.Error(), ErrClientInitialization.Error()) + if !errors.Is(err, store.ErrClientInitialization) { + t.Fatalf(testutils.TestPhraseWithContext, "azappconf client init", err.Error(), store.ErrClientInitialization.Error()) } } diff --git a/internal/store/azhelpers.go b/internal/store/azhelpers.go index 7b85387..29e66e8 100644 --- a/internal/store/azhelpers.go +++ b/internal/store/azhelpers.go @@ -8,13 +8,14 @@ import ( /* Generic Azure Service Init Helpers */ -// azServiceHelper returns a service URI and the stripped token -type azServiceHelper struct { - serviceUri string - token string + +// AzServiceHelper returns a service URI and the stripped token +type AzServiceHelper struct { + ServiceUri string + Token string } -// azServiceFromToken for azure the first part of the token __must__ always be the +// AzServiceFromToken for azure the first part of the token __must__ always be the // identifier of the service e.g. the account name for tableStore or the Vault name for KVSecret or // AppConfig instance // take parameter specifies the number of elements to take from the start only @@ -22,7 +23,7 @@ type azServiceHelper struct { // e.g. a value of 2 for take will take first 2 elements from the slices // // For AppConfig or KeyVault we ONLY need the AppConfig instance or KeyVault instance name -func azServiceFromToken(token string, formatUri string, take int) azServiceHelper { +func AzServiceFromToken(token string, formatUri string, take int) AzServiceHelper { // ensure preceding slash is trimmed stringToken := strings.Split(strings.TrimPrefix(token, "/"), "/") splitToken := []any{} @@ -32,5 +33,5 @@ func azServiceFromToken(token string, formatUri string, take int) azServiceHelpe } uri := fmt.Sprintf(formatUri, splitToken[0:take]...) - return azServiceHelper{serviceUri: uri, token: strings.Join(stringToken[take:], "/")} + return AzServiceHelper{ServiceUri: uri, Token: strings.Join(stringToken[take:], "/")} } diff --git a/internal/store/azkeyvault.go b/internal/store/azkeyvault.go index 84f1715..781b066 100644 --- a/internal/store/azkeyvault.go +++ b/internal/store/azkeyvault.go @@ -8,8 +8,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) type kvApi interface { @@ -45,8 +45,8 @@ func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger token: token, } - srvInit := azServiceFromToken(token.StoreToken(), "https://%s.vault.azure.net", 1) - backingStore.strippedToken = srvInit.token + srvInit := AzServiceFromToken(token.StoreToken(), "https://%s.vault.azure.net", 1) + backingStore.strippedToken = srvInit.Token cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -54,7 +54,7 @@ func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger return nil, err } - c, err := azsecrets.NewClient(srvInit.serviceUri, cred, nil) + c, err := azsecrets.NewClient(srvInit.ServiceUri, cred, nil) if err != nil { logger.Error("%v\n%w", err, ErrClientInitialization) return nil, err @@ -65,10 +65,14 @@ func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger } +func (s *KvScrtStore) WithSvc(svc kvApi) { + s.svc = svc +} + // setToken already happens in AzureKVClient in the constructor func (implmt *KvScrtStore) SetToken(token *config.ParsedTokenConfig) {} -func (imp *KvScrtStore) Token() (string, error) { +func (imp *KvScrtStore) Value() (string, error) { imp.logger.Info("Concrete implementation AzKeyVault Secret") imp.logger.Info("AzKeyVault Token: %s", imp.token.String()) diff --git a/internal/store/azkeyvault_test.go b/internal/store/azkeyvault_test.go index 98ccda8..35b5c7d 100644 --- a/internal/store/azkeyvault_test.go +++ b/internal/store/azkeyvault_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -8,58 +8,59 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func Test_azSplitToken(t *testing.T) { tests := []struct { name string token string - expect azServiceHelper + expect store.AzServiceHelper }{ { name: "simple_with_preceding_slash", token: "/test-vault/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "somejsontest", }, }, { name: "missing_initial_slash", token: "test-vault/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "somejsontest", }, }, { name: "missing_initial_slash_multislash_secretname", token: "test-vault/some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "some/json/test", }, }, { name: "with_initial_slash_multislash_secretname", token: "test-vault//some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "/some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "/some/json/test", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := azServiceFromToken(tt.token, "https://%s.vault.azure.net", 1) - if got.token != tt.expect.token { - t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) + got := store.AzServiceFromToken(tt.token, "https://%s.vault.azure.net", 1) + if got.Token != tt.expect.Token { + t.Errorf(testutils.TestPhrase, tt.expect.Token, got.Token) } - if got.serviceUri != tt.expect.serviceUri { - t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) + if got.ServiceUri != tt.expect.ServiceUri { + t.Errorf(testutils.TestPhrase, tt.expect.ServiceUri, got.ServiceUri) } }) } @@ -93,82 +94,115 @@ func (m mockAzKvSecretApi) GetSecret(ctx context.Context, name string, version s } func TestAzKeyVault(t *testing.T) { - t.Parallel() - tsuccessParam := "dssdfdweiuyh" tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) kvApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzKvSecretApi }{ - "successVal": {"AZKVSECRET#/test-vault//token/1", tsuccessParam, func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "/token/1") - resp := azsecrets.GetSecretResponse{} - resp.Value = &tsuccessParam - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "successVal with version": {"AZKVSECRET#/test-vault//token/1[version:123]", tsuccessParam, func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "/token/1") - resp := azsecrets.GetSecretResponse{} - resp.Value = &tsuccessParam - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "successVal with keyseparator": {"AZKVSECRET#/test-vault/token/1|somekey", tsuccessParam, func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "token/1") - - resp := azsecrets.GetSecretResponse{} - resp.Value = &tsuccessParam - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "successVal": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + tsuccessParam, func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "/token/1") + resp := azsecrets.GetSecretResponse{} + resp.Value = &tsuccessParam + return resp, nil + }) + }, }, - "errored": {"AZKVSECRET#/test-vault/token/1|somekey", "unable to retrieve secret", func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "token/1") - - resp := azsecrets.GetSecretResponse{} - return resp, fmt.Errorf("unable to retrieve secret") - }) + "successVal with version": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version:123") + return tkn + }, tsuccessParam, func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "/token/1") + resp := azsecrets.GetSecretResponse{} + resp.Value = &tsuccessParam + return resp, nil + }) + }, }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "successVal with keyseparator": { + func() *config.ParsedTokenConfig { + // "AZKVSECRET#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, tsuccessParam, func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "token/1") + + resp := azsecrets.GetSecretResponse{} + resp.Value = &tsuccessParam + return resp, nil + }) + }, }, - "empty": {"AZKVSECRET#/test-vault/token/1|somekey", "", func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "token/1") - - resp := azsecrets.GetSecretResponse{} - return resp, nil - }) + "errored": { + func() *config.ParsedTokenConfig { + // "AZKVSECRET#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, + "unable to retrieve secret", + func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "token/1") + + resp := azsecrets.GetSecretResponse{} + return resp, fmt.Errorf("unable to retrieve secret") + }) + }, }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "empty": { + func() *config.ParsedTokenConfig { + // "AZKVSECRET#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, "", func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "token/1") + + resp := azsecrets.GetSecretResponse{} + return resp, nil + }) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - - impl, err := NewKvScrtStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewKvScrtStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Errorf("failed to init azkvstore") } - impl.svc = tt.mockClient(t) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/aztablestorage.go b/internal/store/aztablestorage.go index 539979b..eedef16 100644 --- a/internal/store/aztablestorage.go +++ b/internal/store/aztablestorage.go @@ -12,8 +12,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) var ErrIncorrectlyStructuredToken = errors.New("incorrectly structured token") @@ -52,8 +52,8 @@ func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logge token: token, } - srvInit := azServiceFromToken(token.StoreToken(), "https://%s.table.core.windows.net/%s", 2) - backingStore.strippedToken = srvInit.token + srvInit := AzServiceFromToken(token.StoreToken(), "https://%s.table.core.windows.net/%s", 2) + backingStore.strippedToken = srvInit.Token cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -61,7 +61,7 @@ func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logge return nil, err } - c, err := aztables.NewClient(srvInit.serviceUri, cred, nil) + c, err := aztables.NewClient(srvInit.ServiceUri, cred, nil) if err != nil { logger.Error("failed to init the client: %v", err) return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) @@ -71,6 +71,10 @@ func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logge return backingStore, nil } +func (s *AzTableStore) WithSvc(svc tableStoreApi) { + s.svc = svc +} + // setToken already happens in the constructor func (implmt *AzTableStore) SetToken(token *config.ParsedTokenConfig) {} @@ -79,7 +83,7 @@ func (implmt *AzTableStore) SetToken(token *config.ParsedTokenConfig) {} // // From this point then normal rules of configmanager apply, // including keySeperator and lookup. -func (imp *AzTableStore) Token() (string, error) { +func (imp *AzTableStore) Value() (string, error) { imp.logger.Info("AzTableSTore Token: %s", imp.token.String()) imp.logger.Info("Concrete implementation AzTableSTore") diff --git a/internal/store/aztablestorage_test.go b/internal/store/aztablestorage_test.go index 54006cc..892ee9e 100644 --- a/internal/store/aztablestorage_test.go +++ b/internal/store/aztablestorage_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -9,9 +9,10 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func azTableStoreCommonChecker(t *testing.T, partitionKey, rowKey, expectedPartitionKey, expectedRowKey string) { @@ -41,55 +42,62 @@ func (m mockAzTableStoreApi) GetEntity(ctx context.Context, partitionKey string, func Test_AzTableStore_Success(t *testing.T) { tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) tableStoreApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzTableStoreApi }{ - "successVal": {"AZTABLESTORE#/test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") - resp := aztables.GetEntityResponse{} - resp.Value = []byte("tsuccessParam") - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "successVal with :// token Separator": {"AZTABLESTORE:///test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") - resp := aztables.GetEntityResponse{} - resp.Value = []byte("tsuccessParam") - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), + "successVal": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-account/table//token/1" + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-account/table//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "tsuccessParam", func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + resp := aztables.GetEntityResponse{} + resp.Value = []byte("tsuccessParam") + return resp, nil + }) + }, }, - "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", "", func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + // "successVal with :// token Separator": {"AZTABLESTORE:///test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + // resp := aztables.GetEntityResponse{} + // resp.Value = []byte("tsuccessParam") + // return resp, nil + // }) + // }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), + // }, + // "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", "", func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") - resp := aztables.GetEntityResponse{} - resp.Value = nil - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, + // resp := aztables.GetEntityResponse{} + // resp.Value = nil + // return resp, nil + // }) + // }, + // config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + // }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewAzTableStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Errorf("failed to init aztablestore") } - impl.svc = tt.mockClient(t) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) @@ -105,75 +113,95 @@ func Test_AzTableStore_Success(t *testing.T) { } func Test_azstorage_with_value_property(t *testing.T) { - t.Parallel() + conf := config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://") ttests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) tableStoreApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzTableStoreApi }{ "return value property with json like object": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey|host", + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey|host", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + tkn.WithKeyPath("host") + return tkn + }, "map[bool:true host:foo port:1234]", - func(t *testing.T) tableStoreApi { + func(t *testing.T) mockAzTableStoreApi { return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { t.Helper() resp := aztables.GetEntityResponse{Value: []byte(`{"value":{"host":"foo","port":1234,"bool":true}}`)} return resp, nil }) }, - conf, }, "return value property with string only": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + // tkn.WithKeyPath("host") + // tkn.WithMetadata("version:123]") + return tkn + }, "foo.bar.com", - func(t *testing.T) tableStoreApi { + func(t *testing.T) mockAzTableStoreApi { return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { t.Helper() resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} return resp, nil }) }, - conf, }, "return value property with numeric only": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + // tkn.WithKeyPath("host") + // tkn.WithMetadata("version:123]") + return tkn + }, "1234", - func(t *testing.T) tableStoreApi { + func(t *testing.T) mockAzTableStoreApi { return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { t.Helper() resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} return resp, nil }) }, - conf, }, "return value property with boolean only": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + return tkn + }, "false", - func(t *testing.T) tableStoreApi { + func(t *testing.T) mockAzTableStoreApi { return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { t.Helper() resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} return resp, nil }) }, - conf, }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) + // token, _ := config.NewToken(tt.token(), *tt.config) - impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewAzTableStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Fatal("failed to init aztablestore") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) - got, err := impl.Token() + got, err := impl.Value() if err != nil { t.Fatalf(testutils.TestPhrase, err.Error(), nil) } @@ -186,55 +214,71 @@ func Test_azstorage_with_value_property(t *testing.T) { } func Test_AzTableStore_Error(t *testing.T) { - t.Parallel() tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect error - mockClient func(t *testing.T) tableStoreApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzTableStoreApi }{ - "errored on token parsing to partiationKey": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{} - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "errored on service method call": {"AZTABLESTORE#/test-account/table/token/ok", ErrRetrieveFailed, func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{} - return resp, fmt.Errorf("network error") - }) + "errored on token parsing to partiationKey": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, store.ErrIncorrectlyStructuredToken, func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, nil + }) + }, }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "errored on service method call": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-account/table/token/ok", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-account/table/token/ok") + return tkn + }, + store.ErrRetrieveFailed, + func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, fmt.Errorf("network error") + }) + }, }, - "empty": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{} - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "empty": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-vault/token/1|somekey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1|somekey") + return tkn + }, + store.ErrIncorrectlyStructuredToken, func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, nil + }) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - - impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewAzTableStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Fatal("failed to init aztablestore") } - impl.svc = tt.mockClient(t) - if _, err := impl.Token(); !errors.Is(err, tt.expect) { + impl.WithSvc(tt.mockClient(t)) + if _, err := impl.Value(); !errors.Is(err, tt.expect) { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) } }) @@ -244,66 +288,66 @@ func Test_AzTableStore_Error(t *testing.T) { func Test_fail_AzTable_Client_init(t *testing.T) { // this is basically a wrap around test for the url.Parse method in the stdlib // as that is what the client uses under the hood - token, _ := config.NewParsedTokenConfig("AZTABLESTORE:///%25%65%6e%301-._~/") } - if !errors.Is(err, ErrClientInitialization) { - t.Fatalf(testutils.TestPhraseWithContext, "aztables client init", err.Error(), ErrClientInitialization.Error()) + if !errors.Is(err, store.ErrClientInitialization) { + t.Fatalf(testutils.TestPhraseWithContext, "aztables client init", err.Error(), store.ErrClientInitialization.Error()) } } func Test_azSplitTokenTableStore(t *testing.T) { - t.Parallel() tests := []struct { name string token string - expect azServiceHelper + expect store.AzServiceHelper }{ { name: "simple_with_preceding_slash", token: "/test-account/tablename/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "somejsontest", }, }, { name: "missing_initial_slash", token: "test-account/tablename/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "somejsontest", }, }, { name: "missing_initial_slash_multislash_secretname", token: "test-account/tablename/some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "some/json/test", }, }, { name: "with_initial_slash_multislash_secretname", token: "test-account/tablename//some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "/some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "/some/json/test", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := azServiceFromToken(tt.token, "https://%s.table.core.windows.net/%s", 2) - if got.token != tt.expect.token { - t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) + got := store.AzServiceFromToken(tt.token, "https://%s.table.core.windows.net/%s", 2) + if got.Token != tt.expect.Token { + t.Errorf(testutils.TestPhrase, tt.expect.Token, got.Token) } - if got.serviceUri != tt.expect.serviceUri { - t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) + if got.ServiceUri != tt.expect.ServiceUri { + t.Errorf(testutils.TestPhrase, tt.expect.ServiceUri, got.ServiceUri) } }) } diff --git a/internal/store/gcpsecrets.go b/internal/store/gcpsecrets.go index 1df7199..07c43e2 100644 --- a/internal/store/gcpsecrets.go +++ b/internal/store/gcpsecrets.go @@ -6,8 +6,8 @@ import ( gcpsecrets "cloud.google.com/go/secretmanager/apiv1" gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/googleapis/gax-go/v2" ) @@ -42,6 +42,10 @@ func NewGcpSecrets(ctx context.Context, logger log.ILogger) (*GcpSecrets, error) }, nil } +func (s *GcpSecrets) WithSvc(svc gcpSecretsApi) { + s.svc = svc +} + func (imp *GcpSecrets) SetToken(token *config.ParsedTokenConfig) { storeConf := &GcpSecretsConfig{} _ = token.ParseMetadata(storeConf) @@ -49,7 +53,7 @@ func (imp *GcpSecrets) SetToken(token *config.ParsedTokenConfig) { imp.config = storeConf } -func (imp *GcpSecrets) Token() (string, error) { +func (imp *GcpSecrets) Value() (string, error) { // Close client currently as new one would be created per iteration defer func() { _ = imp.close() diff --git a/internal/store/gcpsecrets_test.go b/internal/store/gcpsecrets_test.go index 54bdf7b..5f859ba 100644 --- a/internal/store/gcpsecrets_test.go +++ b/internal/store/gcpsecrets_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -9,9 +9,10 @@ import ( "testing" gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/googleapis/gax-go/v2" ) @@ -21,6 +22,10 @@ func (m mockGcpSecretsApi) AccessSecretVersion(ctx context.Context, req *gcpsecr return m(ctx, req, opts...) } +func (m mockGcpSecretsApi) Close() error { + return nil +} + var TEST_GCP_CREDS = []byte(`{ "type": "service_account", "project_id": "xxxxx", @@ -74,49 +79,76 @@ func gcpSecretsGetChecker(t *testing.T, req *gcpsecretspb.AccessSecretVersionReq } func Test_GetGcpSecretVarHappy(t *testing.T) { - // t.Parallel() tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) gcpSecretsApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockGcpSecretsApi }{ - "success": {"GCPSECRETS#/token/1", "someValue", func(t *testing.T) gcpSecretsApi { - return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { - gcpSecretsGetChecker(t, req) - return &gcpsecretspb.AccessSecretVersionResponse{ - Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, - }, nil - }) - }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), + "success": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "someValue", func(t *testing.T) mockGcpSecretsApi { + return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { + gcpSecretsGetChecker(t, req) + return &gcpsecretspb.AccessSecretVersionResponse{ + Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, + }, nil + }) + }, }, - "success with version": {"GCPSECRETS#/token/1[version=123]", "someValue", func(t *testing.T) gcpSecretsApi { - return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { - gcpSecretsGetChecker(t, req) - return &gcpsecretspb.AccessSecretVersionResponse{ - Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, - }, nil - }) - }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), + "success with version": { + func() *config.ParsedTokenConfig { + // "GCPSECRETS#/token/1[version=123]" + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version=123") + return tkn + }, "someValue", func(t *testing.T) mockGcpSecretsApi { + return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { + gcpSecretsGetChecker(t, req) + return &gcpsecretspb.AccessSecretVersionResponse{ + Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, + }, nil + }) + }, }, - "error": {"GCPSECRETS#/token/1", "unable to retrieve secret", func(t *testing.T) gcpSecretsApi { - return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { - gcpSecretsGetChecker(t, req) - return nil, fmt.Errorf("unable to retrieve secret") - }) - }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), + "error": { + func() *config.ParsedTokenConfig { + // "GCPSECRETS#/token/1" + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "unable to retrieve secret", func(t *testing.T) mockGcpSecretsApi { + return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { + gcpSecretsGetChecker(t, req) + return nil, fmt.Errorf("unable to retrieve secret") + }) + }, }, "found but empty": { - "GCPSECRETS#/token/1", + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "", - func(t *testing.T) gcpSecretsApi { + func(t *testing.T) mockGcpSecretsApi { return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { gcpSecretsGetChecker(t, req) return &gcpsecretspb.AccessSecretVersionResponse{}, nil }) }, - config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), }, } for name, tt := range tests { @@ -126,18 +158,17 @@ func Test_GetGcpSecretVarHappy(t *testing.T) { defer fixture.delete(fixture.name) os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fixture.name) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - impl, err := NewGcpSecrets(context.TODO(), log.New(io.Discard)) + impl, err := store.NewGcpSecrets(context.TODO(), log.New(io.Discard)) if err != nil { t.Errorf(testutils.TestPhrase, err.Error(), nil) } - impl.svc = tt.mockClient(t) - impl.close = func() error { return nil } - impl.SetToken(token) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + + impl.SetToken(tt.token()) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { diff --git a/internal/store/hashivault.go b/internal/store/hashivault.go index 048039b..558c9b2 100644 --- a/internal/store/hashivault.go +++ b/internal/store/hashivault.go @@ -8,17 +8,17 @@ import ( "strconv" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" vault "github.com/hashicorp/vault/api" auth "github.com/hashicorp/vault/api/auth/aws" ) -// vaultHelper provides a broken up string -type vaultHelper struct { - path string - token string +// HashiVaultHelper provides a broken up string +type HashiVaultHelper struct { + Path string + Token string } type hashiVaultApi interface { @@ -52,8 +52,8 @@ func NewVaultStore(ctx context.Context, token *config.ParsedTokenConfig, logger } config := vault.DefaultConfig() - vt := splitToken(token.StoreToken()) - imp.strippedToken = vt.token + vt := SplitHashiVaultToken(token.StoreToken()) + imp.strippedToken = vt.Token client, err := vault.NewClient(config) if err != nil { return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) @@ -66,10 +66,14 @@ func NewVaultStore(ctx context.Context, token *config.ParsedTokenConfig, logger } client = awsclient } - imp.svc = client.KVv2(vt.path) + imp.svc = client.KVv2(vt.Path) return imp, nil } +func (s *VaultStore) WithSvc(svc hashiVaultApi) { + s.svc = svc +} + // newVaultStoreWithAWSAuthIAM returns an initialised client with AWSIAMAuth // EC2 auth type is not supported currently func newVaultStoreWithAWSAuthIAM(client *vault.Client, role string) (*vault.Client, error) { @@ -107,7 +111,7 @@ func (imp *VaultStore) SetToken(token *config.ParsedTokenConfig) {} // getTokenValue implements the underlying techonology // token retrieval and returns a stringified version // of the secret -func (imp *VaultStore) Token() (string, error) { +func (imp *VaultStore) Value() (string, error) { imp.logger.Info("%s", "Concrete implementation HashiVault") imp.logger.Info("Getting Secret: %s", imp.token) @@ -145,14 +149,14 @@ func (imp *VaultStore) getSecret(ctx context.Context, token string, version stri return imp.svc.Get(ctx, token) } -func splitToken(token string) vaultHelper { - vh := vaultHelper{} +func SplitHashiVaultToken(token string) HashiVaultHelper { + vh := HashiVaultHelper{} // split token to extract the mount path s := strings.Split(strings.TrimPrefix(token, "/"), "___") // grab token and trim prefix if slash - vh.token = strings.TrimPrefix(strings.Join(s[1:], ""), "/") + vh.Token = strings.TrimPrefix(strings.Join(s[1:], ""), "/") // assign mount path as extracted from input token - vh.path = s[0] + vh.Path = s[0] return vh } diff --git a/internal/store/hashivault_test.go b/internal/store/hashivault_test.go index 71d010a..8c9aed8 100644 --- a/internal/store/hashivault_test.go +++ b/internal/store/hashivault_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -10,30 +10,65 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" vault "github.com/hashicorp/vault/api" ) func TestMountPathExtract(t *testing.T) { ttests := map[string]struct { - token string - tokenSeparator string - keySeparator string - expect string + token func() *config.ParsedTokenConfig + expect string }{ - "without leading slash": {"VAULT://secret___/demo/configmanager", "://", "|", "secret"}, - "with leading slash": {"VAULT:///secret___/demo/configmanager", "://", "|", "secret"}, - "with underscore in path name": {"VAULT://_secret___/demo/configmanager", "://", "|", "_secret"}, - "with double underscore in path name": {"VAULT://__secret___/demo/configmanager", "://", "|", "__secret"}, - "with multiple paths in mountpath": {"VAULT://secret/bar/path___/demo/configmanager", "://", "|", "secret/bar/path"}, + "without leading slash": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/demo/configmanager" + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "secret"}, + "with leading slash": { + func() *config.ParsedTokenConfig { + // "VAULT:///secret___/demo/configmanager", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "secret"}, + "with underscore in path name": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("_secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "_secret"}, + "with double underscore in path name": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("__secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "__secret"}, + "with multiple paths in mountpath": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret/bar/path___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "secret/bar/path"}, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *config.NewConfig().WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - got := splitToken(token.StoreToken()) - if got.path != tt.expect { + got := store.SplitHashiVaultToken(tt.token().StoreToken()) + if got.Path != tt.expect { t.Errorf("got %q, expected %q", got, tt.expect) } }) @@ -54,17 +89,23 @@ func (m mockVaultApi) GetVersion(ctx context.Context, secretPath string, version } func TestVaultScenarios(t *testing.T) { - t.Parallel() ttests := map[string]struct { - token string - conf *config.GenVarsConfig + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) hashiVaultApi + mockClient func(t *testing.T) mockVaultApi setupEnv func() func() }{ - "happy return": {"VAULT://secret___/foo", config.NewConfig(), `{"foo":"test2130-9sd-0ds"}`, - func(t *testing.T) hashiVaultApi { + "happy return": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/foo") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `{"foo":"test2130-9sd-0ds"}`, + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -84,8 +125,17 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "incorrect json": {"VAULT://secret___/foo", config.NewConfig(), `json: unsupported type: func() error`, - func(t *testing.T) hashiVaultApi { + "incorrect json": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/foo", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/foo") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `json: unsupported type: func() error`, + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -106,10 +156,15 @@ func TestVaultScenarios(t *testing.T) { }, }, "another return": { - "VAULT://secret/engine1___/some/other/foo2", - config.NewConfig(), + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret/engine1___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, `{"foo1":"test2130-9sd-0ds","foo2":"dsfsdf3454456"}`, - func(t *testing.T) hashiVaultApi { + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -130,8 +185,17 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "not found": {"VAULT://secret___/foo", config.NewConfig(), `secret not found`, - func(t *testing.T) hashiVaultApi { + "not found": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/foo", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/foo") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `secret not found`, + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -149,8 +213,17 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "403": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `client 403`, - func(t *testing.T) hashiVaultApi { + "403": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `client 403`, + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -168,18 +241,29 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "found but empty": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `{}`, func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - return &vault.KVSecret{Data: m}, nil - } - return mv - }, + "found but empty": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + // config.NewConfig(), + `{}`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + } + m := make(map[string]interface{}) + return &vault.KVSecret{Data: m}, nil + } + return mv + }, func() func() { os.Setenv("VAULT_TOKEN", "129378y1231283") return func() { @@ -187,17 +271,26 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "found but nil returned": {"VAULT://secret___/some/other/foo2", config.NewConfig(), "", func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return &vault.KVSecret{Data: nil}, nil - } - return mv - }, + "found but nil returned": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "", + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return &vault.KVSecret{Data: nil}, nil + } + return mv + }, func() func() { os.Setenv("VAULT_TOKEN", "129378y1231283") return func() { @@ -205,19 +298,29 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "version provided correctly": {"VAULT://secret___/some/other/foo2[version=1]", config.NewConfig(), `{"foo2":"dsfsdf3454456"}`, func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - m["foo2"] = "dsfsdf3454456" - return &vault.KVSecret{Data: m}, nil - } - return mv - }, + "version provided correctly": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2[version=1]", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1") + return tkn + }, + `{"foo2":"dsfsdf3454456"}`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + m := make(map[string]interface{}) + m["foo2"] = "dsfsdf3454456" + return &vault.KVSecret{Data: m}, nil + } + return mv + }, func() func() { os.Setenv("VAULT_TOKEN", "129378y1231283") return func() { @@ -225,17 +328,27 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "version provided but unable to parse": {"VAULT://secret___/some/other/foo2[version=1a]", config.NewConfig(), "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return nil, nil - } - return mv - }, + "version provided but unable to parse": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2[version=1a]", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1a") + return tkn + }, + "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return nil, nil + } + return mv + }, func() func() { os.Setenv("VAULT_TOKEN", "129378y1231283") return func() { @@ -244,11 +357,16 @@ func TestVaultScenarios(t *testing.T) { }, }, "vault rate limit incorrect": { - "VAULT://secret___/some/other/foo2", - config.NewConfig(), + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, `error encountered setting up default configuration: VAULT_RATE_LIMIT was provided but incorrectly formatted failed to initialize the client`, - func(t *testing.T) hashiVaultApi { + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -273,9 +391,8 @@ failed to initialize the client`, t.Run(name, func(t *testing.T) { tearDown := tt.setupEnv() defer tearDown() - token, _ := config.NewParsedTokenConfig(tt.token, *tt.conf) - impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewVaultStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { if err.Error() != tt.expect { t.Fatalf("failed to init hashivault, %v", err.Error()) @@ -283,8 +400,8 @@ failed to initialize the client`, return } - impl.svc = tt.mockClient(t) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) @@ -300,17 +417,22 @@ failed to initialize the client`, func TestAwsIamAuth(t *testing.T) { ttests := map[string]struct { - token string - conf *config.GenVarsConfig + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) hashiVaultApi + mockClient func(t *testing.T) mockVaultApi mockHanlder func(t *testing.T) http.Handler setupEnv func(addr string) func() }{ "aws_iam auth no role specified": { - "VAULT://secret___/some/other/foo2[version:1]", config.NewConfig(), + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1") + return tkn + }, "role provided is empty, EC2 auth not supported", - func(t *testing.T) hashiVaultApi { + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -336,14 +458,20 @@ func TestAwsIamAuth(t *testing.T) { }, }, "aws_iam auth incorrectly formatted request": { - "VAULT://secret___/some/other/foo2[version=1,iam_role=not_a_role]", config.NewConfig(), + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1,iam_role=not_a_role") + return tkn + }, `unable to login to AWS auth method: unable to log in to auth method: unable to log in with AWS auth: Error making API request. URL: PUT %s/v1/auth/aws/login Code: 400. Raw Message: incorrect values supplied. failed to initialize the client`, - func(t *testing.T) hashiVaultApi { + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -377,16 +505,23 @@ incorrect values supplied. failed to initialize the client`, }, }, "aws_iam auth success": { - "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("iam_role=arn:aws:iam::1111111:role/i-orchestration") + return tkn + }, + // `{"foo2":"dsfsdf3454456"}`, - func(t *testing.T) hashiVaultApi { + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() if secretPath != "some/other/foo2" { t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) } - m := make(map[string]interface{}) + m := make(map[string]any) m["foo2"] = "dsfsdf3454456" return &vault.KVSecret{Data: m}, nil } @@ -414,9 +549,15 @@ incorrect values supplied. failed to initialize the client`, }, }, "aws_iam auth no token returned": { - "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("iam_role=arn:aws:iam::1111111:role/i-orchestration") + return tkn + }, `unable to login to AWS auth method: response did not return ClientToken, client token not set. failed to initialize the client`, - func(t *testing.T) hashiVaultApi { + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -432,7 +573,6 @@ incorrect values supplied. failed to initialize the client`, func(t *testing.T) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Write([]byte(`{"auth":{}}`)) }) @@ -458,9 +598,7 @@ incorrect values supplied. failed to initialize the client`, ts := httptest.NewServer(tt.mockHanlder(t)) tearDown := tt.setupEnv(ts.URL) defer tearDown() - token, _ := config.NewParsedTokenConfig(tt.token, *tt.conf) - - impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewVaultStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { // WHAT A CRAP way to do this... if err.Error() != strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0] { @@ -470,8 +608,8 @@ incorrect values supplied. failed to initialize the client`, return } - impl.svc = tt.mockClient(t) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/paramstore.go b/internal/store/paramstore.go index 72b43a8..aa45ace 100644 --- a/internal/store/paramstore.go +++ b/internal/store/paramstore.go @@ -3,8 +3,8 @@ package store import ( "context" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/aws/aws-sdk-go-v2/aws" awsConf "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ssm" @@ -41,6 +41,10 @@ func NewParamStore(ctx context.Context, logger log.ILogger) (*ParamStore, error) }, nil } +func (s *ParamStore) WithSvc(svc paramStoreApi) { + s.svc = svc +} + func (imp *ParamStore) SetToken(token *config.ParsedTokenConfig) { storeConf := &ParamStrConfig{} _ = token.ParseMetadata(storeConf) @@ -48,7 +52,7 @@ func (imp *ParamStore) SetToken(token *config.ParsedTokenConfig) { imp.config = storeConf } -func (imp *ParamStore) Token() (string, error) { +func (imp *ParamStore) Value() (string, error) { imp.logger.Info("%s", "Concrete implementation ParameterStore") imp.logger.Info("ParamStore Token: %s", imp.token.String()) diff --git a/internal/store/paramstore_test.go b/internal/store/paramstore_test.go index 19c027a..8fc11d4 100644 --- a/internal/store/paramstore_test.go +++ b/internal/store/paramstore_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -7,18 +7,14 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" ) -// var ( -// tsuccessParam = "someVal" -// tsuccessObj map[string]string = map[string]string{"AWSPARAMSTR#/token/1": "someVal"} -// ) - type mockParamApi func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) func (m mockParamApi) GetParameter(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { @@ -44,76 +40,104 @@ func awsParamtStoreCommonGetChecker(t *testing.T, params *ssm.GetParameterInput) } func Test_GetParamStore(t *testing.T) { - t.Parallel() - var ( tsuccessParam = "someVal" // tsuccessObj map[string]string = map[string]string{"AWSPARAMSTR#/token/1": "someVal"} ) tests := map[string]struct { - token string - keySeparator string - tokenSeparator string - expect string - mockClient func(t *testing.T) paramStoreApi - config *config.GenVarsConfig + token func() *config.ParsedTokenConfig + expect string + mockClient func(t *testing.T) mockParamApi }{ - "successVal": {"AWSPARAMSTR#/token/1", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) - return &ssm.GetParameterOutput{ - Parameter: &types.Parameter{Value: &tsuccessParam}, - }, nil - }) - }, config.NewConfig(), + "successVal": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/demo/configmanager" + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + tsuccessParam, func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) + return &ssm.GetParameterOutput{ + Parameter: &types.Parameter{Value: &tsuccessParam}, + }, nil + }) + }, }, - "successVal with keyseparator": {"AWSPARAMSTR#/token/1|somekey", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) + "successVal with keyseparator": { + func() *config.ParsedTokenConfig { + // "AWSPARAMSTR#/token/1|somekey", + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, + tsuccessParam, func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) - if strings.Contains(*params.Name, "|somekey") { - t.Errorf("incorrectly stripped key separator") - } + if strings.Contains(*params.Name, "|somekey") { + t.Errorf("incorrectly stripped key separator") + } - return &ssm.GetParameterOutput{ - Parameter: &types.Parameter{Value: &tsuccessParam}, - }, nil - }) - }, config.NewConfig(), + return &ssm.GetParameterOutput{ + Parameter: &types.Parameter{Value: &tsuccessParam}, + }, nil + }) + }, }, - "errored": {"AWSPARAMSTR#/token/1", "|", "#", "unable to retrieve", func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) - return nil, fmt.Errorf("unable to retrieve") - }) - }, config.NewConfig(), + "errored": { + func() *config.ParsedTokenConfig { + // "AWSPARAMSTR#/token/1", + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "unable to retrieve", func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) + return nil, fmt.Errorf("unable to retrieve") + }) + }, }, - "nil to empty": {"AWSPARAMSTR#/token/1", "|", "#", "", func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) - return &ssm.GetParameterOutput{ - Parameter: &types.Parameter{Value: nil}, - }, nil - }) - }, config.NewConfig(), + "nil to empty": { + func() *config.ParsedTokenConfig { + // "AWSPARAMSTR#/token/1", + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "", func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) + return &ssm.GetParameterOutput{ + Parameter: &types.Parameter{Value: nil}, + }, nil + }) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - - impl, err := NewParamStore(context.TODO(), log.New(io.Discard)) + impl, err := store.NewParamStore(context.TODO(), log.New(io.Discard)) if err != nil { t.Errorf(testutils.TestPhrase, err.Error(), nil) } - impl.svc = tt.mockClient(t) - impl.SetToken(token) - got, err := impl.Token() + impl.WithSvc(tt.mockClient(t)) + impl.SetToken(tt.token()) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/secretsmanager.go b/internal/store/secretsmanager.go index 6b0f7a2..6744d8a 100644 --- a/internal/store/secretsmanager.go +++ b/internal/store/secretsmanager.go @@ -3,8 +3,8 @@ package store import ( "context" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/aws/aws-sdk-go-v2/aws" awsconf "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" @@ -42,6 +42,10 @@ func NewSecretsMgr(ctx context.Context, logger log.ILogger) (*SecretsMgr, error) } +func (s *SecretsMgr) WithSvc(svc secretsMgrApi) { + s.svc = svc +} + func (imp *SecretsMgr) SetToken(token *config.ParsedTokenConfig) { storeConf := &SecretsMgrConfig{} if err := token.ParseMetadata(storeConf); err != nil { @@ -51,7 +55,7 @@ func (imp *SecretsMgr) SetToken(token *config.ParsedTokenConfig) { imp.config = storeConf } -func (imp *SecretsMgr) Token() (string, error) { +func (imp *SecretsMgr) Value() (string, error) { imp.logger.Info("Concrete implementation SecretsManager") imp.logger.Debug("SecretsManager Token: %s", imp.token.String()) diff --git a/internal/store/secretsmanager_test.go b/internal/store/secretsmanager_test.go index 3b29dd2..870bb75 100644 --- a/internal/store/secretsmanager_test.go +++ b/internal/store/secretsmanager_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -7,9 +7,10 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" ) @@ -34,77 +35,111 @@ func awsSecretsMgrGetChecker(t *testing.T, params *secretsmanager.GetSecretValue } func Test_GetSecretMgr(t *testing.T) { - t.Parallel() tsuccessSecret := "dsgkbdsf" tests := map[string]struct { - token string - keySeparator string - tokenSeparator string - expect string - mockClient func(t *testing.T) secretsMgrApi - config *config.GenVarsConfig + token func() *config.ParsedTokenConfig + expect string + mockClient func(t *testing.T) mockSecretsApi }{ - "success": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretString: &tsuccessSecret, - }, nil - }) - }, config.NewConfig(), + "success": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, tsuccessSecret, func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretString: &tsuccessSecret, + }, nil + }) + }, }, - "success with version": {"AWSSECRETS#/token/1[version=123]", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretString: &tsuccessSecret, - }, nil - }) - }, config.NewConfig(), + "success with version": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version=123") + return tkn + }, + tsuccessSecret, func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretString: &tsuccessSecret, + }, nil + }) + }, }, - "success with binary": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretBinary: []byte(tsuccessSecret), - }, nil - }) - }, config.NewConfig(), + "success with binary": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + tsuccessSecret, func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretBinary: []byte(tsuccessSecret), + }, nil + }) + }, }, - "errored": {"AWSSECRETS#/token/1", "|", "#", "unable to retrieve secret", func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return nil, fmt.Errorf("unable to retrieve secret") - }) - }, config.NewConfig(), + "errored": { + func() *config.ParsedTokenConfig { + // "AWSSECRETS#/token/1", "|", "#", + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "unable to retrieve secret", func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return nil, fmt.Errorf("unable to retrieve secret") + }) + }, }, - "ok but empty": {"AWSSECRETS#/token/1", "|", "#", "", func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretString: nil, - }, nil - }) - }, config.NewConfig(), + "ok but empty": { + func() *config.ParsedTokenConfig { + // "AWSSECRETS#/token/1", "|", "#", + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version=123") + return tkn + }, + "", func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretString: nil, + }, nil + }) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + impl, _ := store.NewSecretsMgr(context.TODO(), log.New(io.Discard)) + impl.WithSvc(tt.mockClient(t)) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - - impl, _ := NewSecretsMgr(context.TODO(), log.New(io.Discard)) - impl.svc = tt.mockClient(t) - - impl.SetToken(token) - got, err := impl.Token() + impl.SetToken(tt.token()) + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/store.go b/internal/store/store.go index 42adf5b..eceeb45 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -3,7 +3,7 @@ package store import ( "errors" - "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/config" ) const implementationNetworkErr string = "implementation %s error: %v for token: %s" @@ -20,6 +20,8 @@ var ( // // Defined on the package for easier re-use across the program type Strategy interface { - Token() (s string, e error) + // Value retrieves the underlying value for the token + Value() (s string, e error) + // SetToken SetToken(s *config.ParsedTokenConfig) } diff --git a/internal/strategy/strategy.go b/internal/strategy/strategy.go index a5b4981..ac19a30 100644 --- a/internal/strategy/strategy.go +++ b/internal/strategy/strategy.go @@ -1,6 +1,4 @@ -// Package strategy is a strategy pattern wrapper around the store implementations -// -// NOTE: this may be refactored out into the store package directly +// Package strategy is a factory method wrapper around the backing store implementations package strategy import ( @@ -9,9 +7,9 @@ import ( "fmt" "sync" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" ) var ErrTokenInvalid = errors.New("invalid token - cannot get prefix") @@ -22,46 +20,16 @@ type StrategyFunc func(ctx context.Context, token *config.ParsedTokenConfig) (st // StrategyFuncMap type StrategyFuncMap map[config.ImplementationPrefix]StrategyFunc -func defaultStrategyFuncMap(logger log.ILogger) map[config.ImplementationPrefix]StrategyFunc { - return map[config.ImplementationPrefix]StrategyFunc{ - config.AzTableStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewAzTableStore(ctx, token, logger) - }, - config.AzAppConfigPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewAzAppConf(ctx, token, logger) - }, - config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewGcpSecrets(ctx, logger) - }, - config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewSecretsMgr(ctx, logger) - }, - config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewParamStore(ctx, logger) - }, - config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewKvScrtStore(ctx, token, logger) - }, - config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewVaultStore(ctx, token, logger) - }, - } -} - -type strategyFnMap struct { - mu sync.Mutex - funcMap StrategyFuncMap -} -type RetrieveStrategy struct { - implementation store.Strategy +type Strategy struct { config config.GenVarsConfig strategyFuncMap strategyFnMap } -type Opts func(*RetrieveStrategy) + +type Opts func(*Strategy) // New -func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *RetrieveStrategy { - rs := &RetrieveStrategy{ +func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *Strategy { + rs := &Strategy{ config: config, strategyFuncMap: strategyFnMap{mu: sync.Mutex{}, funcMap: defaultStrategyFuncMap(logger)}, } @@ -78,25 +46,36 @@ func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *Retriev // Mainly used for testing // NOTE: this may lead to eventual optional configurations by users func WithStrategyFuncMap(funcMap StrategyFuncMap) Opts { - return func(rs *RetrieveStrategy) { + return func(rs *Strategy) { + rs.strategyFuncMap.mu.Lock() + defer rs.strategyFuncMap.mu.Unlock() for prefix, implementation := range funcMap { - rs.strategyFuncMap.mu.Lock() - defer rs.strategyFuncMap.mu.Unlock() rs.strategyFuncMap.funcMap[config.ImplementationPrefix(prefix)] = implementation } } } -func (rs *RetrieveStrategy) setImplementation(strategy store.Strategy) { - rs.implementation = strategy -} +// GetImplementation is a factory method returning the concrete implementation for the retrieval of the token value +// i.e. facilitating the exchange of the supplied token for the underlying value +func (rs *Strategy) GetImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + if token == nil { + return nil, fmt.Errorf("unable to get prefix, %w", ErrTokenInvalid) + } -func (rs *RetrieveStrategy) setTokenVal(s *config.ParsedTokenConfig) { - rs.implementation.SetToken(s) + if store, found := rs.strategyFuncMap.funcMap[token.Prefix()]; found { + return store(ctx, token) + } + + return nil, fmt.Errorf("implementation not found for input string: %s", token) } -func (rs *RetrieveStrategy) getTokenValue() (string, error) { - return rs.implementation.Token() +func ExchangeToken(s store.Strategy, token *config.ParsedTokenConfig) *TokenResponse { + cr := &TokenResponse{} + cr.Err = nil + cr.key = token + s.SetToken(token) + cr.value, cr.Err = s.Value() + return cr } type TokenResponse struct { @@ -113,31 +92,33 @@ func (tr *TokenResponse) Value() string { return tr.value } -// retrieveSpecificCh wraps around the specific strategy implementation -// and publishes results to a channel -func (rs *RetrieveStrategy) RetrieveByToken(ctx context.Context, impl store.Strategy, tokenConf *config.ParsedTokenConfig) *TokenResponse { - cr := &TokenResponse{} - cr.Err = nil - cr.key = tokenConf - rs.setImplementation(impl) - rs.setTokenVal(tokenConf) - s, err := rs.getTokenValue() - if err != nil { - cr.Err = err - return cr +func defaultStrategyFuncMap(logger log.ILogger) StrategyFuncMap { + return map[config.ImplementationPrefix]StrategyFunc{ + config.AzTableStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewAzTableStore(ctx, token, logger) + }, + config.AzAppConfigPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewAzAppConf(ctx, token, logger) + }, + config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewGcpSecrets(ctx, logger) + }, + config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewSecretsMgr(ctx, logger) + }, + config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewParamStore(ctx, logger) + }, + config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewKvScrtStore(ctx, token, logger) + }, + config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewVaultStore(ctx, token, logger) + }, } - cr.value = s - return cr } -func (rs *RetrieveStrategy) SelectImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - if token == nil { - return nil, fmt.Errorf("unable to get prefix, %w", ErrTokenInvalid) - } - - if store, found := rs.strategyFuncMap.funcMap[token.Prefix()]; found { - return store(ctx, token) - } - - return nil, fmt.Errorf("implementation not found for input string: %s", token) +type strategyFnMap struct { + mu sync.Mutex + funcMap StrategyFuncMap } diff --git a/internal/strategy/strategy_test.go b/internal/strategy/strategy_test.go index 93c4c6c..acae5e1 100644 --- a/internal/strategy/strategy_test.go +++ b/internal/strategy/strategy_test.go @@ -7,11 +7,11 @@ import ( "os" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - log "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/strategy" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + log "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/strategy" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/go-test/deep" ) @@ -23,7 +23,7 @@ type mockGenerate struct { func (m mockGenerate) SetToken(s *config.ParsedTokenConfig) { } -func (m mockGenerate) Token() (s string, e error) { +func (m mockGenerate) Value() (s string, e error) { return m.value, m.err } @@ -41,20 +41,21 @@ var TEST_GCP_CREDS = []byte(`{ }`) func Test_Strategy_Retrieve_succeeds(t *testing.T) { - t.Parallel() ttests := map[string]struct { - impl func(t *testing.T) store.Strategy - config *config.GenVarsConfig - token string - expect string + impl func(t *testing.T) store.Strategy + config *config.GenVarsConfig + token string + expect string + impPrefix config.ImplementationPrefix }{ "with mocked implementation AZTABLESTORAGE": { func(t *testing.T) store.Strategy { - return &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} + return &mockGenerate{"mountPath/token", "bar", nil} }, - config.NewConfig().WithOutputPath("stdout").WithTokenSeparator("://"), - "AZTABLESTORE://mountPath/token", + config.NewConfig().WithOutputPath("stdout"), + "mountPath/token", "bar", + config.AzTableStorePrefix, }, // "error in retrieval": { // func(t *testing.T) store.Strategy { @@ -68,24 +69,23 @@ func Test_Strategy_Retrieve_succeeds(t *testing.T) { } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - rs := strategy.New(*tt.config, log.New(io.Discard)) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - got := rs.RetrieveByToken(context.TODO(), tt.impl(t), token) + token, _ := config.NewToken(tt.impPrefix, *tt.config) + token.WithSanitizedToken(tt.token) + got := strategy.ExchangeToken(tt.impl(t), token) if got.Err != nil { t.Errorf(testutils.TestPhraseWithContext, "Token response errored", got.Err.Error(), tt.expect) } if got.Value() != tt.expect { t.Errorf(testutils.TestPhraseWithContext, "Value not correct", got.Value(), tt.expect) } - if got.Key().String() != tt.token { - t.Errorf(testutils.TestPhraseWithContext, "INcorrect Token returned in Key", got.Key().String(), tt.token) + if got.Key().StoreToken() != tt.token { + t.Errorf(testutils.TestPhraseWithContext, "Incorrect Token returned in Key", got.Key().StoreToken(), tt.token) } }) } } func Test_CustomStrategyFuncMap_add_own(t *testing.T) { - t.Parallel() ttests := map[string]struct { }{ @@ -95,7 +95,8 @@ func Test_CustomStrategyFuncMap_add_own(t *testing.T) { t.Run(name, func(t *testing.T) { called := 0 genVarsConf := config.NewConfig() - token, _ := config.NewParsedTokenConfig("AZTABLESTORE://mountPath/token", *genVarsConf) + token, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig()) + token.WithSanitizedToken("mountPath/token") var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { m := &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} @@ -105,8 +106,8 @@ func Test_CustomStrategyFuncMap_add_own(t *testing.T) { s := strategy.New(*genVarsConf, log.New(io.Discard), strategy.WithStrategyFuncMap(strategy.StrategyFuncMap{config.AzTableStorePrefix: custFunc})) - store, _ := s.SelectImplementation(context.TODO(), token) - _ = s.RetrieveByToken(context.TODO(), store, token) + store, _ := s.GetImplementation(context.TODO(), token) + _ = strategy.ExchangeToken(store, token) if called != 1 { t.Errorf(testutils.TestPhraseWithContext, "custom func not called", called, 1) @@ -123,16 +124,18 @@ func Test_SelectImpl_With(t *testing.T) { config *config.GenVarsConfig expect func() store.Strategy expErr error + impPrefix config.ImplementationPrefix }{ "unknown": { func() func() { return func() { } }, - "UNKNOWN#foo/bar", + "foo/bar", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { return nil }, fmt.Errorf("implementation not found for input string: UNKNOWN#foo/bar"), + config.UnknownPrefix, }, "success AZTABLESTORE": { func() func() { @@ -141,14 +144,17 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AZTABLESTORE#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("AZTABLESTORE#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") + s, _ := store.NewAzTableStore(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.AzTableStorePrefix, }, "success AWSPARAMSTR": { func() func() { @@ -158,13 +164,14 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AWSPARAMSTR#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { s, _ := store.NewParamStore(context.TODO(), log.New(io.Discard)) return s }, nil, + config.ParamStorePrefix, }, "success AWSSECRETS": { func() func() { @@ -174,13 +181,14 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AWSSECRETS#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { s, _ := store.NewSecretsMgr(context.TODO(), log.New(io.Discard)) return s }, nil, + config.SecretMgrPrefix, }, "success AZKVSECRET": { func() func() { @@ -190,14 +198,16 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AZKVSECRET#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("AZKVSECRET#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") s, _ := store.NewKvScrtStore(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.AzKeyVaultSecretsPrefix, }, "success AZAPPCONF": { func() func() { @@ -205,14 +215,16 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AZAPPCONF#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("AZAPPCONF#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") s, _ := store.NewAzAppConf(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.AzAppConfigPrefix, }, "success VAULT": { func() func() { @@ -221,14 +233,16 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "VAULT#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("VAULT#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") s, _ := store.NewVaultStore(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.HashicorpVaultPrefix, }, "success GCPSECRETS": { func() func() { @@ -240,32 +254,15 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "GCPSECRETS#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { s, _ := store.NewGcpSecrets(context.TODO(), log.New(io.Discard)) return s }, nil, + config.GcpSecretsPrefix, }, - // "default Error": { - // func() func() { - // os.Setenv("AWS_ACCESS_KEY", "AAAAAAAAAAAAAAA") - // os.Setenv("AWS_SECRET_ACCESS_KEY", "00000000000000000000111111111") - // return func() { - // os.Clearenv() - // } - // }, - // context.TODO(), - // UnknownPrefix, "AWSPARAMSTR://foo/bar", (&GenVarsConfig{}).WithKeySeparator("|").WithTokenSeparator("://"), - // func(t *testing.T, ctx context.Context, conf GenVarsConfig) genVarsStrategy { - // imp, err := NewParamStore(ctx) - // if err != nil { - // t.Errorf(testutils.TestPhraseWithContext, "init impl error", err.Error(), nil) - // } - // return imp - // }, - // }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { @@ -273,8 +270,9 @@ func Test_SelectImpl_With(t *testing.T) { defer tearDown() want := tt.expect() rs := strategy.New(*tt.config, log.New(io.Discard)) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - got, err := rs.SelectImplementation(context.TODO(), token) + token, _ := config.NewToken(tt.impPrefix, *tt.config) + token.WithSanitizedToken(tt.token) + got, err := rs.GetImplementation(context.TODO(), token) if err != nil { if err.Error() != tt.expErr.Error() { diff --git a/pkg/.gitkeep b/pkg/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go deleted file mode 100644 index 4d8bd69..0000000 --- a/pkg/generator/generator.go +++ /dev/null @@ -1,266 +0,0 @@ -package generator - -import ( - "context" - "encoding/json" - "fmt" - "io" - "strconv" - "sync" - - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/strategy" - "github.com/spyzhov/ajson" -) - -// GenVars is the main struct holding the -// strategy patterns iface -// any initialised config if overridded with withers -// as well as the final outString and the initial rawMap -// which wil be passed in a loop into a goroutine to perform the -// relevant strategy network calls to the config store implementations -type GenVars struct { - Logger log.ILogger - strategy strategy.StrategyFuncMap - ctx context.Context - config config.GenVarsConfig - // rawMap is the internal object that holds the values - // of original token => retrieved value - decrypted in plain text - // with a mutex RW locker - rawMap tokenMapSafe //ParsedMap -} - -type Opts func(*GenVars) - -// NewGenerator returns a new instance of Generator -// with a default strategy pattern wil be overwritten -// during the first run of a found tokens map -func NewGenerator(ctx context.Context, opts ...Opts) *GenVars { - // defaultStrategy := NewDefatultStrategy() - return newGenVars(ctx, opts...) -} - -func newGenVars(ctx context.Context, opts ...Opts) *GenVars { - m := make(ParsedMap) - conf := config.NewConfig() - g := &GenVars{ - Logger: log.New(io.Discard), - rawMap: tokenMapSafe{ - tokenMap: m, - mu: &sync.Mutex{}, - }, - ctx: ctx, - // return using default config - config: *conf, - } - g.strategy = nil - - // now apply additional opts - for _, o := range opts { - o(g) - } - - return g -} - -// WithStrategyMap -// -// Adds addtional funcs for storageRetrieval used for testing only -func (c *GenVars) WithStrategyMap(sm strategy.StrategyFuncMap) *GenVars { - c.strategy = sm - return c -} - -// WithConfig uses custom config -func (c *GenVars) WithConfig(cfg *config.GenVarsConfig) *GenVars { - // backwards compatibility - if cfg != nil { - c.config = *cfg - } - return c -} - -// WithContext uses caller passed context -func (c *GenVars) WithContext(ctx context.Context) *GenVars { - c.ctx = ctx - return c -} - -// Config gets Config on the GenVars -func (c *GenVars) Config() *config.GenVarsConfig { - return &c.config -} - -// ParsedMap is the internal working object definition and -// the return type if results are not flushed to file -type ParsedMap map[string]any - -func (pm ParsedMap) MapKeys() (keys []string) { - for k := range pm { - keys = append(keys, k) - } - return -} - -type tokenMapSafe struct { - mu *sync.Mutex - tokenMap ParsedMap -} - -func (tms *tokenMapSafe) getTokenMap() ParsedMap { - tms.mu.Lock() - defer tms.mu.Unlock() - return tms.tokenMap -} - -func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { - tms.mu.Lock() - defer tms.mu.Unlock() - // NOTE: still use the metadata in the key - // there could be different versions / labels for the same token and hence different values - // However the JSONpath look up - tms.tokenMap[key.String()] = keySeparatorLookup(key, val) -} - -type rawTokenMap struct { - mu sync.Mutex - tokenMap map[string]*config.ParsedTokenConfig -} - -func newRawTokenMap() *rawTokenMap { - return &rawTokenMap{mu: sync.Mutex{}, tokenMap: map[string]*config.ParsedTokenConfig{}} -} - -func (rtm *rawTokenMap) addToken(name string, parsedToken *config.ParsedTokenConfig) { - rtm.mu.Lock() - defer rtm.mu.Unlock() - rtm.tokenMap[name] = parsedToken -} - -func (rtm *rawTokenMap) mapOfToken() map[string]*config.ParsedTokenConfig { - rtm.mu.Lock() - defer rtm.mu.Unlock() - return rtm.tokenMap -} - -// Generate generates a k/v map of the tokens with their corresponding secret/paramstore values -// the standard pattern of a token should follow a path like string -func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { - - rtm := newRawTokenMap() - for _, token := range tokens { - // TODO: normalize tokens here potentially - // merge any tokens that only differ in keys lookup inside the object - parsedToken, err := config.NewParsedTokenConfig(token, c.config) - if err != nil { - c.Logger.Info(err.Error()) - continue - } - rtm.addToken(token, parsedToken) - } - // pass in default initialised retrieveStrategy - // input should be - if err := c.generate(rtm); err != nil { - return nil, err - } - return c.rawMap.getTokenMap(), nil -} - -// generate checks if any tokens found -// initiates groutines with fixed size channel map -// to capture responses and errors -// generates ParsedMap which includes -func (c *GenVars) generate(rawMap *rawTokenMap) error { - rtm := rawMap.mapOfToken() - if len(rtm) < 1 { - c.Logger.Debug("no replaceable tokens found in input") - return nil - } - - tokenCount := len(rtm) - outCh := make(chan *strategy.TokenResponse, tokenCount) - - // TODO: initialise the singleton serviceContainer - // pass into each goroutine - for _, parsedToken := range rtm { - token := parsedToken // safe closure capture - // take value from config allocation on a per iteration basis - go func() { - s := strategy.New(c.config, c.Logger, strategy.WithStrategyFuncMap(c.strategy)) - storeStrategy, err := s.SelectImplementation(c.ctx, token) - if err != nil { - outCh <- &strategy.TokenResponse{Err: err} - return - } - outCh <- s.RetrieveByToken(c.ctx, storeStrategy, token) - }() - } - - // Fan-in: receive results with pure select - received := 0 - for received < tokenCount { - select { - case cr := <-outCh: - if cr == nil { - continue // defensive (shouldn't happen) - } - c.Logger.Debug("cro: %+v", cr) - if cr.Err != nil { - c.Logger.Debug("cr.err %v, for token: %s", cr.Err, cr.Key()) - } else { - c.rawMap.addKeyVal(cr.Key(), cr.Value()) - } - received++ - case <-c.ctx.Done(): - c.Logger.Debug("context done: %v", c.ctx.Err()) - return c.ctx.Err() // propagate context error (cancel/timeout) - } - } - return nil -} - -// IsParsed will try to parse the return found string into -// map[string]string -// If found it will convert that to a map with all keys uppercased -// and any characters -func IsParsed(v any, trm ParsedMap) bool { - str := fmt.Sprint(v) - err := json.Unmarshal([]byte(str), &trm) - return err == nil -} - -// keySeparatorLookup checks if the key contains -// keySeparator character -// If it does contain one then it tries to parse -func keySeparatorLookup(key *config.ParsedTokenConfig, val string) string { - // key has separator - k := key.LookupKeys() - if k == "" { - // c.logger.Info("no keyseparator found") - return val - } - - keys, err := ajson.JSONPath([]byte(val), fmt.Sprintf("$..%s", k)) - if err != nil { - // c.logger.Debug("unable to parse as json object %v", err.Error()) - return val - } - - if len(keys) == 1 { - v := keys[0] - if v.Type() == ajson.String { - str, err := strconv.Unquote(fmt.Sprintf("%v", v)) - if err != nil { - // c.logger.Debug("unable to unquote value: %v returning as is", v) - return fmt.Sprintf("%v", v) - } - return str - } - - return fmt.Sprintf("%v", v) - } - - // c.logger.Info("no value found in json using path expression") - return "" -} diff --git a/pkg/generator/generator_test.go b/pkg/generator/generator_test.go deleted file mode 100644 index bfb91b9..0000000 --- a/pkg/generator/generator_test.go +++ /dev/null @@ -1,548 +0,0 @@ -package generator_test - -import ( - "bytes" - "context" - "fmt" - "testing" - - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/strategy" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" -) - -type mockGenerate struct { - inToken, value string - err error -} - -func (m *mockGenerate) SetToken(s *config.ParsedTokenConfig) { -} -func (m *mockGenerate) Token() (s string, e error) { - return m.value, m.err -} - -func Test_Generate(t *testing.T) { - - t.Run("succeeds with funcMap", func(t *testing.T) { - var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"UNKNOWN://mountPath/token", "bar", nil} - return m, nil - } - - g := generator.NewGenerator(context.TODO(), func(gv *generator.GenVars) { - gv.Logger = log.New(&bytes.Buffer{}) - }) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) - got, err := g.Generate([]string{"UNKNOWN://mountPath/token"}) - - if err != nil { - t.Fatal("errored on generate") - } - if len(got) != 1 { - t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 1) - } - }) - - t.Run("errors in retrieval and logs it out", func(t *testing.T) { - var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"UNKNOWN://mountPath/token", "bar", fmt.Errorf("failed to get value")} - return m, nil - } - - g := generator.NewGenerator(context.TODO()) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) - got, err := g.Generate([]string{"UNKNOWN://mountPath/token"}) - - if err != nil { - t.Fatal("errored on generate") - } - if len(got) != 0 { - t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) - } - }) - - t.Run("retrieves values correctly from a keylookup inside", func(t *testing.T) { - var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"token-unused", `{"foo":"bar","key1":{"key2":"val"}}`, nil} - return m, nil - } - - g := generator.NewGenerator(context.TODO()) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) - got, err := g.Generate([]string{"UNKNOWN://mountPath/token|key1.key2"}) - - if err != nil { - t.Fatal("errored on generate") - } - if len(got) != 1 { - t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) - } - if got["UNKNOWN://mountPath/token|key1.key2"] != "val" { - t.Errorf(testutils.TestPhraseWithContext, "incorrect value returned in parsedMap", got["UNKNOWN://mountPath/token|key1.key2"], "val") - } - }) -} - -func Test_generate_withKeys_lookup(t *testing.T) { - ttests := map[string]struct { - custFunc strategy.StrategyFunc - token string - expectVal string - }{ - "retrieves string value correctly from a keylookup inside": { - custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":"val"}}`, nil} - return m, nil - }, - token: "UNKNOWN://mountPath/token|key1.key2", - expectVal: "val", - }, - "retrieves number value correctly from a keylookup inside": { - custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":123}}`, nil} - return m, nil - }, - token: "UNKNOWN://mountPath/token|key1.key2", - expectVal: "123", - }, - "retrieves nothing as keylookup is incorrect": { - custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":123}}`, nil} - return m, nil - }, - token: "UNKNOWN://mountPath/token|noprop", - expectVal: "", - }, - "retrieves value as is due to incorrectly stored json in backing store": { - custFunc: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"token", `foo":"bar","key1":{"key2":123}}`, nil} - return m, nil - }, - token: "UNKNOWN://mountPath/token|noprop", - expectVal: `foo":"bar","key1":{"key2":123}}`, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - g := generator.NewGenerator(context.TODO()) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: tt.custFunc}) - got, err := g.Generate([]string{tt.token}) - - if err != nil { - t.Fatal("errored on generate") - } - if len(got) != 1 { - t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) - } - if got[tt.token] != tt.expectVal { - t.Errorf(testutils.TestPhraseWithContext, "incorrect value returned in parsedMap", got[tt.token], tt.expectVal) - } - }) - } -} - -func Test_IsParsed(t *testing.T) { - ttests := map[string]struct { - val any - isParsed bool - }{ - "not parseable": { - `notparseable`, false, - }, - "one level parseable": { - `{"parseable":"foo"}`, true, - }, - "incorrect JSON": { - `parseable":"foo"}`, false, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - typ := generator.ParsedMap{} - got := generator.IsParsed(tt.val, typ) - if got != tt.isParsed { - t.Errorf(testutils.TestPhraseWithContext, "unexpected IsParsed", got, tt.isParsed) - } - }) - } -} - -// import ( -// "context" -// "fmt" -// "strings" -// "testing" - -// "github.com/DevLabFoundry/configmanager/v2/internal/testutils" -// ) - -// var ( -// customts = "___" -// customop = "/foo" -// standardop = "./app.env" -// standardts = "#" -// ) - -// type fixture struct { -// t *testing.T -// c *GenVars -// rs *retrieveStrategy -// } - -// func newFixture(t *testing.T) *fixture { -// f := &fixture{} -// f.t = t -// return f -// } - -// func (f *fixture) configGenVars(op, ts string) { -// conf := NewConfig().WithOutputPath(op).WithTokenSeparator(ts) -// gv := NewGenerator().WithConfig(conf) -// f.rs = newRetrieveStrategy(NewDefatultStrategy(), *conf) -// f.c = gv -// } - -// func TestGenVarsWithConfig(t *testing.T) { - -// f := newFixture(t) - -// f.configGenVars(customop, customts) -// if f.c.config.outpath != customop { -// f.t.Errorf(testutils.TestPhrase, f.c.config.outpath, customop) -// } -// if f.c.config.tokenSeparator != customts { -// f.t.Errorf(testutils.TestPhrase, f.c.config.tokenSeparator, customts) -// } -// } - -// func TestStripPrefixNormal(t *testing.T) { -// ttests := map[string]struct { -// prefix ImplementationPrefix -// token string -// keySeparator string -// tokenSeparator string -// f *fixture -// expect string -// }{ -// "standard azkv": {AzKeyVaultSecretsPrefix, "AZKVSECRET://vault1/secret2", "|", "://", newFixture(t), "vault1/secret2"}, -// "standard hashivault": {HashicorpVaultPrefix, "VAULT://vault1/secret2", "|", "://", newFixture(t), "vault1/secret2"}, -// "custom separator hashivault": {HashicorpVaultPrefix, "VAULT#vault1/secret2", "|", "#", newFixture(t), "vault1/secret2"}, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// tt.f.configGenVars(tt.keySeparator, tt.tokenSeparator) -// got := tt.f.rs.stripPrefix(tt.token, tt.prefix) -// if got != tt.expect { -// t.Errorf(testutils.TestPhrase, got, tt.expect) -// } -// }) -// } -// } - -// func Test_stripPrefix(t *testing.T) { -// f := newFixture(t) -// f.configGenVars(standardop, standardts) -// tests := []struct { -// name string -// token string -// prefix ImplementationPrefix -// expect string -// }{ -// { -// name: "simple", -// token: fmt.Sprintf("%s#/test/123", SecretMgrPrefix), -// prefix: SecretMgrPrefix, -// expect: "/test/123", -// }, -// { -// name: "key appended", -// token: fmt.Sprintf("%s#/test/123|key", ParamStorePrefix), -// prefix: ParamStorePrefix, -// expect: "/test/123", -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := f.rs.stripPrefix(tt.token, tt.prefix) -// if tt.expect != got { -// t.Errorf(testutils.TestPhrase, tt.expect, got) -// } -// }) -// } -// } - -// func Test_NormaliseMap(t *testing.T) { -// f := newFixture(t) -// f.configGenVars(standardop, standardts) -// tests := []struct { -// name string -// gv *GenVars -// input map[string]any -// expected string -// }{ -// { -// name: "foo->FOO", -// gv: f.c, -// input: map[string]any{"foo": "bar"}, -// expected: "FOO", -// }, -// { -// name: "num->NUM", -// gv: f.c, -// input: map[string]any{"num": 123}, -// expected: "NUM", -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := f.c.envVarNormalize(tt.input) -// for k := range got { -// if k != tt.expected { -// t.Errorf(testutils.TestPhrase, tt.expected, k) -// } -// } -// }) -// } -// } - -// func Test_KeyLookup(t *testing.T) { -// f := newFixture(t) -// f.configGenVars(standardop, standardts) - -// tests := []struct { -// name string -// gv *GenVars -// val string -// key string -// expect string -// }{ -// { -// name: "lowercase key found in str val", -// gv: f.c, -// key: `something|key`, -// val: `{"key": "11235"}`, -// expect: "11235", -// }, -// { -// name: "lowercase key found in numeric val", -// gv: f.c, -// key: `something|key`, -// val: `{"key": 11235}`, -// expect: "11235", -// }, -// { -// name: "lowercase nested key found in numeric val", -// gv: f.c, -// key: `something|key.test`, -// val: `{"key":{"bar":"foo","test":12345}}`, -// expect: "12345", -// }, -// { -// name: "uppercase key found in val", -// gv: f.c, -// key: `something|KEY`, -// val: `{"KEY": "upposeres"}`, -// expect: "upposeres", -// }, -// { -// name: "uppercase nested key found in val", -// gv: f.c, -// key: `something|KEY.TEST`, -// val: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: "upposeres", -// }, -// { -// name: "no key found in val", -// gv: f.c, -// key: `something`, -// val: `{"key": "notfound"}`, -// expect: `{"key": "notfound"}`, -// }, -// { -// name: "nested key not found", -// gv: f.c, -// key: `something|KEY.KEY`, -// val: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: "", -// }, -// { -// name: "incorrect json", -// gv: f.c, -// key: "something|key", -// val: `"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: `"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// }, -// { -// name: "no key provided", -// gv: f.c, -// key: "something", -// val: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// }, -// { -// name: "return json object", -// gv: f.c, -// key: "something|key.test", -// val: `{"key":{"bar":"foo","test": {"key": "default"}}}`, -// expect: `{"key": "default"}`, -// }, -// { -// name: "unescapable string", -// gv: f.c, -// key: "something|key.test", -// val: `{"key":{"bar":"foo","test":"\\\"upposeres\\\""}}`, -// expect: `\"upposeres\"`, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := f.c.keySeparatorLookup(tt.key, tt.val) -// if got != tt.expect { -// t.Errorf(testutils.TestPhrase, got, tt.expect) -// } -// }) -// } -// } - -// type mockRetrieve struct //func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) chanResp -// { -// r func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp -// s func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) -// } - -// func (m mockRetrieve) RetrieveByToken(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return m.r(ctx, impl, prefix, in) -// } -// func (m mockRetrieve) SelectImplementation(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return m.s(ctx, prefix, in, config) -// } - -// type mockImpl struct { -// token, value string -// err error -// } - -// func (m *mockImpl) tokenVal(rs *retrieveStrategy) (s string, e error) { -// return m.value, m.err -// } -// func (m *mockImpl) setTokenVal(s string) { -// m.token = s -// } - -// func Test_generate_rawmap_of_tokens_mapped_to_values(t *testing.T) { -// ttests := map[string]struct { -// rawMap func(t *testing.T) map[string]string -// rs func(t *testing.T) retrieveIface -// expectMap func() map[string]string -// }{ -// "success": { -// func(t *testing.T) map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// func(t *testing.T) retrieveIface { -// return mockRetrieve{ -// r: func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return ChanResp{ -// err: nil, -// value: "bar", -// } -// }, -// s: func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return &mockImpl{"foo", "bar", nil}, nil -// }} -// }, -// func() map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// }, -// // as the method swallows errors at the moment this is not very useful -// "error in implementation": { -// func(t *testing.T) map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// func(t *testing.T) retrieveIface { -// return mockRetrieve{ -// r: func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return ChanResp{ -// err: fmt.Errorf("unable to retrieve"), -// } -// }, -// s: func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return &mockImpl{"foo", "bar", nil}, nil -// }} -// }, -// func() map[string]string { -// rm := make(map[string]string) -// return rm -// }, -// }, -// "error in imp selection": { -// func(t *testing.T) map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// func(t *testing.T) retrieveIface { -// return mockRetrieve{ -// r: func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return ChanResp{ -// err: fmt.Errorf("unable to retrieve"), -// } -// }, -// s: func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return nil, fmt.Errorf("implementation not found for input string: %s", in) -// }} -// }, -// func() map[string]string { -// rm := make(map[string]string) -// return rm -// }, -// }, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// generator := newGenVars() -// generator.generate(tt.rawMap(t), tt.rs(t)) -// got := generator.RawMap() -// if len(got) != len(tt.expectMap()) { -// t.Errorf(testutils.TestPhraseWithContext, "generated raw map did not match", len(got), len(tt.expectMap())) -// } -// }) -// } -// } - -// func TestGenerate(t *testing.T) { -// ttests := map[string]struct { -// tokens func(t *testing.T) []string -// expectLength int -// }{ -// "success without correct prefix": { -// func(t *testing.T) []string { -// return []string{"WRONGIMPL://bar-vault/token1", "AZKVNOTSECRET://bar-vault/token1"} -// }, -// 0, -// }, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// generator := newGenVars() -// pm, err := generator.Generate(tt.tokens(t)) -// if err != nil { -// t.Errorf(testutils.TestPhrase, err.Error(), nil) -// } -// if len(pm) < tt.expectLength { -// t.Errorf(testutils.TestPhrase, len(pm), tt.expectLength) -// } -// }) -// } -// }