From 4c7c694820a9ada9ab64e2c7721c558b81d4afe8 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Sun, 19 Oct 2025 09:05:04 +0100 Subject: [PATCH 01/19] fix: intial regex only implementation we can do better --- configmanager.go | 15 +++++++++++- configmanager_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/configmanager.go b/configmanager.go index a422dfb..748c4b7 100644 --- a/configmanager.go +++ b/configmanager.go @@ -16,7 +16,7 @@ import ( ) const ( - TERMINATING_CHAR string = `[^\'\"\s\n\\\,]` + TERMINATING_CHAR string = `[^\'\"\s\n\\\,]` // :\@\?\/ ) // generateAPI @@ -95,6 +95,19 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) return replaceString(m, input), nil } +func (c *ConfigManager) DiscoverTokens(input string) []string { + tokens := []string{} + // termChar := `[^\'\"\s\n\\\,\:\@]` // :\@\?\/ + // (?:[^\'\"\\\,\:\@\&\s]|(?:[^/?]))+ + for k := range config.VarPrefix { + re := regexp.MustCompile(regexp.QuoteMeta(string(k)+c.Config.TokenSeparator()) + `(?:[^'"\\,:@&\s/]|/(?:[^?]|$))+`) + // re := regexp.MustCompile(regexp.QuoteMeta(string(k)+c.Config.TokenSeparator()) + `(?:[^\'\"\s\n\\\,:@&]|(?:[^/?]))+`) + matches := re.FindAllString(input, -1) + tokens = append(tokens, matches...) + } + return tokens +} + // FindTokens extracts all replaceable tokens // from a given input string func FindTokens(input string) []string { diff --git a/configmanager_test.go b/configmanager_test.go index 43e0ef8..15bf871 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -520,6 +520,63 @@ func TestFindTokens(t *testing.T) { } } +func Test_ConfigManager_DiscoverTokens(t *testing.T) { + ttests := map[string]struct { + input string + separator string + expect []string + }{ + "multiple tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR:///path/config|foo.user:AWSPARAMSTR:///path/config|password@AWSPARAMSTR:///path/config|foo.endpoint:AWSPARAMSTR:///path/config|foo.port/?someQ=AWSPARAMSTR:///path/queryparam[version=123]|p1&anotherQ=false`, + "://", + []string{ + "AWSPARAMSTR:///path/config|foo.user", + "AWSPARAMSTR:///path/config|password", + "AWSPARAMSTR:///path/config|foo.endpoint", + "AWSPARAMSTR:///path/config|foo.port", + "AWSPARAMSTR:///path/queryparam[version=123]|p1"}, + }, + "# tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR#/path/config|foo.user:AWSPARAMSTR#/path/config|password@AWSPARAMSTR#/path/config|foo.endpoint:AWSPARAMSTR#/path/config|foo.port/?someQ=AWSPARAMSTR#/path/queryparam[version=123]|p1&anotherQ=false`, + "#", + []string{ + "AWSPARAMSTR#/path/config|foo.user", + "AWSPARAMSTR#/path/config|password", + "AWSPARAMSTR#/path/config|foo.endpoint", + "AWSPARAMSTR#/path/config|foo.port", + "AWSPARAMSTR#/path/queryparam[version=123]|p1"}, + }, + "without leading slash and path like name # tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR#path_config|foo.user:AWSPARAMSTR#path_config|password@AWSPARAMSTR#path_config|foo.endpoint:AWSPARAMSTR#path_config|foo.port/?someQ=AWSPARAMSTR#path_queryparam[version=123]|p1&anotherQ=false`, + "#", + []string{ + "AWSPARAMSTR#path_config|foo.user", + "AWSPARAMSTR#path_config|password", + "AWSPARAMSTR#path_config|foo.endpoint", + "AWSPARAMSTR#path_config|foo.port", + "AWSPARAMSTR#path_queryparam[version=123]|p1"}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + config.VarPrefix = map[config.ImplementationPrefix]bool{"AWSPARAMSTR": true} + c := configmanager.New(context.TODO()) + c.Config.WithTokenSeparator(tt.separator) + got := c.DiscoverTokens(tt.input) + sort.Strings(got) + sort.Strings(tt.expect) + + if len(got) != len(tt.expect) { + t.Errorf("wrong length - got %d, want %d", len(got), len(tt.expect)) + } + + if !reflect.DeepEqual(got, tt.expect) { + t.Errorf("input=(%q)\n\ngot: %v\n\nwant: %v", tt.input, got, tt.expect) + } + }) + } +} + func Test_YamlRetrieveMarshalled_errored_in_generator(t *testing.T) { m := &mockGenerator{} m.generate = func(tokens []string) (generator.ParsedMap, error) { From 72cf86d70293e3ae42e1d964fbeb03af8dc59fad Mon Sep 17 00:00:00 2001 From: dnitsch Date: Tue, 21 Oct 2025 10:05:57 +0100 Subject: [PATCH 02/19] fix: parser implemented --- README.md | 4 +- internal/config/config.go | 35 ++++- internal/config/token.go | 95 ++++++++++++ internal/lexer/lexer.go | 255 ++++++++++++++++++++++++++++++++ internal/lexer/lexer_test.go | 99 +++++++++++++ internal/parser/doc.go | 5 + internal/parser/parser.go | 257 +++++++++++++++++++++++++++++++++ internal/parser/parser_test.go | 237 ++++++++++++++++++++++++++++++ 8 files changed, 983 insertions(+), 4 deletions(-) create mode 100644 internal/config/token.go create mode 100644 internal/lexer/lexer.go create mode 100644 internal/lexer/lexer_test.go create mode 100644 internal/parser/doc.go create mode 100644 internal/parser/parser.go create mode 100644 internal/parser/parser_test.go diff --git a/README.md b/README.md index 635e376..d62f2fa 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Where `configVar` can be either a parseable string `'som3#!S$CRet'` or a number - Kubernetes - Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS#/$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases. + Avoid storing overly large configmaps and especially using secrets objects to store actual secrets e.g. DB passwords, 3rd party API creds, etc... By only storing a config file or a script containing only the tokens e.g. `AWSSECRETS:///$ENV/service/db-config` it can be git committed without writing numerous shell scripts, only storing either some interpolation vars like `$ENV` in a configmap or the entire configmanager token for smaller use cases. - VMs @@ -86,7 +86,7 @@ The token is made up of the following parts: _An example token would look like this_ -#### `AWSSECRETS#/path/to/my/key|lookup.Inside.Object[meta=data]` +#### `AWSSECRETS:///path/to/my/key|lookup.Inside.Object[meta=data]` ### Implementation indicator diff --git a/internal/config/config.go b/internal/config/config.go index 50aecce..9965d7d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -49,6 +49,7 @@ var ( GcpSecretsPrefix: true, HashicorpVaultPrefix: true, AzTableStorePrefix: true, AzAppConfigPrefix: true, UnknownPrefix: true, } + ErrConfigValidation = errors.New("config validation failed") ) // GenVarsConfig defines the input config object to be passed @@ -60,7 +61,9 @@ type GenVarsConfig struct { // parseAdditionalVars func(token string) TokenConfigVars } -// NewConfig +// NewConfig returns a new GenVarsConfig with default values +// +// keySeparator should be only a single character func NewConfig() *GenVarsConfig { return &GenVarsConfig{ tokenSeparator: tokenSeparator, @@ -120,8 +123,15 @@ func (c *GenVarsConfig) Config() GenVarsConfig { return cc } -// Parsed token config section +// Config returns the derefed value +func (c *GenVarsConfig) Validate() error { + if len(c.keySeparator) > 1 { + return fmt.Errorf("%w, keyseparator can only be 1 character", ErrConfigValidation) + } + return nil +} +// Parsed token config section var ErrInvalidTokenPrefix = errors.New("token prefix has no implementation") type ParsedTokenConfig struct { @@ -153,6 +163,27 @@ func NewParsedTokenConfig(token string, config GenVarsConfig) (*ParsedTokenConfi return ptc.new(), nil } +func NewTokenConfig(prefix ImplementationPrefix, config GenVarsConfig) (*ParsedTokenConfig, error) { + tokenConf := &ParsedTokenConfig{} + if err := config.Validate(); err != nil { + return nil, err + } + tokenConf.keySeparator = config.keySeparator + tokenConf.tokenSeparator = config.tokenSeparator + + tokenConf.prefix = prefix + + return tokenConf, nil +} + +func (ptc *ParsedTokenConfig) WithKeyPath(kp string) { + ptc.keysPath = kp +} + +func (ptc *ParsedTokenConfig) WithMetadata(md string) { + ptc.metadataStr = md +} + func (ptc *ParsedTokenConfig) new() *ParsedTokenConfig { // order must be respected here // diff --git a/internal/config/token.go b/internal/config/token.go new file mode 100644 index 0000000..7b1c2d1 --- /dev/null +++ b/internal/config/token.go @@ -0,0 +1,95 @@ +package config + +// TokenType is the lexer parsed TokenType +type TokenType string + +const ( + ILLEGAL TokenType = "ILLEGAL" + EOF TokenType = "EOF" + + SPACE TokenType = "SPACE" // ' ' + TAB TokenType = "TAB" // '\t' + NEW_LINE TokenType = "NEW_LINE" // '\n' + CARRIAGE_RETURN TokenType = "CARRIAGE_RETURN" // '\r' + CONTROL TokenType = "CONTROL" + + // Identifiers + literals + TEXT TokenType = "TEXT" + + EXCLAMATION TokenType = "!" + DOUBLE_QUOTE TokenType = "\"" + SINGLE_QUOTE TokenType = "'" + // other separators + AT_SIGN TokenType = "AT_SIGN" // `@` + PIPE TokenType = "PIPE" // `|` + COLON TokenType = "COLON" // `:` + EQUALS TokenType = "EQUALS" // `=` + DOT TokenType = "DOT" // `.` + COMMA TokenType = "COMMA" // `,` + QUESTION_MARK TokenType = "QUESTION_MARK" // `?` + BACK_SLASH TokenType = "BACK_SLASH" // `\` + FORWARD_SLASH TokenType = "FORWARD_SLASH" // `/` + SLASH_QUESTION_MARK TokenType = "SLASH_QUESTION_MARK" // `/?` + + // Comment Tokens + DOUBLE_FORWARD_SLASH TokenType = "DOUBLE_FORWARD_SLASH" // `//` + HASH TokenType = "HASH" // `#` + + // CONFIGMANAGER_TOKEN Keywords + // CONFIGMANAGER_TOKEN_SEPARATOR TokenType = "CONFIGMANAGER_TOKEN_SEPARATOR" // Dynamically set + BEGIN_CONFIGMANAGER_TOKEN TokenType = "BEGIN_CONFIGMANAGER_TOKEN" // Dynamically set + CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR TokenType = "CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR" // Dynamically set + BEGIN_META_CONFIGMANAGER_TOKEN TokenType = "BEGIN_META_CONFIGMANAGER_TOKEN" // `[` + END_META_CONFIGMANAGER_TOKEN TokenType = "END_META_CONFIGMANAGER_TOKEN" // `]` + // This may not possible + END_CONFIGMANAGER_TOKEN TokenType = "END_CONFIGMANAGER_TOKEN" + + // Parsed "expressions" + CONFIGMANAGER_TOKEN_CONTENT TokenType = "CONFIGMANAGER_TOKEN_CONTENT" + UNUSED_TEXT TokenType = "UNUSED_TEXT" +) + +type Source struct { + File string `json:"file"` + Path string `json:"path"` +} + +// Token is the basic structure of the captured token +type Token struct { + Type TokenType + Literal string + ImpPrefix ImplementationPrefix + Line int + Column int + Source Source +} + +var keywords = map[string]TokenType{ + " ": SPACE, + "\n": NEW_LINE, + "\r": CARRIAGE_RETURN, + "\t": TAB, + "\f": CONTROL, +} + +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[ident]; ok { + return tok + } + return TEXT +} + +// var typeMapper = map[string]TokenType{ +// "MESSAGE": MESSAGE, +// "OPERATION": OPERATION, +// "CHANNEL": CHANNEL, +// "INFO": INFO, +// "SERVER": SERVER, +// } + +// func LookupType(typ string) TokenType { +// if tok, ok := typeMapper[strings.ToUpper(typ)]; ok { +// return tok +// } +// return "" +// } diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go new file mode 100644 index 0000000..25a9be2 --- /dev/null +++ b/internal/lexer/lexer.go @@ -0,0 +1,255 @@ +// Package lexer +// +// Performs lexical analysis on the source files and emits tokens. +package lexer + +import ( + "github.com/DevLabFoundry/configmanager/v2/internal/config" +) + +// nonText characters captures all character sets that are _not_ assignable to TEXT +var nonText = map[string]bool{ + // separators + " ": true, "\n": true, "\r": true, "\t": true, + "=": true, ".": true, ",": true, "|": true, "?": true, "/": true, "@": true, ":": true, + "]": true, "[": true, + // initial chars of potential identifiers + // TODO: when a new implementation is added we should add it here + // AWS|AZure... + "A": true, + // VAULT (HashiCorp) + "V": true, + // GCP + "G": true, + // Unknown + "U": true, +} + +type Source struct { + Input string + FileName string + FullPath string +} + +// Lexer +type Lexer struct { + config config.GenVarsConfig + keySeparator byte + length int + source Source + position int // current position in input (points to current char) + readPosition int // current reading position in input (after current char) + ch byte // current char under examination + line int // current line - start at 1 + column int // column of text - gets set to 0 on every new line - start at 0 +} + +// New returns a Lexer pointer allocation +func New(source Source, config config.GenVarsConfig) *Lexer { + l := &Lexer{ + source: source, + line: 1, + column: 0, + length: len(source.Input), + config: config, + keySeparator: config.KeySeparator()[0], + } + l.readChar() + return l +} + +// NextToken advances through the source returning a found token +func (l *Lexer) NextToken() config.Token { + var tok config.Token + + switch l.ch { + // identify the dynamically selected key separator + case l.keySeparator: + tok = config.Token{Type: config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, Literal: string(l.ch)} + case 'A': + // AWS store types + if l.peekChar() == 'W' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.SecretMgrPrefix, config.ParamStorePrefix}, "AW"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AW as text + tok = config.Token{Type: config.TEXT, Literal: "AW"} + } + // Azure Store Types + } else if l.peekChar() == 'Z' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.AzKeyVaultSecretsPrefix, config.AzTableStorePrefix, config.AzAppConfigPrefix}, "AZ"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AZ as text + tok = config.Token{Type: config.TEXT, Literal: "AZ"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "A"} + } + case 'G': + if l.peekChar() == 'C' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.GcpSecretsPrefix}, "GC"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AW as text + tok = config.Token{Type: config.TEXT, Literal: "GC"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "G"} + } + case 'V': + if l.peekChar() == 'A' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.HashicorpVaultPrefix}, "VA"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AW as text + tok = config.Token{Type: config.TEXT, Literal: "VA"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "V"} + } + case 'U': + if l.peekChar() == 'N' { + l.readChar() + if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.UnknownPrefix}, "UN"); found { + tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} + } else { + // it is not a marker AW as text + tok = config.Token{Type: config.TEXT, Literal: "UN"} + } + } else { + tok = config.Token{Type: config.TEXT, Literal: "U"} + } + case '=': + tok = config.Token{Type: config.EQUALS, Literal: "="} + case '.': + tok = config.Token{Type: config.DOT, Literal: "."} + case ',': + tok = config.Token{Type: config.COMMA, Literal: ","} + case '/': + if l.peekChar() == '?' { + l.readChar() + tok = config.Token{Type: config.SLASH_QUESTION_MARK, Literal: "/?"} + } else { + tok = config.Token{Type: config.FORWARD_SLASH, Literal: "/"} + } + case '\\': + tok = config.Token{Type: config.BACK_SLASH, Literal: "\\"} + case '?': + tok = config.Token{Type: config.QUESTION_MARK, Literal: "?"} + case ']': + tok = config.Token{Type: config.END_META_CONFIGMANAGER_TOKEN, Literal: "]"} + case '[': + tok = config.Token{Type: config.BEGIN_META_CONFIGMANAGER_TOKEN, Literal: "["} + case '|': + tok = config.Token{Type: config.PIPE, Literal: "|"} + case '@': + tok = config.Token{Type: config.AT_SIGN, Literal: "@"} + case ':': + tok = config.Token{Type: config.COLON, Literal: ":"} + case '\n': + l.line = l.line + 1 + l.column = 0 // reset column count + tok = l.setTextSeparatorToken() + // want to preserve all indentations and punctuation + case ' ', '\r', '\t', '\f': + tok = l.setTextSeparatorToken() + case 0: + tok.Literal = "" + tok.Type = config.EOF + // case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + // 'B', 'C', 'D', 'E', 'F', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'W', 'X', 'Y', 'Z', + // 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z': + + default: + if isText(l.ch) { + tok.Literal = l.readText() + tok.Type = config.TEXT + return tok + } + tok = newToken(config.ILLEGAL, l.ch) + } + // add general properties to each token + tok.Line = l.line + tok.Column = l.column + tok.Source = config.Source{Path: l.source.FullPath, File: l.source.FileName} + l.readChar() + return tok +} + +// readChar moves cursor along +func (l *Lexer) readChar() { + if l.readPosition >= l.length { + l.ch = 0 + } else { + l.ch = l.source.Input[l.readPosition] + } + l.position = l.readPosition + l.readPosition += 1 + l.column += 1 +} + +// peekChar reveals next char withouh advancing the cursor along +func (l *Lexer) peekChar() byte { + if l.readPosition >= l.length { + return 0 + } else { + return l.source.Input[l.readPosition] + } +} + +func (l *Lexer) readText() string { + position := l.position + for isText(l.ch) && l.readPosition <= l.length { + l.readChar() + } + return l.source.Input[position:l.position] +} + +func (l *Lexer) setTextSeparatorToken() config.Token { + tok := newToken(config.LookupIdent(string(l.ch)), l.ch) + return tok +} + +// peekIsBeginOfToken attempts to identify the gendoc keyword after 2 slashes +func (l *Lexer) peekIsBeginOfToken(possibleBeginToken []config.ImplementationPrefix, charsRead string) (bool, string, config.ImplementationPrefix) { + for _, pbt := range possibleBeginToken { + configToken := "" + pbtWithTokenSep := string(pbt[len(charsRead):]) + l.config.TokenSeparator() + for i := 0; i < len(pbtWithTokenSep); i++ { + configToken += string(l.peekChar()) + l.readChar() + } + + if configToken == pbtWithTokenSep { + return true, charsRead + configToken, pbt + } + l.resetAfterPeek(len(pbtWithTokenSep)) + } + return false, "", "" +} + +// peekIsEndOfToken +func (l *Lexer) peekIsEndOfToken() bool { + return false +} + +// resetAfterPeek will go back specified amount on the cursor +func (l *Lexer) resetAfterPeek(back int) { + l.position = l.position - back + l.readPosition = l.readPosition - back +} + +// isText only deals with any text characters defined as +// outside of the capture group +func isText(ch byte) bool { + return !nonText[string(ch)] +} + +func newToken(tokenType config.TokenType, ch byte) config.Token { + return config.Token{Type: tokenType, Literal: string(ch)} +} diff --git a/internal/lexer/lexer_test.go b/internal/lexer/lexer_test.go new file mode 100644 index 0000000..5ab297b --- /dev/null +++ b/internal/lexer/lexer_test.go @@ -0,0 +1,99 @@ +package lexer_test + +import ( + "testing" + + "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v2/internal/lexer" +) + +func Test_Lexer_NextToken(t *testing.T) { + input := `foo stuyfsdfsf +foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo +META_INCLUDED=VAULT://baz/bar/123|key1.prop2[role=arn:aws:iam::1111111:role,version=1082313] +` + ttests := []struct { + expectedType config.TokenType + expectedLiteral string + }{ + {config.TEXT, "foo"}, + {config.SPACE, " "}, + {config.TEXT, "stuyfsdfsf"}, + {config.NEW_LINE, "\n"}, + {config.TEXT, "foo"}, + {config.EQUALS, "="}, + {config.BEGIN_CONFIGMANAGER_TOKEN, "AWSPARAMSTR://"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "path"}, + {config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {config.TEXT, "key"}, + {config.BEGIN_CONFIGMANAGER_TOKEN, "AWSSECRETS://"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "foo"}, + {config.NEW_LINE, "\n"}, + {config.TEXT, "MET"}, + {config.TEXT, "A"}, + {config.TEXT, "_INCL"}, + {config.TEXT, "U"}, + {config.TEXT, "DED"}, + {config.EQUALS, "="}, + {config.BEGIN_CONFIGMANAGER_TOKEN, "VAULT://"}, + {config.TEXT, "baz"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "bar"}, + {config.FORWARD_SLASH, "/"}, + {config.TEXT, "123"}, + {config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {config.TEXT, "key1"}, + {config.DOT, "."}, + {config.TEXT, "prop2"}, + {config.BEGIN_META_CONFIGMANAGER_TOKEN, "["}, + {config.TEXT, "role"}, + {config.EQUALS, "="}, + {config.TEXT, "arn"}, + {config.COLON, ":"}, + {config.TEXT, "aws"}, + {config.COLON, ":"}, + {config.TEXT, "iam"}, + {config.COLON, ":"}, + {config.COLON, ":"}, + {config.TEXT, "1111111"}, + {config.COLON, ":"}, + {config.TEXT, "role"}, + {config.COMMA, ","}, + {config.TEXT, "version"}, + {config.EQUALS, "="}, + {config.TEXT, "1082313"}, + {config.END_META_CONFIGMANAGER_TOKEN, "]"}, + {config.NEW_LINE, "\n"}, + {config.EOF, ""}, + } + + l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig()) + + for i, tt := range ttests { + + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. got=%q, expected=%q", + i, tok.Type, tt.expectedType) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - literal wrong. got=%q, expected=%q", + i, tok.Literal, tt.expectedLiteral) + } + if tok.Type == config.BEGIN_CONFIGMANAGER_TOKEN { + + } + } +} + +func Test_empty_file(t *testing.T) { + input := `` + l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig()) + tok := l.NextToken() + if tok.Type != config.EOF { + t.Fatal("expected EOF") + } +} diff --git a/internal/parser/doc.go b/internal/parser/doc.go new file mode 100644 index 0000000..c33ca52 --- /dev/null +++ b/internal/parser/doc.go @@ -0,0 +1,5 @@ +// Package parser +// Analyses a given string of text and extracts any configmanager tokens +// +// It builds any additiona metadata as part of the analysis +package parser diff --git a/internal/parser/parser.go b/internal/parser/parser.go new file mode 100644 index 0000000..0e619c1 --- /dev/null +++ b/internal/parser/parser.go @@ -0,0 +1,257 @@ +package parser + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v2/internal/lexer" + "github.com/DevLabFoundry/configmanager/v2/internal/log" + + "github.com/a8m/envsubst" +) + +func wrapErr(file string, line, position int, etyp error) error { + return fmt.Errorf("\n - [%s:%d:%d] %w", file, line, position, etyp) +} + +var ( + ErrNoEndTagFound = errors.New("no corresponding end tag found") + ErrUnableToReplaceVarPlaceholder = errors.New("variable specified in the content was not found in the environment") +) + +type ConfigManagerTokenBlock struct { + BeginToken config.Token + ParsedToken config.ParsedTokenConfig + Value string + EndToken config.Token +} + +type Parser struct { + l *lexer.Lexer + errors []error + log log.ILogger + curToken config.Token + peekToken config.Token + config *config.GenVarsConfig + environ []string +} + +func New(l *lexer.Lexer, c *config.GenVarsConfig) *Parser { + p := &Parser{ + l: l, + log: log.New(os.Stderr), + errors: []error{}, + config: c, + environ: os.Environ(), + } + + // Read two tokens, so curToken and peekToken are both set + // first one sets the curToken to the value of peekToken - + // which at this point is just the first upcoming token + p.nextToken() + // second one sets the curToken to the actual value of the first upcoming + // token and peekToken is the actual second upcoming token + p.nextToken() + + return p +} + +func (p *Parser) WithEnvironment(environ []string) *Parser { + p.environ = environ + return p +} + +func (p *Parser) WithLogger(logger log.ILogger) *Parser { + p.log = nil //speed up GC + p.log = logger + return p +} + +// Parse creates a flat list of ConfigManagerTokenBlock +// In the order they were declared in the source text +// +// The parser does not do a second pass and interprets the source from top to bottom +func (p *Parser) Parse() ([]ConfigManagerTokenBlock, []error) { + genDocStms := []ConfigManagerTokenBlock{} + + for !p.currentTokenIs(config.EOF) { + if p.currentTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { + // parseGenDocBlocks will advance the token until + // it hits the END_DOC_GEN token + configManagerToken, err := config.NewTokenConfig(p.curToken.ImpPrefix, *p.config) + if err != nil { + return nil, []error{err} + } + if stmt := p.buildConfigManagerTokenFromBlocks(configManagerToken); stmt != nil { + genDocStms = append(genDocStms, *stmt) + } + } + p.nextToken() + } + + return genDocStms, p.errors +} + +// ExpandEnvVariables expands the env vars inside DocContent +// to their environment var values. +// +// Failing when a variable is either not set or set but empty. +func ExpandEnvVariables(input string, vars []string) (string, error) { + for _, v := range vars { + kv := strings.Split(v, "=") + key, value := kv[0], kv[1] // kv[1] will be an empty string = "" + os.Setenv(key, value) + } + + return envsubst.StringRestrictedNoDigit(input, true, true, false) +} + +func (p *Parser) nextToken() { + p.curToken = p.peekToken + p.peekToken = p.l.NextToken() +} + +func (p *Parser) currentTokenIs(t config.TokenType) bool { + return p.curToken.Type == t +} + +func (p *Parser) peekTokenIs(t config.TokenType) bool { + return p.peekToken.Type == t +} + +func (p *Parser) peekTokenIsEnd() bool { + endTokens := map[config.TokenType]bool{ + config.AT_SIGN: true, config.QUESTION_MARK: true, config.COLON: true, + config.DOUBLE_QUOTE: true, config.SINGLE_QUOTE: true, + config.NEW_LINE: true, + } + return endTokens[p.peekToken.Type] +} + +// buildConfigManagerTokenFromBlocks throws away all other content other +// than what is inside //+gendoc tags +// parses any annotation and creates GenDocBlock +// for later analysis +func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.ParsedTokenConfig) *ConfigManagerTokenBlock { + currentToken := p.curToken + stmt := &ConfigManagerTokenBlock{BeginToken: currentToken} + + // move past current token + p.nextToken() + + fullToken := currentToken.Literal + // pathLookup := "" + // metadataPortion := "" + + // should exit the loop if no end doc tag found + notFoundEnd := true + // stop on end of file + for !p.peekTokenIs(config.EOF) { + + // when next token is another token + // i.e. the tokens are adjacent + if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { + notFoundEnd = false + fullToken += p.curToken.Literal + stmt.EndToken = p.curToken + p.nextToken() + break + } + + // reached the end of a potential token + if p.peekTokenIsEnd() { + notFoundEnd = false + fullToken += p.curToken.Literal + stmt.EndToken = p.curToken + break + } + + //sample token will be consumed like this + // AWSSECRETS:///path/to/my/key|lookup.Inside.Object[meta=data] + // + // everything is token path until (if any key separator exists) + // check key separator this marks the end of a normal token path + if p.currentTokenIs(config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR) { + // advance to next token i.e. start of the path separator + p.nextToken() + if err := p.buildKeyPathSeparator(configManagerToken); err != nil { + p.errors = append(p.errors, err) + return nil + } + p.nextToken() + continue + } + // optionally at the end of the path (i.e. a JSONPath look up) + // check metadata there can be a metadata bracket `[key=val,k1=v2]` + if p.currentTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { + if err := p.buildMetadata(configManagerToken); err != nil { + p.errors = append(p.errors, err) + return nil + } + } + fullToken += p.curToken.Literal + p.nextToken() + } + + if notFoundEnd { + p.errors = append(p.errors, wrapErr(currentToken.Source.File, currentToken.Line, currentToken.Column, ErrNoEndTagFound)) + return nil + } + + stmt.ParsedToken = *configManagerToken + stmt.Value = fullToken + + return stmt +} + +// buildKeyPathSeparator already advanced to the first token +func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenConfig) error { + keyPath := "" + for !p.peekTokenIs(config.EOF) { + if p.peekTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { + if err := p.buildMetadata(configManagerToken); err != nil { + return err + } + break + } + // touching another token or end of token + if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() { + keyPath += p.curToken.Literal + break + } + keyPath += p.curToken.Literal + p.nextToken() + } + configManagerToken.WithKeyPath(keyPath) + return nil +} + +// buildMetadata adds metadata to the ParsedTokenConfig +func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) error { + // inLoop := true + // errNoClose := fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) + metadata := "" + found := false + for !p.peekTokenIs(config.EOF) { + if p.peekTokenIsEnd() { + return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) + } + if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { + metadata += p.curToken.Literal + found = true + p.nextToken() + break + } + metadata += p.curToken.Literal + p.nextToken() + } + configManagerToken.WithMetadata(metadata) + + if !found { + return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) + } + return nil +} diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go new file mode 100644 index 0000000..35d9225 --- /dev/null +++ b/internal/parser/parser_test.go @@ -0,0 +1,237 @@ +package parser_test + +import ( + "errors" + "os" + "testing" + + "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v2/internal/lexer" + "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/parser" +) + +var lexerSource = lexer.Source{FileName: "bar", FullPath: "/foo/bar"} + +func Test_ParserBlocks(t *testing.T) { + ttests := map[string]struct { + input string + expected [][]string + }{ + "tokens touching each other in source": {`foo stuyfsdfsf +foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo +other text her +BAR=something + `, [][]string{ + {string(config.ParamStorePrefix), "/path|key"}, + {string(config.SecretMgrPrefix), "/foo"}, + }}, + "full URL of tokens": {`foo stuyfsdfsf +foo=proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2 +# some comment +BAR=something +`, [][]string{ + {string(config.ParamStorePrefix), "/config|user"}, + {string(config.SecretMgrPrefix), "/creds|password"}, + {string(config.ParamStorePrefix), "/config|endpoint"}, + {string(config.ParamStorePrefix), "/config|port"}, + {string(config.ParamStorePrefix), "/config|qp2"}, + }}, + } + + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + l := lexer.New(lexerSource, *config.NewConfig()) + p := parser.New(l, config.NewConfig()).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("parser had errors, expected \nerror: %v", errs) + } + + if len(parsed) != len(tt.expected) { + t.Fatalf("parsed statements count does not match\ngot=%d want=%d\nparsed %q", + len(parsed), + len(tt.expected), + parsed) + } + + for idx, stmt := range parsed { + if !testHelperGenDocBlock(t, stmt, config.ImplementationPrefix(tt.expected[idx][0]), tt.expected[idx][1]) { + return + } + } + }) + } +} + +func Test_ShouldError_when_no_End_tag_found(t *testing.T) { + input := `let x = 5; + //+gendoc category=message type=nameId parent=id1 id=id + ` + + lexerSource.Input = input + l := lexer.New(lexerSource, *config.NewConfig()) + p := parser.New(l, &config.GenVarsConfig{}).WithLogger(log.New(os.Stderr)) + _, errs := p.Parse() + if len(errs) != 1 { + t.Errorf("unexpected number of errors\n got: %v, wanted: 1", errs) + } + if !errors.Is(errs[0], parser.ErrNoEndTagFound) { + t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, parser.ErrNoEndTagFound) + } +} + +func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBlock, tokenType config.ImplementationPrefix, tokenValue string) bool { + t.Helper() + if stmtBlock.BeginToken.ImpPrefix != tokenType { + t.Errorf("got=%q, wanted stmtBlock.ImpPrefix = '%v'.", stmtBlock.BeginToken.Literal, tokenType) + return false + } + + if stmtBlock.ParsedToken.StripPrefix() != tokenValue { + t.Errorf("stmtBlock.Value. got=%s, wanted=%s", stmtBlock.ParsedToken.StripPrefix(), tokenValue) + return false + } + + return true +} + +func Test_ExpandEnvVariables_succeeds(t *testing.T) { + ttests := map[string]struct { + input string + expect string + envVar []string + }{ + "with single var": { + "some var is $var", + "some var is foo", + []string{"var=foo"}, + }, + "with multiple var": { + "some var is $var and docs go [here]($DOC_LINK/stuff)", + "some var is foo and docs go [here](https://somestuff.com/stuff)", + []string{"var=foo", "DOC_LINK=https://somestuff.com"}, + }, + "with no vars in content": { + "some var is foo and docs go [here](foo.com/stuff)", + "some var is foo and docs go [here](foo.com/stuff)", + []string{"var=foo", "DOC_LINK=https://somestuff.com"}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + defer os.Clearenv() + got, err := parser.ExpandEnvVariables(tt.input, tt.envVar) + if err != nil { + t.Errorf("expected %v to be ", err) + } + if got != tt.expect { + t.Errorf("want: %s, got: %s", got, tt.expect) + } + }) + } +} + +func Test_ExpandEnvVariables_fails(t *testing.T) { + + ttests := map[string]struct { + input string + setup func() func() + envVar []string + }{ + "with single var": { + "some var is $var", + func() func() { + return func() { + os.Clearenv() + } + }, + []string{"v=foo"}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + clear := tt.setup() + defer clear() + _, err := parser.ExpandEnvVariables(tt.input, tt.envVar) + if err == nil { + t.Errorf("wanted error, got ") + } + }) + } +} + +func Test_Parse_WithOwnEnviron_passed_in_succeeds(t *testing.T) { + ttests := map[string]struct { + input string + expect string + environ []string + }{ + "test1": { + input: `let x = 42; +//+gendoc category=message type=description id=foo +this is some description with $foo +//-gendoc`, + environ: []string{"foo=bar"}, + expect: "this is some description with bar", + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + defer os.Clearenv() + lexerSource.Input = tt.input + l := lexer.New(lexerSource, *config.NewConfig()) + p := parser.New(l, &config.GenVarsConfig{}).WithLogger(log.New(os.Stderr)).WithEnvironment(tt.environ) + got, errs := p.Parse() + if len(errs) > 0 { + t.Error(errs) + } + if got[0].Value != tt.expect { + t.Error("") + } + }) + } +} + +func Test_Parse_WithOwnEnviron_passed_in_fails(t *testing.T) { + ttests := map[string]struct { + input string + expect error + environ []string + }{ + "if variable is not set": { + input: `let x = 42; + //+gendoc category=message type=description id=foo + this is some description with $foo + //-gendoc`, + expect: parser.ErrUnableToReplaceVarPlaceholder, + environ: []string{"notfoo=bar"}, + }, + "if variable is not set but empty": { + input: `let x = 42; +//+gendoc category=message type=description id=foo +this is some description with $foo +//-gendoc`, + expect: parser.ErrUnableToReplaceVarPlaceholder, + environ: []string{"foo="}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + defer os.Clearenv() + lexerSource.Input = tt.input + l := lexer.New(lexerSource, *config.NewConfig()) + p := parser.New(l, &config.GenVarsConfig{}).WithLogger(log.New(os.Stderr)).WithEnvironment(tt.environ) + _, errs := p.Parse() + + if len(errs) < 1 { + t.Error("expected errors to occur") + t.Fail() + } + if !errors.Is(errs[0], tt.expect) { + t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, tt.expect) + } + }) + } +} From dcd72bd28bad0d3fb85d86def95a4541ff5fb342 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Tue, 21 Oct 2025 11:07:33 +0100 Subject: [PATCH 03/19] fix: remove unused docs: update VAULT docs --- README.md | 2 +- internal/config/config.go | 23 +++- internal/parser/parser.go | 21 ++-- internal/parser/parser_test.go | 202 +++++++-------------------------- 4 files changed, 70 insertions(+), 178 deletions(-) diff --git a/README.md b/README.md index d62f2fa..a7762ad 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ All the usual token rules apply e.g. of `keySeparator` For HashicorpVault the first part of the token needs to be the name of the mountpath. In Dev Vaults this is `"secret"`, e.g.: `VAULT://secret___demo/configmanager|test` -or if the secrets are at another location: `VAULT://another/mount/path__config/app1/db` +or if the secrets are at another location: `VAULT://another/mount/path___config/app1/db` The hardcoded separator cannot be modified and you must separate your `mountPath` with `___` (3x `_`) followed by the key to the secret. diff --git a/internal/config/config.go b/internal/config/config.go index 9965d7d..88537a2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -135,11 +135,17 @@ func (c *GenVarsConfig) Validate() error { var ErrInvalidTokenPrefix = errors.New("token prefix has no implementation") type ParsedTokenConfig struct { - prefix ImplementationPrefix + prefix ImplementationPrefix + // cofig values keySeparator, tokenSeparator string - prefixLessToken, fullToken string - metadataStr, keysPath string - storeToken, metadataLess string + + fullToken string + metadataStr string + keysPath string + sanitizedToken string + // depracated + prefixLessToken string + storeToken, metadataLess string } // NewParsedTokenConfig returns a pointer to a new TokenConfig struct @@ -184,6 +190,11 @@ func (ptc *ParsedTokenConfig) WithMetadata(md string) { ptc.metadataStr = md } +func (ptc *ParsedTokenConfig) WithSanitizedToken(v string) { + ptc.sanitizedToken = v +} + +// depracated func (ptc *ParsedTokenConfig) new() *ParsedTokenConfig { // order must be respected here // @@ -230,7 +241,7 @@ func (t *ParsedTokenConfig) StripMetadata() string { return t.metadataLess } -// Strip +// StoreToken // // returns the only the store indicator string // without any of the configmanager token enrichment: @@ -243,7 +254,7 @@ func (t *ParsedTokenConfig) StripMetadata() string { // // - prefix func (t *ParsedTokenConfig) StoreToken() string { - return t.storeToken + return t.sanitizedToken } // Full returns the full Token path. diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 0e619c1..f93dffc 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -25,7 +25,6 @@ var ( type ConfigManagerTokenBlock struct { BeginToken config.Token ParsedToken config.ParsedTokenConfig - Value string EndToken config.Token } @@ -126,7 +125,7 @@ func (p *Parser) peekTokenIsEnd() bool { endTokens := map[config.TokenType]bool{ config.AT_SIGN: true, config.QUESTION_MARK: true, config.COLON: true, config.DOUBLE_QUOTE: true, config.SINGLE_QUOTE: true, - config.NEW_LINE: true, + config.NEW_LINE: true, config.SLASH_QUESTION_MARK: true, } return endTokens[p.peekToken.Type] } @@ -143,8 +142,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa p.nextToken() fullToken := currentToken.Literal - // pathLookup := "" - // metadataPortion := "" + // built as part of the below parser + sanitizedToken := "" // should exit the loop if no end doc tag found notFoundEnd := true @@ -156,8 +155,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { notFoundEnd = false fullToken += p.curToken.Literal + sanitizedToken += p.curToken.Literal stmt.EndToken = p.curToken - p.nextToken() break } @@ -165,6 +164,7 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa if p.peekTokenIsEnd() { notFoundEnd = false fullToken += p.curToken.Literal + sanitizedToken += p.curToken.Literal stmt.EndToken = p.curToken break } @@ -181,10 +181,11 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa p.errors = append(p.errors, err) return nil } - p.nextToken() - continue + notFoundEnd = false + // keyPath would have built the keyPath and metadata if any + break } - // optionally at the end of the path (i.e. a JSONPath look up) + // optionally at the end of the path without key separator // check metadata there can be a metadata bracket `[key=val,k1=v2]` if p.currentTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { if err := p.buildMetadata(configManagerToken); err != nil { @@ -192,6 +193,7 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa return nil } } + sanitizedToken += p.curToken.Literal fullToken += p.curToken.Literal p.nextToken() } @@ -201,8 +203,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa return nil } + configManagerToken.WithSanitizedToken(sanitizedToken) stmt.ParsedToken = *configManagerToken - stmt.Value = fullToken return stmt } @@ -225,6 +227,7 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon keyPath += p.curToken.Literal p.nextToken() } + // p.nextToken() configManagerToken.WithKeyPath(keyPath) return nil } diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 35d9225..222d869 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -15,27 +15,28 @@ var lexerSource = lexer.Source{FileName: "bar", FullPath: "/foo/bar"} func Test_ParserBlocks(t *testing.T) { ttests := map[string]struct { - input string - expected [][]string + input string + // prefix,path,keyLookup + expected [][3]string }{ "tokens touching each other in source": {`foo stuyfsdfsf foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo other text her BAR=something - `, [][]string{ - {string(config.ParamStorePrefix), "/path|key"}, - {string(config.SecretMgrPrefix), "/foo"}, + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", "key"}, + {string(config.SecretMgrPrefix), "/foo", ""}, }}, "full URL of tokens": {`foo stuyfsdfsf foo=proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2 # some comment BAR=something -`, [][]string{ - {string(config.ParamStorePrefix), "/config|user"}, - {string(config.SecretMgrPrefix), "/creds|password"}, - {string(config.ParamStorePrefix), "/config|endpoint"}, - {string(config.ParamStorePrefix), "/config|port"}, - {string(config.ParamStorePrefix), "/config|qp2"}, + `, [][3]string{ + {string(config.ParamStorePrefix), "/config", "user"}, + {string(config.SecretMgrPrefix), "/creds", "password"}, + {string(config.ParamStorePrefix), "/config", "endpoint"}, + {string(config.ParamStorePrefix), "/config", "port"}, + {string(config.ParamStorePrefix), "/config", "qp2"}, }}, } @@ -57,7 +58,7 @@ BAR=something } for idx, stmt := range parsed { - if !testHelperGenDocBlock(t, stmt, config.ImplementationPrefix(tt.expected[idx][0]), tt.expected[idx][1]) { + if !testHelperGenDocBlock(t, stmt, config.ImplementationPrefix(tt.expected[idx][0]), tt.expected[idx][1], tt.expected[idx][2]) { return } } @@ -65,173 +66,50 @@ BAR=something } } -func Test_ShouldError_when_no_End_tag_found(t *testing.T) { - input := `let x = 5; - //+gendoc category=message type=nameId parent=id1 id=id - ` - - lexerSource.Input = input - l := lexer.New(lexerSource, *config.NewConfig()) - p := parser.New(l, &config.GenVarsConfig{}).WithLogger(log.New(os.Stderr)) - _, errs := p.Parse() - if len(errs) != 1 { - t.Errorf("unexpected number of errors\n got: %v, wanted: 1", errs) - } - if !errors.Is(errs[0], parser.ErrNoEndTagFound) { - t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, parser.ErrNoEndTagFound) - } -} - -func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBlock, tokenType config.ImplementationPrefix, tokenValue string) bool { - t.Helper() - if stmtBlock.BeginToken.ImpPrefix != tokenType { - t.Errorf("got=%q, wanted stmtBlock.ImpPrefix = '%v'.", stmtBlock.BeginToken.Literal, tokenType) - return false - } - - if stmtBlock.ParsedToken.StripPrefix() != tokenValue { - t.Errorf("stmtBlock.Value. got=%s, wanted=%s", stmtBlock.ParsedToken.StripPrefix(), tokenValue) - return false - } - - return true -} - -func Test_ExpandEnvVariables_succeeds(t *testing.T) { +func Test_Parse_should_when_no_End_tag_found(t *testing.T) { ttests := map[string]struct { - input string - expect string - envVar []string + input string }{ - "with single var": { - "some var is $var", - "some var is foo", - []string{"var=foo"}, + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3`, }, - "with multiple var": { - "some var is $var and docs go [here]($DOC_LINK/stuff)", - "some var is foo and docs go [here](https://somestuff.com/stuff)", - []string{"var=foo", "DOC_LINK=https://somestuff.com"}, - }, - "with no vars in content": { - "some var is foo and docs go [here](foo.com/stuff)", - "some var is foo and docs go [here](foo.com/stuff)", - []string{"var=foo", "DOC_LINK=https://somestuff.com"}, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3`, }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - defer os.Clearenv() - got, err := parser.ExpandEnvVariables(tt.input, tt.envVar) - if err != nil { - t.Errorf("expected %v to be ", err) + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + _, errs := p.Parse() + if len(errs) != 1 { + t.Fatalf("unexpected number of errors\n got: %v, wanted: 1", errs) } - if got != tt.expect { - t.Errorf("want: %s, got: %s", got, tt.expect) + if !errors.Is(errs[0], parser.ErrNoEndTagFound) { + t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, parser.ErrNoEndTagFound) } }) } } -func Test_ExpandEnvVariables_fails(t *testing.T) { - - ttests := map[string]struct { - input string - setup func() func() - envVar []string - }{ - "with single var": { - "some var is $var", - func() func() { - return func() { - os.Clearenv() - } - }, - []string{"v=foo"}, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - clear := tt.setup() - defer clear() - _, err := parser.ExpandEnvVariables(tt.input, tt.envVar) - if err == nil { - t.Errorf("wanted error, got ") - } - }) +func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBlock, tokenType config.ImplementationPrefix, tokenValue, keysLookupPath string) bool { + t.Helper() + if stmtBlock.ParsedToken.Prefix() != tokenType { + t.Errorf("got=%q, wanted stmtBlock.ImpPrefix = '%v'.", stmtBlock.ParsedToken.Prefix(), tokenType) + return false } -} -func Test_Parse_WithOwnEnviron_passed_in_succeeds(t *testing.T) { - ttests := map[string]struct { - input string - expect string - environ []string - }{ - "test1": { - input: `let x = 42; -//+gendoc category=message type=description id=foo -this is some description with $foo -//-gendoc`, - environ: []string{"foo=bar"}, - expect: "this is some description with bar", - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - defer os.Clearenv() - lexerSource.Input = tt.input - l := lexer.New(lexerSource, *config.NewConfig()) - p := parser.New(l, &config.GenVarsConfig{}).WithLogger(log.New(os.Stderr)).WithEnvironment(tt.environ) - got, errs := p.Parse() - if len(errs) > 0 { - t.Error(errs) - } - if got[0].Value != tt.expect { - t.Error("") - } - }) + if stmtBlock.ParsedToken.StoreToken() != tokenValue { + t.Errorf("token StoreToken got=%s, wanted=%s", stmtBlock.ParsedToken.StoreToken(), tokenValue) + return false } -} -func Test_Parse_WithOwnEnviron_passed_in_fails(t *testing.T) { - ttests := map[string]struct { - input string - expect error - environ []string - }{ - "if variable is not set": { - input: `let x = 42; - //+gendoc category=message type=description id=foo - this is some description with $foo - //-gendoc`, - expect: parser.ErrUnableToReplaceVarPlaceholder, - environ: []string{"notfoo=bar"}, - }, - "if variable is not set but empty": { - input: `let x = 42; -//+gendoc category=message type=description id=foo -this is some description with $foo -//-gendoc`, - expect: parser.ErrUnableToReplaceVarPlaceholder, - environ: []string{"foo="}, - }, + if stmtBlock.ParsedToken.LookupKeys() != keysLookupPath { + t.Errorf("token LookupKeys. got=%s, wanted=%s", stmtBlock.ParsedToken.LookupKeys(), tokenValue) + return false } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - defer os.Clearenv() - lexerSource.Input = tt.input - l := lexer.New(lexerSource, *config.NewConfig()) - p := parser.New(l, &config.GenVarsConfig{}).WithLogger(log.New(os.Stderr)).WithEnvironment(tt.environ) - _, errs := p.Parse() - if len(errs) < 1 { - t.Error("expected errors to occur") - t.Fail() - } - if !errors.Is(errs[0], tt.expect) { - t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, tt.expect) - } - }) - } + return true } From 1a796b1250ce97bffe9084e41dae5ab6aa3e0627 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Tue, 21 Oct 2025 18:48:06 +0100 Subject: [PATCH 04/19] fix: correct store pkg and tests --- README.md | 4 +- configmanager.go | 2 + configmanager_test.go | 1 - internal/cmdutils/cmdutils_test.go | 4 - internal/config/config.go | 134 +---- internal/config/config_test.go | 102 ++-- internal/parser/parser.go | 6 +- internal/parser/parser_test.go | 73 ++- internal/store/azappconf.go | 10 +- internal/store/azappconf_test.go | 97 +-- internal/store/azhelpers.go | 15 +- internal/store/azkeyvault.go | 10 +- internal/store/azkeyvault_test.go | 188 +++--- internal/store/aztablestorage.go | 10 +- internal/store/aztablestorage_test.go | 298 +++++----- internal/store/gcpsecrets.go | 4 + internal/store/gcpsecrets_test.go | 101 ++-- internal/store/hashivault.go | 26 +- internal/store/hashivault_test.go | 809 ++++++++++++++------------ internal/store/paramstore.go | 4 + internal/store/paramstore_test.go | 112 ++-- internal/store/secretsmanager.go | 4 + internal/store/secretsmanager_test.go | 124 ++-- internal/strategy/strategy_test.go | 82 ++- pkg/generator/generator.go | 38 +- 25 files changed, 1223 insertions(+), 1035 deletions(-) diff --git a/README.md b/README.md index a7762ad..5b73d7a 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,9 @@ See [examples of working with files](docs/examples.md#working-with-files) for mo The `[meta=data]` from the [example token](#awssecretspathtomykeylookupinsideobjectmetadata) - is the optional metadata about the target in the backing provider -IT must have this format `[key=value]` - IT IS OPTIONAL + +> IT must have this format `[key=value]` - IT IS OPTIONAL +> IT must be specified last - either after a path lookup or if there is no key look up path specified then after the full path The `key` and `value` would be provider specific. Meaning that different providers support different config, these values _CAN_ be safely omitted configmanager would just use the defaults where applicable or not specify the additional diff --git a/configmanager.go b/configmanager.go index 748c4b7..de0c2c5 100644 --- a/configmanager.go +++ b/configmanager.go @@ -110,6 +110,8 @@ func (c *ConfigManager) DiscoverTokens(input string) []string { // FindTokens extracts all replaceable tokens // from a given input string +// +// Deprecated: FindTokens relies on Regex. Use DiscoverTokens func FindTokens(input string) []string { tokens := []string{} for k := range config.VarPrefix { diff --git a/configmanager_test.go b/configmanager_test.go index 15bf871..1088b9f 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -171,7 +171,6 @@ foo23 = val1 } func Test_replaceString_with_envsubst(t *testing.T) { - t.Parallel() ttests := map[string]struct { expect string setup func() func() diff --git a/internal/cmdutils/cmdutils_test.go b/internal/cmdutils/cmdutils_test.go index f8fad0f..68e1ab4 100644 --- a/internal/cmdutils/cmdutils_test.go +++ b/internal/cmdutils/cmdutils_test.go @@ -61,8 +61,6 @@ func cmdTestHelper(t *testing.T, err error, got []byte, expect []string) { } func Test_GenerateFromCmd(t *testing.T) { - t.Parallel() - ttests := map[string]struct { mockMap generator.ParsedMap tokens []string @@ -106,8 +104,6 @@ func (m *mockWriter) Write(in []byte) (int, error) { } func Test_GenerateStrOut(t *testing.T) { - t.Parallel() - inputStr := `FOO://bar/qusx FOO://bar/lorem FOO://bar/ducks` mockParsedStr := `aksujg fooLorem Mighty` expect := []string{"aksujg", "fooLorem", "Mighty"} diff --git a/internal/config/config.go b/internal/config/config.go index 88537a2..c66ad14 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -138,38 +138,14 @@ type ParsedTokenConfig struct { prefix ImplementationPrefix // cofig values keySeparator, tokenSeparator string - - fullToken string + // tokenb parts metadataStr string keysPath string sanitizedToken string - // depracated - prefixLessToken string - storeToken, metadataLess string } -// NewParsedTokenConfig returns a pointer to a new TokenConfig struct -// returns nil if current prefix does not correspond to an Implementation -// -// The caller needs to make sure it is not nil -// TODO: a custom parser would be best here -func NewParsedTokenConfig(token string, config GenVarsConfig) (*ParsedTokenConfig, error) { - ptc := &ParsedTokenConfig{} - prfx := strings.Split(token, config.TokenSeparator())[0] - - // This should already only be a list of properly supported tokens but just in case - if found := VarPrefix[ImplementationPrefix(prfx)]; !found { - return nil, fmt.Errorf("prefix: %s\n%w", prfx, ErrInvalidTokenPrefix) - } - - ptc.keySeparator = config.keySeparator - ptc.tokenSeparator = config.tokenSeparator - ptc.prefix = ImplementationPrefix(prfx) - ptc.fullToken = token - return ptc.new(), nil -} - -func NewTokenConfig(prefix ImplementationPrefix, config GenVarsConfig) (*ParsedTokenConfig, error) { +// NewToken initialises a *ParsedTokenConfig +func NewToken(prefix ImplementationPrefix, config GenVarsConfig) (*ParsedTokenConfig, error) { tokenConf := &ParsedTokenConfig{} if err := config.Validate(); err != nil { return nil, err @@ -196,15 +172,16 @@ func (ptc *ParsedTokenConfig) WithSanitizedToken(v string) { // depracated func (ptc *ParsedTokenConfig) new() *ParsedTokenConfig { - // order must be respected here - // - ptc.prefixLessToken = strings.Replace(ptc.fullToken, fmt.Sprintf("%s%s", ptc.prefix, ptc.tokenSeparator), "", 1) - - // token without metadata and the string itself - ptc.extractMetadataStr() - // token without keys - ptc.keysLookup() - return ptc + // // order must be respected here + // // + // ptc.prefixLessToken = strings.Replace(ptc.fullToken, fmt.Sprintf("%s%s", ptc.prefix, ptc.tokenSeparator), "", 1) + + // // token without metadata and the string itself + // ptc.extractMetadataStr() + // // token without keys + // ptc.keysLookup() + // return ptc + return nil } func (t *ParsedTokenConfig) ParseMetadata(metadataTyp any) error { @@ -231,28 +208,11 @@ func (t *ParsedTokenConfig) ParseMetadata(metadataTyp any) error { return nil } -func (t *ParsedTokenConfig) StripPrefix() string { - return t.prefixLessToken -} - -// StripMetadata returns the fullToken without the -// metadata -func (t *ParsedTokenConfig) StripMetadata() string { - return t.metadataLess -} - -// StoreToken -// -// returns the only the store indicator string -// without any of the configmanager token enrichment: -// -// - metadata -// -// - keySeparator -// -// - keys -// -// - prefix +// StoreToken returns the sanitized token without: +// - metadata +// - keySeparator +// - keys +// - prefix func (t *ParsedTokenConfig) StoreToken() string { return t.sanitizedToken } @@ -260,57 +220,25 @@ func (t *ParsedTokenConfig) StoreToken() string { // Full returns the full Token path. // Including key separator and metadata values func (t *ParsedTokenConfig) String() string { - return t.fullToken + token := fmt.Sprintf("%s%s%s", t.prefix, t.tokenSeparator, t.sanitizedToken) + if len(t.keysPath) > 0 { + token += t.keySeparator + t.keysPath + } + + if len(t.metadataStr) > 0 { + token += fmt.Sprintf("[%s]", t.metadataStr) + } + return token } func (t *ParsedTokenConfig) LookupKeys() string { return t.keysPath } -func (t *ParsedTokenConfig) Prefix() ImplementationPrefix { - return t.prefix -} - -const ( - startMetaStr string = `[` - endMetaStr string = `]` -) - -// extractMetadataStr returns anything between the start and end -// metadata markers in the token string itself -// returns the token without meta -func (t *ParsedTokenConfig) extractMetadataStr() { - token := t.prefixLessToken - t.metadataLess = token - startIndex := strings.Index(token, startMetaStr) - // token has no startMetaStr - if startIndex == -1 { - return - } - newS := token[startIndex+len(startMetaStr):] - - endIndex := strings.Index(newS, endMetaStr) - // token has no meta end - if endIndex == -1 { - return - } - // metastring extracted - // complete [key=value] has been found - metaString := newS[:endIndex] - t.metadataStr = metaString - // Set Metadataless token - t.metadataLess = strings.ReplaceAll(token, startMetaStr+metaString+endMetaStr, "") +func (t *ParsedTokenConfig) Metadata() string { + return t.metadataStr } -// keysLookup returns the keysLookup path and the string without it -// -// NOTE: metadata was already stripped at this point -func (t *ParsedTokenConfig) keysLookup() { - keysIndex := strings.Index(t.metadataLess, t.keySeparator) - if keysIndex >= 0 { - t.keysPath = t.metadataLess[keysIndex+len(t.keySeparator):] - t.storeToken = t.metadataLess[:keysIndex] - return - } - t.storeToken = t.metadataLess +func (t *ParsedTokenConfig) Prefix() ImplementationPrefix { + return t.prefix } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6bc6803..488e1b8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -30,69 +30,77 @@ func Test_MarshalMetadata_with_label_struct_succeeds(t *testing.T) { } ttests := map[string]struct { - config *config.GenVarsConfig - rawToken string + token func() *config.ParsedTokenConfig wantLabel string wantMetaStrippedToken string }{ "when provider expects label on token and label exists": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88[label=dev]`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithMetadata("label=dev") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "dev", "basjh/dskjuds/123", }, "when provider expects label on token and label does not exist": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88[someother=dev]`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithMetadata("someother=dev") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "", "basjh/dskjuds/123", }, "no metadata found": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "", "basjh/dskjuds/123", }, "no metadata found incorrect marker placement": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123|d88]asdas=bar[`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88]asdas=bar[") + tkn.WithSanitizedToken("basjh/dskjuds/123") + return tkn + }, "", "basjh/dskjuds/123", }, "no metadata found incorrect marker placement and no key separator": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123]asdas=bar[`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithSanitizedToken("basjh/dskjuds/123]asdas=bar[") + return tkn + }, "", "basjh/dskjuds/123]asdas=bar[", }, - "no end found incorrect marker placement and no key separator": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123[asdas=bar`, - "", - "basjh/dskjuds/123[asdas=bar", - }, "no start found incorrect marker placement and no key separator": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123]asdas=bar]`, + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("://")) + tkn.WithKeyPath("d88") + tkn.WithMetadata("someother=dev") + tkn.WithSanitizedToken("basjh/dskjuds/123]asdas=bar]") + return tkn + }, "", "basjh/dskjuds/123]asdas=bar]", }, - "metadata is in the middle of path lookup": { - config.NewConfig().WithTokenSeparator("://"), - `AZTABLESTORE://basjh/dskjuds/123[label=bar]|lookup`, - "bar", - "basjh/dskjuds/123", - }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { inputTyp := &labelMeta{} - got, err := config.NewParsedTokenConfig(tt.rawToken, *tt.config) - - if err != nil { - t.Fatalf("got an error on NewParsedTokenconfig (%s)\n", tt.rawToken) - } - + got := tt.token() if got == nil { t.Errorf(testutils.TestPhraseWithContext, "Unable to parse token", nil, config.ParsedTokenConfig{}) } @@ -100,7 +108,7 @@ func Test_MarshalMetadata_with_label_struct_succeeds(t *testing.T) { got.ParseMetadata(inputTyp) if got.StoreToken() != tt.wantMetaStrippedToken { - t.Errorf(testutils.TestPhraseWithContext, "Token does not match", got.StripMetadata(), tt.wantMetaStrippedToken) + t.Errorf(testutils.TestPhraseWithContext, "Token does not match", got.StoreToken(), tt.wantMetaStrippedToken) } if inputTyp.Label != tt.wantLabel { @@ -115,23 +123,27 @@ func Test_TokenParser_config(t *testing.T) { Version string `json:"version"` } ttests := map[string]struct { - input string - expPrefix config.ImplementationPrefix - expLookupKeys string - expStoreToken string - expString string // fullToken - expMetadataVersion string + rawToken, keyPath, metadataStr string + expPrefix config.ImplementationPrefix + expLookupKeys string + expStoreToken string // sanitised + expString string // fullToken + expMetadataVersion string }{ - "bare": {"AWSSECRETS://foo/bar", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar", ""}, - "with metadata version": {"AWSSECRETS://foo/bar[version=123]", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar[version=123]", "123"}, - "with keys lookup and label": {"AWSSECRETS://foo/bar|key1.key2[version=123]", config.SecretMgrPrefix, "key1.key2", "foo/bar", "AWSSECRETS://foo/bar|key1.key2[version=123]", "123"}, - "with keys lookup and longer token": {"AWSSECRETS://foo/bar|key1.key2]version=123]", config.SecretMgrPrefix, "key1.key2]version=123]", "foo/bar", "AWSSECRETS://foo/bar|key1.key2]version=123]", ""}, - "with keys lookup but no keys": {"AWSSECRETS://foo/bar/sdf/sddd.90dsfsd|[version=123]", config.SecretMgrPrefix, "", "foo/bar/sdf/sddd.90dsfsd", "AWSSECRETS://foo/bar/sdf/sddd.90dsfsd|[version=123]", "123"}, + "bare": {"foo/bar", "", "", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar", ""}, + "with metadata version": {"foo/bar", "", "version=123", config.SecretMgrPrefix, "", "foo/bar", "AWSSECRETS://foo/bar[version=123]", "123"}, + "with keys lookup and label": {"foo/bar", "key1.key2", "version=123", config.SecretMgrPrefix, "key1.key2", "foo/bar", "AWSSECRETS://foo/bar|key1.key2[version=123]", "123"}, + "with keys lookup and longer token": {"foo/bar", "key1.key2]version=123]", "", config.SecretMgrPrefix, "key1.key2]version=123]", "foo/bar", "AWSSECRETS://foo/bar|key1.key2]version=123]", ""}, + "with keys lookup but no keys": {"foo/bar/sdf/sddd.90dsfsd", "", "version=123", config.SecretMgrPrefix, "", "foo/bar/sdf/sddd.90dsfsd", "AWSSECRETS://foo/bar/sdf/sddd.90dsfsd[version=123]", "123"}, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { conf := &mockConfAwsSecrMgr{} - got, _ := config.NewParsedTokenConfig(tt.input, *config.NewConfig()) + got, _ := config.NewToken(tt.expPrefix, *config.NewConfig()) + got.WithSanitizedToken(tt.rawToken) + got.WithKeyPath(tt.keyPath) + got.WithMetadata(tt.metadataStr) + got.ParseMetadata(conf) if got.LookupKeys() != tt.expLookupKeys { diff --git a/internal/parser/parser.go b/internal/parser/parser.go index f93dffc..c7561a7 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -80,7 +80,7 @@ func (p *Parser) Parse() ([]ConfigManagerTokenBlock, []error) { if p.currentTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { // parseGenDocBlocks will advance the token until // it hits the END_DOC_GEN token - configManagerToken, err := config.NewTokenConfig(p.curToken.ImpPrefix, *p.config) + configManagerToken, err := config.NewToken(p.curToken.ImpPrefix, *p.config) if err != nil { return nil, []error{err} } @@ -185,6 +185,7 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // keyPath would have built the keyPath and metadata if any break } + // optionally at the end of the path without key separator // check metadata there can be a metadata bracket `[key=val,k1=v2]` if p.currentTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { @@ -192,6 +193,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa p.errors = append(p.errors, err) return nil } + notFoundEnd = false + break } sanitizedToken += p.curToken.Literal fullToken += p.curToken.Literal @@ -238,6 +241,7 @@ func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) err // errNoClose := fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) metadata := "" found := false + p.nextToken() for !p.peekTokenIs(config.EOF) { if p.peekTokenIsEnd() { return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 222d869..66ef3c4 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -9,6 +9,7 @@ import ( "github.com/DevLabFoundry/configmanager/v2/internal/lexer" "github.com/DevLabFoundry/configmanager/v2/internal/log" "github.com/DevLabFoundry/configmanager/v2/internal/parser" + "github.com/DevLabFoundry/configmanager/v2/internal/store" ) var lexerSource = lexer.Source{FileName: "bar", FullPath: "/foo/bar"} @@ -66,7 +67,7 @@ BAR=something } } -func Test_Parse_should_when_no_End_tag_found(t *testing.T) { +func Test_Parse_should_faile_when_no_metadata_end_tag_found(t *testing.T) { ttests := map[string]struct { input string }{ @@ -94,6 +95,76 @@ func Test_Parse_should_when_no_End_tag_found(t *testing.T) { } } +func Test_Parse_should_pass_with_metadata_end_tag(t *testing.T) { + ttests := map[string]struct { + input string + metdataStr string + }{ + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3]`, + `version=1.2.3`, + }, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3]`, + `version=1.2.3`, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("unexpected number of errors\n got: %v, wanted: 0", errs) + } + for _, prsd := range parsed { + prsd.ParsedToken.LookupKeys() + + } + }) + } +} + +func Test_Parse_ParseMetadata(t *testing.T) { + + ttests := map[string]struct { + input string + typ *store.SecretsMgrConfig + }{ + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3]`, + &store.SecretsMgrConfig{}, + }, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3]`, + &store.SecretsMgrConfig{}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("%v", errs) + } + + for _, p := range parsed { + if err := p.ParsedToken.ParseMetadata(tt.typ); err != nil { + t.Fatal(err) + } + if tt.typ.Version != "1.2.3" { + t.Errorf("got %v wanted 1.2.3", tt.typ.Version) + } + } + }) + } +} + func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBlock, tokenType config.ImplementationPrefix, tokenValue, keysLookupPath string) bool { t.Helper() if stmtBlock.ParsedToken.Prefix() != tokenType { diff --git a/internal/store/azappconf.go b/internal/store/azappconf.go index a37cc8a..92adab5 100644 --- a/internal/store/azappconf.go +++ b/internal/store/azappconf.go @@ -51,8 +51,8 @@ func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger l token: token, logger: logger, } - srvInit := azServiceFromToken(token.StoreToken(), "https://%s.azconfig.io", 1) - backingStore.strippedToken = srvInit.token + srvInit := AzServiceFromToken(token.StoreToken(), "https://%s.azconfig.io", 1) + backingStore.strippedToken = srvInit.Token cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -60,7 +60,7 @@ func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger l return nil, err } - c, err := azappconfig.NewClient(srvInit.serviceUri, cred, nil) + c, err := azappconfig.NewClient(srvInit.ServiceUri, cred, nil) if err != nil { logger.Error("failed to init the client: %v", err) return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) @@ -71,6 +71,10 @@ func NewAzAppConf(ctx context.Context, token *config.ParsedTokenConfig, logger l } +func (s *AzAppConf) WithSvc(svc appConfApi) { + s.svc = svc +} + // setTokenVal sets the token func (implmt *AzAppConf) SetToken(token *config.ParsedTokenConfig) {} diff --git a/internal/store/azappconf_test.go b/internal/store/azappconf_test.go index 82ed7e9..eff0872 100644 --- a/internal/store/azappconf_test.go +++ b/internal/store/azappconf_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "bytes" @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" "github.com/DevLabFoundry/configmanager/v2/internal/config" logger "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" ) @@ -36,20 +37,25 @@ func (m mockAzAppConfApi) GetSetting(ctx context.Context, key string, options *a } func Test_AzAppConf_Success(t *testing.T) { - t.Parallel() tsuccessParam := "somecvla" logr := logger.New(&bytes.Buffer{}) tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) appConfApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzAppConfApi }{ "successVal": { - "AZAPPCONF#/test-app-config-instance/table//token/1", + func() *config.ParsedTokenConfig { + // "AZAPPCONF#/test-app-config-instance/table//token/1", + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/table//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, tsuccessParam, - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "table//token/1", "", options) resp := azappconfig.GetSettingResponse{} @@ -57,12 +63,18 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), }, "successVal with :// token Separator": { - "AZAPPCONF:///test-app-config-instance/conf_key[label=dev]", + func() *config.ParsedTokenConfig { + // "AZAPPCONF:///test-app-config-instance/conf_key[label=dev]", + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://")) + tkn.WithSanitizedToken("/test-app-config-instance/conf_key") + tkn.WithKeyPath("") + tkn.WithMetadata("label=dev") + return tkn + }, tsuccessParam, - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "conf_key", "dev", options) resp := azappconfig.GetSettingResponse{} @@ -70,12 +82,17 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), }, "successVal with :// token Separator and etag specified": { - "AZAPPCONF:///test-app-config-instance/conf_key[label=dev,etag=sometifdsssdsfdi_string01209222]", + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/conf_key") + tkn.WithKeyPath("") + tkn.WithMetadata("label=dev,etag=sometifdsssdsfdi_string01209222") + return tkn + }, tsuccessParam, - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "conf_key", "dev", options) if !options.OnlyIfChanged.Equals("sometifdsssdsfdi_string01209222") { @@ -86,12 +103,17 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), }, "successVal with keyseparator but no val returned": { - "AZAPPCONF#/test-app-config-instance/try_to_find|key_separator.lookup", + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/try_to_find") + tkn.WithKeyPath("key_separator.lookup") + tkn.WithMetadata("") + return tkn + }, "", - func(t *testing.T) appConfApi { + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { azAppConfCommonChecker(t, key, "try_to_find", "", options) resp := azappconfig.GetSettingResponse{} @@ -99,20 +121,17 @@ func Test_AzAppConf_Success(t *testing.T) { return resp, nil }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - - impl, err := NewAzAppConf(context.TODO(), token, logr) + impl, err := store.NewAzAppConf(context.TODO(), tt.token(), logr) if err != nil { t.Errorf("failed to init AZAPPCONF") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) got, err := impl.Token() if err != nil { if err.Error() != tt.expect { @@ -129,38 +148,40 @@ func Test_AzAppConf_Success(t *testing.T) { } func Test_AzAppConf_Error(t *testing.T) { - t.Parallel() - logr := logger.New(&bytes.Buffer{}) tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect error - mockClient func(t *testing.T) appConfApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzAppConfApi }{ "errored on service method call": { - "AZAPPCONF#/test-app-config-instance/table/token/ok", - ErrRetrieveFailed, - func(t *testing.T) appConfApi { + func() *config.ParsedTokenConfig { + // "AZAPPCONF#/test-app-config-instance/table/token/ok", + tkn, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-app-config-instance/table/token/ok") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + store.ErrRetrieveFailed, + func(t *testing.T) mockAzAppConfApi { return mockAzAppConfApi(func(ctx context.Context, key string, options *azappconfig.GetSettingOptions) (azappconfig.GetSettingResponse, error) { t.Helper() resp := azappconfig.GetSettingResponse{} return resp, fmt.Errorf("network error") }) }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - impl, err := NewAzAppConf(context.TODO(), token, logr) + impl, err := store.NewAzAppConf(context.TODO(), tt.token(), logr) if err != nil { t.Fatal("failed to init AZAPPCONF") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) if _, err := impl.Token(); !errors.Is(err, tt.expect) { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) } @@ -169,19 +190,19 @@ func Test_AzAppConf_Error(t *testing.T) { } func Test_fail_AzAppConf_Client_init(t *testing.T) { - t.Parallel() logr := logger.New(&bytes.Buffer{}) // this is basically a wrap around test for the url.Parse method in the stdlib // as that is what the client uses under the hood - token, _ := config.NewParsedTokenConfig("AZAPPCONF:///%25%65%6e%301-._~/") } - if !errors.Is(err, ErrClientInitialization) { - t.Fatalf(testutils.TestPhraseWithContext, "azappconf client init", err.Error(), ErrClientInitialization.Error()) + if !errors.Is(err, store.ErrClientInitialization) { + t.Fatalf(testutils.TestPhraseWithContext, "azappconf client init", err.Error(), store.ErrClientInitialization.Error()) } } diff --git a/internal/store/azhelpers.go b/internal/store/azhelpers.go index 7b85387..29e66e8 100644 --- a/internal/store/azhelpers.go +++ b/internal/store/azhelpers.go @@ -8,13 +8,14 @@ import ( /* Generic Azure Service Init Helpers */ -// azServiceHelper returns a service URI and the stripped token -type azServiceHelper struct { - serviceUri string - token string + +// AzServiceHelper returns a service URI and the stripped token +type AzServiceHelper struct { + ServiceUri string + Token string } -// azServiceFromToken for azure the first part of the token __must__ always be the +// AzServiceFromToken for azure the first part of the token __must__ always be the // identifier of the service e.g. the account name for tableStore or the Vault name for KVSecret or // AppConfig instance // take parameter specifies the number of elements to take from the start only @@ -22,7 +23,7 @@ type azServiceHelper struct { // e.g. a value of 2 for take will take first 2 elements from the slices // // For AppConfig or KeyVault we ONLY need the AppConfig instance or KeyVault instance name -func azServiceFromToken(token string, formatUri string, take int) azServiceHelper { +func AzServiceFromToken(token string, formatUri string, take int) AzServiceHelper { // ensure preceding slash is trimmed stringToken := strings.Split(strings.TrimPrefix(token, "/"), "/") splitToken := []any{} @@ -32,5 +33,5 @@ func azServiceFromToken(token string, formatUri string, take int) azServiceHelpe } uri := fmt.Sprintf(formatUri, splitToken[0:take]...) - return azServiceHelper{serviceUri: uri, token: strings.Join(stringToken[take:], "/")} + return AzServiceHelper{ServiceUri: uri, Token: strings.Join(stringToken[take:], "/")} } diff --git a/internal/store/azkeyvault.go b/internal/store/azkeyvault.go index 84f1715..bd2a2a4 100644 --- a/internal/store/azkeyvault.go +++ b/internal/store/azkeyvault.go @@ -45,8 +45,8 @@ func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger token: token, } - srvInit := azServiceFromToken(token.StoreToken(), "https://%s.vault.azure.net", 1) - backingStore.strippedToken = srvInit.token + srvInit := AzServiceFromToken(token.StoreToken(), "https://%s.vault.azure.net", 1) + backingStore.strippedToken = srvInit.Token cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -54,7 +54,7 @@ func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger return nil, err } - c, err := azsecrets.NewClient(srvInit.serviceUri, cred, nil) + c, err := azsecrets.NewClient(srvInit.ServiceUri, cred, nil) if err != nil { logger.Error("%v\n%w", err, ErrClientInitialization) return nil, err @@ -65,6 +65,10 @@ func NewKvScrtStore(ctx context.Context, token *config.ParsedTokenConfig, logger } +func (s *KvScrtStore) WithSvc(svc kvApi) { + s.svc = svc +} + // setToken already happens in AzureKVClient in the constructor func (implmt *KvScrtStore) SetToken(token *config.ParsedTokenConfig) {} diff --git a/internal/store/azkeyvault_test.go b/internal/store/azkeyvault_test.go index 98ccda8..f4562ab 100644 --- a/internal/store/azkeyvault_test.go +++ b/internal/store/azkeyvault_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" ) @@ -17,49 +18,49 @@ func Test_azSplitToken(t *testing.T) { tests := []struct { name string token string - expect azServiceHelper + expect store.AzServiceHelper }{ { name: "simple_with_preceding_slash", token: "/test-vault/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "somejsontest", }, }, { name: "missing_initial_slash", token: "test-vault/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "somejsontest", }, }, { name: "missing_initial_slash_multislash_secretname", token: "test-vault/some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "some/json/test", }, }, { name: "with_initial_slash_multislash_secretname", token: "test-vault//some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-vault.vault.azure.net", - token: "/some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-vault.vault.azure.net", + Token: "/some/json/test", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := azServiceFromToken(tt.token, "https://%s.vault.azure.net", 1) - if got.token != tt.expect.token { - t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) + got := store.AzServiceFromToken(tt.token, "https://%s.vault.azure.net", 1) + if got.Token != tt.expect.Token { + t.Errorf(testutils.TestPhrase, tt.expect.Token, got.Token) } - if got.serviceUri != tt.expect.serviceUri { - t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) + if got.ServiceUri != tt.expect.ServiceUri { + t.Errorf(testutils.TestPhrase, tt.expect.ServiceUri, got.ServiceUri) } }) } @@ -93,81 +94,114 @@ func (m mockAzKvSecretApi) GetSecret(ctx context.Context, name string, version s } func TestAzKeyVault(t *testing.T) { - t.Parallel() - tsuccessParam := "dssdfdweiuyh" tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) kvApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzKvSecretApi }{ - "successVal": {"AZKVSECRET#/test-vault//token/1", tsuccessParam, func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "/token/1") - resp := azsecrets.GetSecretResponse{} - resp.Value = &tsuccessParam - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "successVal with version": {"AZKVSECRET#/test-vault//token/1[version:123]", tsuccessParam, func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "/token/1") - resp := azsecrets.GetSecretResponse{} - resp.Value = &tsuccessParam - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "successVal with keyseparator": {"AZKVSECRET#/test-vault/token/1|somekey", tsuccessParam, func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "token/1") - - resp := azsecrets.GetSecretResponse{} - resp.Value = &tsuccessParam - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "successVal": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + tsuccessParam, func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "/token/1") + resp := azsecrets.GetSecretResponse{} + resp.Value = &tsuccessParam + return resp, nil + }) + }, }, - "errored": {"AZKVSECRET#/test-vault/token/1|somekey", "unable to retrieve secret", func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "token/1") - - resp := azsecrets.GetSecretResponse{} - return resp, fmt.Errorf("unable to retrieve secret") - }) + "successVal with version": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version:123") + return tkn + }, tsuccessParam, func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "/token/1") + resp := azsecrets.GetSecretResponse{} + resp.Value = &tsuccessParam + return resp, nil + }) + }, }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "successVal with keyseparator": { + func() *config.ParsedTokenConfig { + // "AZKVSECRET#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, tsuccessParam, func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "token/1") + + resp := azsecrets.GetSecretResponse{} + resp.Value = &tsuccessParam + return resp, nil + }) + }, }, - "empty": {"AZKVSECRET#/test-vault/token/1|somekey", "", func(t *testing.T) kvApi { - return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { - t.Helper() - azKvCommonGetSecretChecker(t, name, "", "token/1") - - resp := azsecrets.GetSecretResponse{} - return resp, nil - }) + "errored": { + func() *config.ParsedTokenConfig { + // "AZKVSECRET#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, + "unable to retrieve secret", + func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "token/1") + + resp := azsecrets.GetSecretResponse{} + return resp, fmt.Errorf("unable to retrieve secret") + }) + }, }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "empty": { + func() *config.ParsedTokenConfig { + // "AZKVSECRET#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, "", func(t *testing.T) mockAzKvSecretApi { + return mockAzKvSecretApi(func(ctx context.Context, name string, version string, options *azsecrets.GetSecretOptions) (azsecrets.GetSecretResponse, error) { + t.Helper() + azKvCommonGetSecretChecker(t, name, "", "token/1") + + resp := azsecrets.GetSecretResponse{} + return resp, nil + }) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - - impl, err := NewKvScrtStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewKvScrtStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Errorf("failed to init azkvstore") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) got, err := impl.Token() if err != nil { if err.Error() != tt.expect { diff --git a/internal/store/aztablestorage.go b/internal/store/aztablestorage.go index 539979b..c9bb275 100644 --- a/internal/store/aztablestorage.go +++ b/internal/store/aztablestorage.go @@ -52,8 +52,8 @@ func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logge token: token, } - srvInit := azServiceFromToken(token.StoreToken(), "https://%s.table.core.windows.net/%s", 2) - backingStore.strippedToken = srvInit.token + srvInit := AzServiceFromToken(token.StoreToken(), "https://%s.table.core.windows.net/%s", 2) + backingStore.strippedToken = srvInit.Token cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -61,7 +61,7 @@ func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logge return nil, err } - c, err := aztables.NewClient(srvInit.serviceUri, cred, nil) + c, err := aztables.NewClient(srvInit.ServiceUri, cred, nil) if err != nil { logger.Error("failed to init the client: %v", err) return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) @@ -71,6 +71,10 @@ func NewAzTableStore(ctx context.Context, token *config.ParsedTokenConfig, logge return backingStore, nil } +func (s *AzTableStore) WithSvc(svc tableStoreApi) { + s.svc = svc +} + // setToken already happens in the constructor func (implmt *AzTableStore) SetToken(token *config.ParsedTokenConfig) {} diff --git a/internal/store/aztablestorage_test.go b/internal/store/aztablestorage_test.go index 54006cc..64d58b4 100644 --- a/internal/store/aztablestorage_test.go +++ b/internal/store/aztablestorage_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" ) @@ -41,54 +42,61 @@ func (m mockAzTableStoreApi) GetEntity(ctx context.Context, partitionKey string, func Test_AzTableStore_Success(t *testing.T) { tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) tableStoreApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzTableStoreApi }{ - "successVal": {"AZTABLESTORE#/test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") - resp := aztables.GetEntityResponse{} - resp.Value = []byte("tsuccessParam") - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "successVal with :// token Separator": {"AZTABLESTORE:///test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") - resp := aztables.GetEntityResponse{} - resp.Value = []byte("tsuccessParam") - return resp, nil - }) - }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), + "successVal": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-account/table//token/1" + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-account/table//token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "tsuccessParam", func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + resp := aztables.GetEntityResponse{} + resp.Value = []byte("tsuccessParam") + return resp, nil + }) + }, }, - "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", "", func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + // "successVal with :// token Separator": {"AZTABLESTORE:///test-account/table//token/1", "tsuccessParam", func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") + // resp := aztables.GetEntityResponse{} + // resp.Value = []byte("tsuccessParam") + // return resp, nil + // }) + // }, config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://"), + // }, + // "successVal with keyseparator but no val returned": {"AZTABLESTORE#/test-account/table/token/1|somekey", "", func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // azTableStoreCommonChecker(t, partitionKey, rowKey, "token", "1") - resp := aztables.GetEntityResponse{} - resp.Value = nil - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, + // resp := aztables.GetEntityResponse{} + // resp.Value = nil + // return resp, nil + // }) + // }, + // config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + // }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewAzTableStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Errorf("failed to init aztablestore") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) + got, err := impl.Token() if err != nil { if err.Error() != tt.expect { @@ -105,73 +113,78 @@ func Test_AzTableStore_Success(t *testing.T) { } func Test_azstorage_with_value_property(t *testing.T) { - t.Parallel() + conf := config.NewConfig().WithKeySeparator("|").WithTokenSeparator("://") ttests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) tableStoreApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzTableStoreApi }{ "return value property with json like object": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey|host", + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey|host", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + tkn.WithKeyPath("host") + tkn.WithMetadata("version:123]") + return tkn + }, "map[bool:true host:foo port:1234]", - func(t *testing.T) tableStoreApi { + func(t *testing.T) mockAzTableStoreApi { return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { t.Helper() resp := aztables.GetEntityResponse{Value: []byte(`{"value":{"host":"foo","port":1234,"bool":true}}`)} return resp, nil }) }, - conf, - }, - "return value property with string only": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey", - "foo.bar.com", - func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} - return resp, nil - }) - }, - conf, - }, - "return value property with numeric only": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey", - "1234", - func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} - return resp, nil - }) - }, - conf, - }, - "return value property with boolean only": { - "AZTABLESTORE:///test-account/table/partitionkey/rowKey", - "false", - func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} - return resp, nil - }) - }, - conf, }, + // "return value property with string only": { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + // "foo.bar.com", + // func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} + // return resp, nil + // }) + // }, + // conf, + // }, + // "return value property with numeric only": { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + // "1234", + // func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} + // return resp, nil + // }) + // }, + // conf, + // }, + // "return value property with boolean only": { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + // "false", + // func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} + // return resp, nil + // }) + // }, + // conf, + // }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) + // token, _ := config.NewToken(tt.token(), *tt.config) - impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewAzTableStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Fatal("failed to init aztablestore") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) got, err := impl.Token() if err != nil { @@ -186,54 +199,57 @@ func Test_azstorage_with_value_property(t *testing.T) { } func Test_AzTableStore_Error(t *testing.T) { - t.Parallel() tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect error - mockClient func(t *testing.T) tableStoreApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockAzTableStoreApi }{ - "errored on token parsing to partiationKey": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{} - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, - "errored on service method call": {"AZTABLESTORE#/test-account/table/token/ok", ErrRetrieveFailed, func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{} - return resp, fmt.Errorf("network error") - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + "errored on token parsing to partiationKey": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-vault/token/1|somekey" + tkn, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, store.ErrIncorrectlyStructuredToken, func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, nil + }) + }, }, + // "errored on service method call": {"AZTABLESTORE#/test-account/table/token/ok", ErrRetrieveFailed, func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // resp := aztables.GetEntityResponse{} + // return resp, fmt.Errorf("network error") + // }) + // }, + // config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + // }, - "empty": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { - return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - t.Helper() - resp := aztables.GetEntityResponse{} - return resp, nil - }) - }, - config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - }, + // "empty": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { + // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + // t.Helper() + // resp := aztables.GetEntityResponse{} + // return resp, nil + // }) + // }, + // config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), + // }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - - impl, err := NewAzTableStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewAzTableStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { t.Fatal("failed to init aztablestore") } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) if _, err := impl.Token(); !errors.Is(err, tt.expect) { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) } @@ -244,66 +260,66 @@ func Test_AzTableStore_Error(t *testing.T) { func Test_fail_AzTable_Client_init(t *testing.T) { // this is basically a wrap around test for the url.Parse method in the stdlib // as that is what the client uses under the hood - token, _ := config.NewParsedTokenConfig("AZTABLESTORE:///%25%65%6e%301-._~/") } - if !errors.Is(err, ErrClientInitialization) { - t.Fatalf(testutils.TestPhraseWithContext, "aztables client init", err.Error(), ErrClientInitialization.Error()) + if !errors.Is(err, store.ErrClientInitialization) { + t.Fatalf(testutils.TestPhraseWithContext, "aztables client init", err.Error(), store.ErrClientInitialization.Error()) } } func Test_azSplitTokenTableStore(t *testing.T) { - t.Parallel() tests := []struct { name string token string - expect azServiceHelper + expect store.AzServiceHelper }{ { name: "simple_with_preceding_slash", token: "/test-account/tablename/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "somejsontest", }, }, { name: "missing_initial_slash", token: "test-account/tablename/somejsontest", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "somejsontest", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "somejsontest", }, }, { name: "missing_initial_slash_multislash_secretname", token: "test-account/tablename/some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "some/json/test", }, }, { name: "with_initial_slash_multislash_secretname", token: "test-account/tablename//some/json/test", - expect: azServiceHelper{ - serviceUri: "https://test-account.table.core.windows.net/tablename", - token: "/some/json/test", + expect: store.AzServiceHelper{ + ServiceUri: "https://test-account.table.core.windows.net/tablename", + Token: "/some/json/test", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := azServiceFromToken(tt.token, "https://%s.table.core.windows.net/%s", 2) - if got.token != tt.expect.token { - t.Errorf(testutils.TestPhrase, tt.expect.token, got.token) + got := store.AzServiceFromToken(tt.token, "https://%s.table.core.windows.net/%s", 2) + if got.Token != tt.expect.Token { + t.Errorf(testutils.TestPhrase, tt.expect.Token, got.Token) } - if got.serviceUri != tt.expect.serviceUri { - t.Errorf(testutils.TestPhrase, tt.expect.serviceUri, got.serviceUri) + if got.ServiceUri != tt.expect.ServiceUri { + t.Errorf(testutils.TestPhrase, tt.expect.ServiceUri, got.ServiceUri) } }) } diff --git a/internal/store/gcpsecrets.go b/internal/store/gcpsecrets.go index 1df7199..67cacec 100644 --- a/internal/store/gcpsecrets.go +++ b/internal/store/gcpsecrets.go @@ -42,6 +42,10 @@ func NewGcpSecrets(ctx context.Context, logger log.ILogger) (*GcpSecrets, error) }, nil } +func (s *GcpSecrets) WithSvc(svc gcpSecretsApi) { + s.svc = svc +} + func (imp *GcpSecrets) SetToken(token *config.ParsedTokenConfig) { storeConf := &GcpSecretsConfig{} _ = token.ParseMetadata(storeConf) diff --git a/internal/store/gcpsecrets_test.go b/internal/store/gcpsecrets_test.go index 54bdf7b..d328650 100644 --- a/internal/store/gcpsecrets_test.go +++ b/internal/store/gcpsecrets_test.go @@ -1,4 +1,4 @@ -package store +package store_test import ( "context" @@ -11,6 +11,7 @@ import ( gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" "github.com/googleapis/gax-go/v2" ) @@ -21,6 +22,10 @@ func (m mockGcpSecretsApi) AccessSecretVersion(ctx context.Context, req *gcpsecr return m(ctx, req, opts...) } +func (m mockGcpSecretsApi) Close() error { + return nil +} + var TEST_GCP_CREDS = []byte(`{ "type": "service_account", "project_id": "xxxxx", @@ -74,49 +79,76 @@ func gcpSecretsGetChecker(t *testing.T, req *gcpsecretspb.AccessSecretVersionReq } func Test_GetGcpSecretVarHappy(t *testing.T) { - // t.Parallel() tests := map[string]struct { - token string + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) gcpSecretsApi - config *config.GenVarsConfig + mockClient func(t *testing.T) mockGcpSecretsApi }{ - "success": {"GCPSECRETS#/token/1", "someValue", func(t *testing.T) gcpSecretsApi { - return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { - gcpSecretsGetChecker(t, req) - return &gcpsecretspb.AccessSecretVersionResponse{ - Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, - }, nil - }) - }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), + "success": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "someValue", func(t *testing.T) mockGcpSecretsApi { + return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { + gcpSecretsGetChecker(t, req) + return &gcpsecretspb.AccessSecretVersionResponse{ + Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, + }, nil + }) + }, }, - "success with version": {"GCPSECRETS#/token/1[version=123]", "someValue", func(t *testing.T) gcpSecretsApi { - return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { - gcpSecretsGetChecker(t, req) - return &gcpsecretspb.AccessSecretVersionResponse{ - Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, - }, nil - }) - }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), + "success with version": { + func() *config.ParsedTokenConfig { + // "GCPSECRETS#/token/1[version=123]" + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version=123") + return tkn + }, "someValue", func(t *testing.T) mockGcpSecretsApi { + return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { + gcpSecretsGetChecker(t, req) + return &gcpsecretspb.AccessSecretVersionResponse{ + Payload: &gcpsecretspb.SecretPayload{Data: []byte("someValue")}, + }, nil + }) + }, }, - "error": {"GCPSECRETS#/token/1", "unable to retrieve secret", func(t *testing.T) gcpSecretsApi { - return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { - gcpSecretsGetChecker(t, req) - return nil, fmt.Errorf("unable to retrieve secret") - }) - }, config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), + "error": { + func() *config.ParsedTokenConfig { + // "GCPSECRETS#/token/1" + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "unable to retrieve secret", func(t *testing.T) mockGcpSecretsApi { + return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { + gcpSecretsGetChecker(t, req) + return nil, fmt.Errorf("unable to retrieve secret") + }) + }, }, "found but empty": { - "GCPSECRETS#/token/1", + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.GcpSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "", - func(t *testing.T) gcpSecretsApi { + func(t *testing.T) mockGcpSecretsApi { return mockGcpSecretsApi(func(ctx context.Context, req *gcpsecretspb.AccessSecretVersionRequest, opts ...gax.CallOption) (*gcpsecretspb.AccessSecretVersionResponse, error) { gcpSecretsGetChecker(t, req) return &gcpsecretspb.AccessSecretVersionResponse{}, nil }) }, - config.NewConfig().WithTokenSeparator("#").WithKeySeparator("|"), }, } for name, tt := range tests { @@ -126,17 +158,16 @@ func Test_GetGcpSecretVarHappy(t *testing.T) { defer fixture.delete(fixture.name) os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fixture.name) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) - impl, err := NewGcpSecrets(context.TODO(), log.New(io.Discard)) + impl, err := store.NewGcpSecrets(context.TODO(), log.New(io.Discard)) if err != nil { t.Errorf(testutils.TestPhrase, err.Error(), nil) } - impl.svc = tt.mockClient(t) - impl.close = func() error { return nil } - impl.SetToken(token) + impl.WithSvc(tt.mockClient(t)) + + impl.SetToken(tt.token()) got, err := impl.Token() if err != nil { diff --git a/internal/store/hashivault.go b/internal/store/hashivault.go index 048039b..6a505e2 100644 --- a/internal/store/hashivault.go +++ b/internal/store/hashivault.go @@ -15,10 +15,10 @@ import ( auth "github.com/hashicorp/vault/api/auth/aws" ) -// vaultHelper provides a broken up string -type vaultHelper struct { - path string - token string +// HashiVaultHelper provides a broken up string +type HashiVaultHelper struct { + Path string + Token string } type hashiVaultApi interface { @@ -52,8 +52,8 @@ func NewVaultStore(ctx context.Context, token *config.ParsedTokenConfig, logger } config := vault.DefaultConfig() - vt := splitToken(token.StoreToken()) - imp.strippedToken = vt.token + vt := SplitHashiVaultToken(token.StoreToken()) + imp.strippedToken = vt.Token client, err := vault.NewClient(config) if err != nil { return nil, fmt.Errorf("%v\n%w", err, ErrClientInitialization) @@ -66,10 +66,14 @@ func NewVaultStore(ctx context.Context, token *config.ParsedTokenConfig, logger } client = awsclient } - imp.svc = client.KVv2(vt.path) + imp.svc = client.KVv2(vt.Path) return imp, nil } +func (s *VaultStore) WithSvc(svc hashiVaultApi) { + s.svc = svc +} + // newVaultStoreWithAWSAuthIAM returns an initialised client with AWSIAMAuth // EC2 auth type is not supported currently func newVaultStoreWithAWSAuthIAM(client *vault.Client, role string) (*vault.Client, error) { @@ -145,14 +149,14 @@ func (imp *VaultStore) getSecret(ctx context.Context, token string, version stri return imp.svc.Get(ctx, token) } -func splitToken(token string) vaultHelper { - vh := vaultHelper{} +func SplitHashiVaultToken(token string) HashiVaultHelper { + vh := HashiVaultHelper{} // split token to extract the mount path s := strings.Split(strings.TrimPrefix(token, "/"), "___") // grab token and trim prefix if slash - vh.token = strings.TrimPrefix(strings.Join(s[1:], ""), "/") + vh.Token = strings.TrimPrefix(strings.Join(s[1:], ""), "/") // assign mount path as extracted from input token - vh.path = s[0] + vh.Path = s[0] return vh } diff --git a/internal/store/hashivault_test.go b/internal/store/hashivault_test.go index 71d010a..90e35f7 100644 --- a/internal/store/hashivault_test.go +++ b/internal/store/hashivault_test.go @@ -1,39 +1,70 @@ -package store +package store_test import ( "context" - "fmt" "io" - "net/http" - "net/http/httptest" "os" - "strings" "testing" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" vault "github.com/hashicorp/vault/api" ) func TestMountPathExtract(t *testing.T) { ttests := map[string]struct { - token string - tokenSeparator string - keySeparator string - expect string + token func() *config.ParsedTokenConfig + expect string }{ - "without leading slash": {"VAULT://secret___/demo/configmanager", "://", "|", "secret"}, - "with leading slash": {"VAULT:///secret___/demo/configmanager", "://", "|", "secret"}, - "with underscore in path name": {"VAULT://_secret___/demo/configmanager", "://", "|", "_secret"}, - "with double underscore in path name": {"VAULT://__secret___/demo/configmanager", "://", "|", "__secret"}, - "with multiple paths in mountpath": {"VAULT://secret/bar/path___/demo/configmanager", "://", "|", "secret/bar/path"}, + "without leading slash": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/demo/configmanager" + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "secret"}, + "with leading slash": { + func() *config.ParsedTokenConfig { + // "VAULT:///secret___/demo/configmanager", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "secret"}, + "with underscore in path name": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("_secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "_secret"}, + "with double underscore in path name": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("__secret___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "__secret"}, + "with multiple paths in mountpath": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret/bar/path___/demo/configmanager") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, "secret/bar/path"}, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *config.NewConfig().WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - got := splitToken(token.StoreToken()) - if got.path != tt.expect { + got := store.SplitHashiVaultToken(tt.token().StoreToken()) + if got.Path != tt.expect { t.Errorf("got %q, expected %q", got, tt.expect) } }) @@ -54,17 +85,22 @@ func (m mockVaultApi) GetVersion(ctx context.Context, secretPath string, version } func TestVaultScenarios(t *testing.T) { - t.Parallel() ttests := map[string]struct { - token string - conf *config.GenVarsConfig + token func() *config.ParsedTokenConfig expect string - mockClient func(t *testing.T) hashiVaultApi + mockClient func(t *testing.T) mockVaultApi setupEnv func() func() }{ - "happy return": {"VAULT://secret___/foo", config.NewConfig(), `{"foo":"test2130-9sd-0ds"}`, - func(t *testing.T) hashiVaultApi { + "happy return": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/foo") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, `{"foo":"test2130-9sd-0ds"}`, + func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { t.Helper() @@ -84,198 +120,197 @@ func TestVaultScenarios(t *testing.T) { } }, }, - "incorrect json": {"VAULT://secret___/foo", config.NewConfig(), `json: unsupported type: func() error`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "foo" { - t.Errorf("got %v; want %s", secretPath, `foo`) - } - m := make(map[string]interface{}) - m["error"] = func() error { return fmt.Errorf("ddodod") } - return &vault.KVSecret{Data: m}, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "another return": { - "VAULT://secret/engine1___/some/other/foo2", - config.NewConfig(), - `{"foo1":"test2130-9sd-0ds","foo2":"dsfsdf3454456"}`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - m["foo1"] = "test2130-9sd-0ds" - m["foo2"] = "dsfsdf3454456" - return &vault.KVSecret{Data: m}, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "not found": {"VAULT://secret___/foo", config.NewConfig(), `secret not found`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "foo" { - t.Errorf("got %v; want %s", secretPath, `foo`) - } - return nil, fmt.Errorf("secret not found") - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "403": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `client 403`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - } - return nil, fmt.Errorf("client 403") - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "found but empty": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `{}`, func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - return &vault.KVSecret{Data: m}, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "found but nil returned": {"VAULT://secret___/some/other/foo2", config.NewConfig(), "", func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return &vault.KVSecret{Data: nil}, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "version provided correctly": {"VAULT://secret___/some/other/foo2[version=1]", config.NewConfig(), `{"foo2":"dsfsdf3454456"}`, func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - m["foo2"] = "dsfsdf3454456" - return &vault.KVSecret{Data: m}, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "version provided but unable to parse": {"VAULT://secret___/some/other/foo2[version=1a]", config.NewConfig(), "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return nil, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "129378y1231283") - return func() { - os.Clearenv() - } - }, - }, - "vault rate limit incorrect": { - "VAULT://secret___/some/other/foo2", - config.NewConfig(), - `error encountered setting up default configuration: VAULT_RATE_LIMIT was provided but incorrectly formatted -failed to initialize the client`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return &vault.KVSecret{Data: nil}, nil - } - return mv - }, - func() func() { - os.Setenv("VAULT_TOKEN", "") - os.Setenv("VAULT_RATE_LIMIT", "wrong") - return func() { - os.Clearenv() - } - }, - }, + // "incorrect json": {"VAULT://secret___/foo", config.NewConfig(), `json: unsupported type: func() error`, + // func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "foo" { + // t.Errorf("got %v; want %s", secretPath, `foo`) + // } + // m := make(map[string]interface{}) + // m["error"] = func() error { return fmt.Errorf("ddodod") } + // return &vault.KVSecret{Data: m}, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "another return": { + // "VAULT://secret/engine1___/some/other/foo2", + // config.NewConfig(), + // `{"foo1":"test2130-9sd-0ds","foo2":"dsfsdf3454456"}`, + // func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + // } + // m := make(map[string]interface{}) + // m["foo1"] = "test2130-9sd-0ds" + // m["foo2"] = "dsfsdf3454456" + // return &vault.KVSecret{Data: m}, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "not found": {"VAULT://secret___/foo", config.NewConfig(), `secret not found`, + // func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "foo" { + // t.Errorf("got %v; want %s", secretPath, `foo`) + // } + // return nil, fmt.Errorf("secret not found") + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "403": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `client 403`, + // func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + // } + // return nil, fmt.Errorf("client 403") + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "found but empty": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `{}`, func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + // } + // m := make(map[string]interface{}) + // return &vault.KVSecret{Data: m}, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "found but nil returned": {"VAULT://secret___/some/other/foo2", config.NewConfig(), "", func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + // } + // return &vault.KVSecret{Data: nil}, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "version provided correctly": {"VAULT://secret___/some/other/foo2[version=1]", config.NewConfig(), `{"foo2":"dsfsdf3454456"}`, func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + // } + // m := make(map[string]interface{}) + // m["foo2"] = "dsfsdf3454456" + // return &vault.KVSecret{Data: m}, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "version provided but unable to parse": {"VAULT://secret___/some/other/foo2[version=1a]", config.NewConfig(), "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + // } + // return nil, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "129378y1231283") + // return func() { + // os.Clearenv() + // } + // }, + // }, + // "vault rate limit incorrect": { + // "VAULT://secret___/some/other/foo2", + // config.NewConfig(), + // `error encountered setting up default configuration: VAULT_RATE_LIMIT was provided but incorrectly formatted + // failed to initialize the client`, + // func(t *testing.T) hashiVaultApi { + // mv := mockVaultApi{} + // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + // t.Helper() + // if secretPath != "some/other/foo2" { + // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + // } + // return &vault.KVSecret{Data: nil}, nil + // } + // return mv + // }, + // func() func() { + // os.Setenv("VAULT_TOKEN", "") + // os.Setenv("VAULT_RATE_LIMIT", "wrong") + // return func() { + // os.Clearenv() + // } + // }, + // }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { tearDown := tt.setupEnv() defer tearDown() - token, _ := config.NewParsedTokenConfig(tt.token, *tt.conf) - impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) + impl, err := store.NewVaultStore(context.TODO(), tt.token(), log.New(io.Discard)) if err != nil { if err.Error() != tt.expect { t.Fatalf("failed to init hashivault, %v", err.Error()) @@ -283,7 +318,7 @@ failed to initialize the client`, return } - impl.svc = tt.mockClient(t) + impl.WithSvc(tt.mockClient(t)) got, err := impl.Token() if err != nil { if err.Error() != tt.expect { @@ -298,189 +333,189 @@ failed to initialize the client`, } } -func TestAwsIamAuth(t *testing.T) { - ttests := map[string]struct { - token string - conf *config.GenVarsConfig - expect string - mockClient func(t *testing.T) hashiVaultApi - mockHanlder func(t *testing.T) http.Handler - setupEnv func(addr string) func() - }{ - "aws_iam auth no role specified": { - "VAULT://secret___/some/other/foo2[version:1]", config.NewConfig(), - "role provided is empty, EC2 auth not supported", - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return &vault.KVSecret{Data: nil}, nil - } - return mv - }, - func(t *testing.T) http.Handler { - return nil - }, - func(_ string) func() { - os.Setenv("VAULT_TOKEN", "aws_iam") - os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") - os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") - os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") - os.Setenv("AWS_REGION", "eu-west-1") - return func() { - os.Clearenv() - } - }, - }, - "aws_iam auth incorrectly formatted request": { - "VAULT://secret___/some/other/foo2[version=1,iam_role=not_a_role]", config.NewConfig(), - `unable to login to AWS auth method: unable to log in to auth method: unable to log in with AWS auth: Error making API request. +// func TestAwsIamAuth(t *testing.T) { +// ttests := map[string]struct { +// token string +// conf *config.GenVarsConfig +// expect string +// mockClient func(t *testing.T) hashiVaultApi +// mockHanlder func(t *testing.T) http.Handler +// setupEnv func(addr string) func() +// }{ +// "aws_iam auth no role specified": { +// "VAULT://secret___/some/other/foo2[version:1]", config.NewConfig(), +// "role provided is empty, EC2 auth not supported", +// func(t *testing.T) hashiVaultApi { +// mv := mockVaultApi{} +// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { +// t.Helper() +// if secretPath != "some/other/foo2" { +// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) +// } +// return &vault.KVSecret{Data: nil}, nil +// } +// return mv +// }, +// func(t *testing.T) http.Handler { +// return nil +// }, +// func(_ string) func() { +// os.Setenv("VAULT_TOKEN", "aws_iam") +// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") +// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") +// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") +// os.Setenv("AWS_REGION", "eu-west-1") +// return func() { +// os.Clearenv() +// } +// }, +// }, +// "aws_iam auth incorrectly formatted request": { +// "VAULT://secret___/some/other/foo2[version=1,iam_role=not_a_role]", config.NewConfig(), +// `unable to login to AWS auth method: unable to log in to auth method: unable to log in with AWS auth: Error making API request. -URL: PUT %s/v1/auth/aws/login -Code: 400. Raw Message: +// URL: PUT %s/v1/auth/aws/login +// Code: 400. Raw Message: -incorrect values supplied. failed to initialize the client`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - return &vault.KVSecret{Data: nil}, nil - } - return mv - }, - func(t *testing.T) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { +// incorrect values supplied. failed to initialize the client`, +// func(t *testing.T) hashiVaultApi { +// mv := mockVaultApi{} +// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { +// t.Helper() +// if secretPath != "some/other/foo2" { +// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) +// } +// return &vault.KVSecret{Data: nil}, nil +// } +// return mv +// }, +// func(t *testing.T) http.Handler { +// mux := http.NewServeMux() +// mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(400) - w.Write([]byte(`incorrect values supplied`)) - }) - return mux - }, - func(addr string) func() { - os.Setenv("VAULT_TOKEN", "aws_iam") - os.Setenv("VAULT_ADDR", addr) - os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") - os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") - os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") - os.Setenv("AWS_REGION", "eu-west-1") - return func() { - os.Clearenv() - } - }, - }, - "aws_iam auth success": { - "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), - `{"foo2":"dsfsdf3454456"}`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - m["foo2"] = "dsfsdf3454456" - return &vault.KVSecret{Data: m}, nil - } - return mv - }, - func(t *testing.T) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { +// w.Header().Set("Content-Type", "application/json; charset=utf-8") +// w.WriteHeader(400) +// w.Write([]byte(`incorrect values supplied`)) +// }) +// return mux +// }, +// func(addr string) func() { +// os.Setenv("VAULT_TOKEN", "aws_iam") +// os.Setenv("VAULT_ADDR", addr) +// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") +// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") +// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") +// os.Setenv("AWS_REGION", "eu-west-1") +// return func() { +// os.Clearenv() +// } +// }, +// }, +// "aws_iam auth success": { +// "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), +// `{"foo2":"dsfsdf3454456"}`, +// func(t *testing.T) hashiVaultApi { +// mv := mockVaultApi{} +// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { +// t.Helper() +// if secretPath != "some/other/foo2" { +// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) +// } +// m := make(map[string]interface{}) +// m["foo2"] = "dsfsdf3454456" +// return &vault.KVSecret{Data: m}, nil +// } +// return mv +// }, +// func(t *testing.T) http.Handler { +// mux := http.NewServeMux() +// mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Write([]byte(`{"auth":{"client_token": "fooresddfasdsasad"}}`)) - }) - return mux - }, - func(addr string) func() { - os.Setenv("VAULT_TOKEN", "aws_iam") - os.Setenv("VAULT_ADDR", addr) - os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") - os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") - os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") - os.Setenv("AWS_REGION", "eu-west-1") - return func() { - os.Clearenv() - } - }, - }, - "aws_iam auth no token returned": { - "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), - `unable to login to AWS auth method: response did not return ClientToken, client token not set. failed to initialize the client`, - func(t *testing.T) hashiVaultApi { - mv := mockVaultApi{} - mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - t.Helper() - if secretPath != "some/other/foo2" { - t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - } - m := make(map[string]interface{}) - m["foo2"] = "dsfsdf3454456" - return &vault.KVSecret{Data: m}, nil - } - return mv - }, - func(t *testing.T) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { +// w.Header().Set("Content-Type", "application/json; charset=utf-8") +// w.Write([]byte(`{"auth":{"client_token": "fooresddfasdsasad"}}`)) +// }) +// return mux +// }, +// func(addr string) func() { +// os.Setenv("VAULT_TOKEN", "aws_iam") +// os.Setenv("VAULT_ADDR", addr) +// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") +// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") +// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") +// os.Setenv("AWS_REGION", "eu-west-1") +// return func() { +// os.Clearenv() +// } +// }, +// }, +// "aws_iam auth no token returned": { +// "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), +// `unable to login to AWS auth method: response did not return ClientToken, client token not set. failed to initialize the client`, +// func(t *testing.T) hashiVaultApi { +// mv := mockVaultApi{} +// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { +// t.Helper() +// if secretPath != "some/other/foo2" { +// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) +// } +// m := make(map[string]interface{}) +// m["foo2"] = "dsfsdf3454456" +// return &vault.KVSecret{Data: m}, nil +// } +// return mv +// }, +// func(t *testing.T) http.Handler { +// mux := http.NewServeMux() +// mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Write([]byte(`{"auth":{}}`)) - }) - return mux - }, - func(addr string) func() { - os.Setenv("VAULT_TOKEN", "aws_iam") - os.Setenv("VAULT_ADDR", addr) - os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") - os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") - os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") - os.Setenv("AWS_REGION", "eu-west-1") - return func() { - os.Clearenv() - } - }, - }, - } +// w.Header().Set("Content-Type", "application/json; charset=utf-8") +// w.Write([]byte(`{"auth":{}}`)) +// }) +// return mux +// }, +// func(addr string) func() { +// os.Setenv("VAULT_TOKEN", "aws_iam") +// os.Setenv("VAULT_ADDR", addr) +// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") +// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") +// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") +// os.Setenv("AWS_REGION", "eu-west-1") +// return func() { +// os.Clearenv() +// } +// }, +// }, +// } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - // - ts := httptest.NewServer(tt.mockHanlder(t)) - tearDown := tt.setupEnv(ts.URL) - defer tearDown() - token, _ := config.NewParsedTokenConfig(tt.token, *tt.conf) +// for name, tt := range ttests { +// t.Run(name, func(t *testing.T) { +// // +// ts := httptest.NewServer(tt.mockHanlder(t)) +// tearDown := tt.setupEnv(ts.URL) +// defer tearDown() +// token, _ := config.NewToken(tt.token, *tt.conf) - impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) - if err != nil { - // WHAT A CRAP way to do this... - if err.Error() != strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0] { - t.Errorf(testutils.TestPhraseWithContext, "aws iam auth", err.Error(), strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0]) - t.Fatalf("failed to init hashivault, %v", err.Error()) - } - return - } +// impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) +// if err != nil { +// // WHAT A CRAP way to do this... +// if err.Error() != strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0] { +// t.Errorf(testutils.TestPhraseWithContext, "aws iam auth", err.Error(), strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0]) +// t.Fatalf("failed to init hashivault, %v", err.Error()) +// } +// return +// } - impl.svc = tt.mockClient(t) - got, err := impl.Token() - if err != nil { - if err.Error() != tt.expect { - t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) - } - return - } - if got != tt.expect { - t.Errorf(testutils.TestPhrase, got, tt.expect) - } - }) - } -} +// impl.svc = tt.mockClient(t) +// got, err := impl.Token() +// if err != nil { +// if err.Error() != tt.expect { +// t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) +// } +// return +// } +// if got != tt.expect { +// t.Errorf(testutils.TestPhrase, got, tt.expect) +// } +// }) +// } +// } diff --git a/internal/store/paramstore.go b/internal/store/paramstore.go index 72b43a8..5e75d5f 100644 --- a/internal/store/paramstore.go +++ b/internal/store/paramstore.go @@ -41,6 +41,10 @@ func NewParamStore(ctx context.Context, logger log.ILogger) (*ParamStore, error) }, nil } +func (s *ParamStore) WithSvc(svc paramStoreApi) { + s.svc = svc +} + func (imp *ParamStore) SetToken(token *config.ParsedTokenConfig) { storeConf := &ParamStrConfig{} _ = token.ParseMetadata(storeConf) diff --git a/internal/store/paramstore_test.go b/internal/store/paramstore_test.go index 19c027a..47537cf 100644 --- a/internal/store/paramstore_test.go +++ b/internal/store/paramstore_test.go @@ -1,14 +1,14 @@ -package store +package store_test import ( "context" - "fmt" "io" "strings" "testing" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" @@ -44,75 +44,79 @@ func awsParamtStoreCommonGetChecker(t *testing.T, params *ssm.GetParameterInput) } func Test_GetParamStore(t *testing.T) { - t.Parallel() - var ( tsuccessParam = "someVal" // tsuccessObj map[string]string = map[string]string{"AWSPARAMSTR#/token/1": "someVal"} ) tests := map[string]struct { - token string - keySeparator string - tokenSeparator string - expect string - mockClient func(t *testing.T) paramStoreApi - config *config.GenVarsConfig + token func() *config.ParsedTokenConfig + expect string + mockClient func(t *testing.T) mockParamApi }{ - "successVal": {"AWSPARAMSTR#/token/1", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) - return &ssm.GetParameterOutput{ - Parameter: &types.Parameter{Value: &tsuccessParam}, - }, nil - }) - }, config.NewConfig(), + "successVal": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/demo/configmanager" + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + tsuccessParam, func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) + return &ssm.GetParameterOutput{ + Parameter: &types.Parameter{Value: &tsuccessParam}, + }, nil + }) + }, }, - "successVal with keyseparator": {"AWSPARAMSTR#/token/1|somekey", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) + // "successVal with keyseparator": {"AWSPARAMSTR#/token/1|somekey", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { + // return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + // t.Helper() + // awsParamtStoreCommonGetChecker(t, params) - if strings.Contains(*params.Name, "|somekey") { - t.Errorf("incorrectly stripped key separator") - } + // if strings.Contains(*params.Name, "|somekey") { + // t.Errorf("incorrectly stripped key separator") + // } - return &ssm.GetParameterOutput{ - Parameter: &types.Parameter{Value: &tsuccessParam}, - }, nil - }) - }, config.NewConfig(), - }, - "errored": {"AWSPARAMSTR#/token/1", "|", "#", "unable to retrieve", func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) - return nil, fmt.Errorf("unable to retrieve") - }) - }, config.NewConfig(), - }, - "nil to empty": {"AWSPARAMSTR#/token/1", "|", "#", "", func(t *testing.T) paramStoreApi { - return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - t.Helper() - awsParamtStoreCommonGetChecker(t, params) - return &ssm.GetParameterOutput{ - Parameter: &types.Parameter{Value: nil}, - }, nil - }) - }, config.NewConfig(), - }, + // return &ssm.GetParameterOutput{ + // Parameter: &types.Parameter{Value: &tsuccessParam}, + // }, nil + // }) + // }, config.NewConfig(), + // }, + // "errored": {"AWSPARAMSTR#/token/1", "|", "#", "unable to retrieve", func(t *testing.T) paramStoreApi { + // return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + // t.Helper() + // awsParamtStoreCommonGetChecker(t, params) + // return nil, fmt.Errorf("unable to retrieve") + // }) + // }, config.NewConfig(), + // }, + // "nil to empty": {"AWSPARAMSTR#/token/1", "|", "#", "", func(t *testing.T) paramStoreApi { + // return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + // t.Helper() + // awsParamtStoreCommonGetChecker(t, params) + // return &ssm.GetParameterOutput{ + // Parameter: &types.Parameter{Value: nil}, + // }, nil + // }) + // }, config.NewConfig(), + // }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) + // token, _ := config.NewToken(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - impl, err := NewParamStore(context.TODO(), log.New(io.Discard)) + impl, err := store.NewParamStore(context.TODO(), log.New(io.Discard)) if err != nil { t.Errorf(testutils.TestPhrase, err.Error(), nil) } - impl.svc = tt.mockClient(t) - impl.SetToken(token) + impl.WithSvc(tt.mockClient(t)) + impl.SetToken(tt.token()) got, err := impl.Token() if err != nil { if err.Error() != tt.expect { diff --git a/internal/store/secretsmanager.go b/internal/store/secretsmanager.go index 6b0f7a2..1f764d4 100644 --- a/internal/store/secretsmanager.go +++ b/internal/store/secretsmanager.go @@ -42,6 +42,10 @@ func NewSecretsMgr(ctx context.Context, logger log.ILogger) (*SecretsMgr, error) } +func (s *SecretsMgr) WithSvc(svc secretsMgrApi) { + s.svc = svc +} + func (imp *SecretsMgr) SetToken(token *config.ParsedTokenConfig) { storeConf := &SecretsMgrConfig{} if err := token.ParseMetadata(storeConf); err != nil { diff --git a/internal/store/secretsmanager_test.go b/internal/store/secretsmanager_test.go index 3b29dd2..30f9849 100644 --- a/internal/store/secretsmanager_test.go +++ b/internal/store/secretsmanager_test.go @@ -1,14 +1,14 @@ -package store +package store_test import ( "context" - "fmt" "io" "strings" "testing" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" ) @@ -34,76 +34,76 @@ func awsSecretsMgrGetChecker(t *testing.T, params *secretsmanager.GetSecretValue } func Test_GetSecretMgr(t *testing.T) { - t.Parallel() tsuccessSecret := "dsgkbdsf" tests := map[string]struct { - token string - keySeparator string - tokenSeparator string - expect string - mockClient func(t *testing.T) secretsMgrApi - config *config.GenVarsConfig + token func() *config.ParsedTokenConfig + expect string + mockClient func(t *testing.T) mockSecretsApi }{ - "success": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretString: &tsuccessSecret, - }, nil - }) - }, config.NewConfig(), - }, - "success with version": {"AWSSECRETS#/token/1[version=123]", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretString: &tsuccessSecret, - }, nil - }) - }, config.NewConfig(), - }, - "success with binary": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretBinary: []byte(tsuccessSecret), - }, nil - }) - }, config.NewConfig(), - }, - "errored": {"AWSSECRETS#/token/1", "|", "#", "unable to retrieve secret", func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return nil, fmt.Errorf("unable to retrieve secret") - }) - }, config.NewConfig(), - }, - "ok but empty": {"AWSSECRETS#/token/1", "|", "#", "", func(t *testing.T) secretsMgrApi { - return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - t.Helper() - awsSecretsMgrGetChecker(t, params) - return &secretsmanager.GetSecretValueOutput{ - SecretString: nil, - }, nil - }) - }, config.NewConfig(), + "success": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, tsuccessSecret, func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretString: &tsuccessSecret, + }, nil + }) + }, }, + // "success with version": {"AWSSECRETS#/token/1[version=123]", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { + // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + // t.Helper() + // awsSecretsMgrGetChecker(t, params) + // return &secretsmanager.GetSecretValueOutput{ + // SecretString: &tsuccessSecret, + // }, nil + // }) + // }, config.NewConfig(), + // }, + // "success with binary": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { + // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + // t.Helper() + // awsSecretsMgrGetChecker(t, params) + // return &secretsmanager.GetSecretValueOutput{ + // SecretBinary: []byte(tsuccessSecret), + // }, nil + // }) + // }, config.NewConfig(), + // }, + // "errored": {"AWSSECRETS#/token/1", "|", "#", "unable to retrieve secret", func(t *testing.T) secretsMgrApi { + // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + // t.Helper() + // awsSecretsMgrGetChecker(t, params) + // return nil, fmt.Errorf("unable to retrieve secret") + // }) + // }, config.NewConfig(), + // }, + // "ok but empty": {"AWSSECRETS#/token/1", "|", "#", "", func(t *testing.T) secretsMgrApi { + // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + // t.Helper() + // awsSecretsMgrGetChecker(t, params) + // return &secretsmanager.GetSecretValueOutput{ + // SecretString: nil, + // }, nil + // }) + // }, config.NewConfig(), + // }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + impl, _ := store.NewSecretsMgr(context.TODO(), log.New(io.Discard)) + impl.WithSvc(tt.mockClient(t)) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - - impl, _ := NewSecretsMgr(context.TODO(), log.New(io.Discard)) - impl.svc = tt.mockClient(t) - - impl.SetToken(token) + impl.SetToken(tt.token()) got, err := impl.Token() if err != nil { if err.Error() != tt.expect { diff --git a/internal/strategy/strategy_test.go b/internal/strategy/strategy_test.go index 93c4c6c..0f642bd 100644 --- a/internal/strategy/strategy_test.go +++ b/internal/strategy/strategy_test.go @@ -41,20 +41,21 @@ var TEST_GCP_CREDS = []byte(`{ }`) func Test_Strategy_Retrieve_succeeds(t *testing.T) { - t.Parallel() ttests := map[string]struct { - impl func(t *testing.T) store.Strategy - config *config.GenVarsConfig - token string - expect string + impl func(t *testing.T) store.Strategy + config *config.GenVarsConfig + token string + expect string + impPrefix config.ImplementationPrefix }{ "with mocked implementation AZTABLESTORAGE": { func(t *testing.T) store.Strategy { - return &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} + return &mockGenerate{"mountPath/token", "bar", nil} }, - config.NewConfig().WithOutputPath("stdout").WithTokenSeparator("://"), - "AZTABLESTORE://mountPath/token", + config.NewConfig().WithOutputPath("stdout"), + "mountPath/token", "bar", + config.AzTableStorePrefix, }, // "error in retrieval": { // func(t *testing.T) store.Strategy { @@ -69,7 +70,8 @@ func Test_Strategy_Retrieve_succeeds(t *testing.T) { for name, tt := range ttests { t.Run(name, func(t *testing.T) { rs := strategy.New(*tt.config, log.New(io.Discard)) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) + token, _ := config.NewToken(tt.impPrefix, *tt.config) + token.WithSanitizedToken(tt.token) got := rs.RetrieveByToken(context.TODO(), tt.impl(t), token) if got.Err != nil { t.Errorf(testutils.TestPhraseWithContext, "Token response errored", got.Err.Error(), tt.expect) @@ -85,7 +87,6 @@ func Test_Strategy_Retrieve_succeeds(t *testing.T) { } func Test_CustomStrategyFuncMap_add_own(t *testing.T) { - t.Parallel() ttests := map[string]struct { }{ @@ -95,7 +96,7 @@ func Test_CustomStrategyFuncMap_add_own(t *testing.T) { t.Run(name, func(t *testing.T) { called := 0 genVarsConf := config.NewConfig() - token, _ := config.NewParsedTokenConfig("AZTABLESTORE://mountPath/token", *genVarsConf) + token, _ := config.NewToken("AZTABLESTORE://mountPath/token", *genVarsConf) var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { m := &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} @@ -123,16 +124,18 @@ func Test_SelectImpl_With(t *testing.T) { config *config.GenVarsConfig expect func() store.Strategy expErr error + impPrefix config.ImplementationPrefix }{ "unknown": { func() func() { return func() { } }, - "UNKNOWN#foo/bar", + "foo/bar", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { return nil }, fmt.Errorf("implementation not found for input string: UNKNOWN#foo/bar"), + config.UnknownPrefix, }, "success AZTABLESTORE": { func() func() { @@ -141,14 +144,17 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AZTABLESTORE#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("AZTABLESTORE#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") + s, _ := store.NewAzTableStore(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.AzTableStorePrefix, }, "success AWSPARAMSTR": { func() func() { @@ -158,13 +164,14 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AWSPARAMSTR#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { s, _ := store.NewParamStore(context.TODO(), log.New(io.Discard)) return s }, nil, + config.ParamStorePrefix, }, "success AWSSECRETS": { func() func() { @@ -174,13 +181,14 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AWSSECRETS#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { s, _ := store.NewSecretsMgr(context.TODO(), log.New(io.Discard)) return s }, nil, + config.SecretMgrPrefix, }, "success AZKVSECRET": { func() func() { @@ -190,14 +198,16 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AZKVSECRET#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("AZKVSECRET#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") s, _ := store.NewKvScrtStore(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.AzKeyVaultSecretsPrefix, }, "success AZAPPCONF": { func() func() { @@ -205,14 +215,16 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "AZAPPCONF#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("AZAPPCONF#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.AzAppConfigPrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") s, _ := store.NewAzAppConf(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.AzAppConfigPrefix, }, "success VAULT": { func() func() { @@ -221,14 +233,16 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "VAULT#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { - token, _ := config.NewParsedTokenConfig("VAULT#foo/bar1", *config.NewConfig().WithTokenSeparator("#")) + token, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig().WithTokenSeparator("#")) + token.WithSanitizedToken("foo/bar1") s, _ := store.NewVaultStore(context.TODO(), token, log.New(io.Discard)) return s }, nil, + config.HashicorpVaultPrefix, }, "success GCPSECRETS": { func() func() { @@ -240,32 +254,15 @@ func Test_SelectImpl_With(t *testing.T) { os.Clearenv() } }, - "GCPSECRETS#foo/bar1", + "foo/bar1", config.NewConfig().WithTokenSeparator("#"), func() store.Strategy { s, _ := store.NewGcpSecrets(context.TODO(), log.New(io.Discard)) return s }, nil, + config.GcpSecretsPrefix, }, - // "default Error": { - // func() func() { - // os.Setenv("AWS_ACCESS_KEY", "AAAAAAAAAAAAAAA") - // os.Setenv("AWS_SECRET_ACCESS_KEY", "00000000000000000000111111111") - // return func() { - // os.Clearenv() - // } - // }, - // context.TODO(), - // UnknownPrefix, "AWSPARAMSTR://foo/bar", (&GenVarsConfig{}).WithKeySeparator("|").WithTokenSeparator("://"), - // func(t *testing.T, ctx context.Context, conf GenVarsConfig) genVarsStrategy { - // imp, err := NewParamStore(ctx) - // if err != nil { - // t.Errorf(testutils.TestPhraseWithContext, "init impl error", err.Error(), nil) - // } - // return imp - // }, - // }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { @@ -273,7 +270,8 @@ func Test_SelectImpl_With(t *testing.T) { defer tearDown() want := tt.expect() rs := strategy.New(*tt.config, log.New(io.Discard)) - token, _ := config.NewParsedTokenConfig(tt.token, *tt.config) + token, _ := config.NewToken(tt.impPrefix, *tt.config) + token.WithSanitizedToken(tt.token) got, err := rs.SelectImplementation(context.TODO(), token) if err != nil { diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 4d8bd69..500ab4b 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -5,11 +5,14 @@ import ( "encoding/json" "fmt" "io" + "os" "strconv" "sync" "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v2/internal/lexer" "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/parser" "github.com/DevLabFoundry/configmanager/v2/internal/strategy" "github.com/spyzhov/ajson" ) @@ -150,14 +153,17 @@ func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { rtm := newRawTokenMap() for _, token := range tokens { - // TODO: normalize tokens here potentially - // merge any tokens that only differ in keys lookup inside the object - parsedToken, err := config.NewParsedTokenConfig(token, c.config) - if err != nil { - c.Logger.Info(err.Error()) + lexerSource := lexer.Source{FileName: token, FullPath: "", Input: token} + l := lexer.New(lexerSource, c.config) + p := parser.New(l, &c.config).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + c.Logger.Info(fmt.Sprintf("%v", errs)) continue } - rtm.addToken(token, parsedToken) + for _, prsdToken := range parsed { + rtm.addToken(token, &prsdToken.ParsedToken) + } } // pass in default initialised retrieveStrategy // input should be @@ -167,6 +173,16 @@ func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { return c.rawMap.getTokenMap(), nil } +// IsParsed will try to parse the return found string into +// map[string]string +// If found it will convert that to a map with all keys uppercased +// and any characters +func IsParsed(v any, trm ParsedMap) bool { + str := fmt.Sprint(v) + err := json.Unmarshal([]byte(str), &trm) + return err == nil +} + // generate checks if any tokens found // initiates groutines with fixed size channel map // to capture responses and errors @@ -220,16 +236,6 @@ func (c *GenVars) generate(rawMap *rawTokenMap) error { return nil } -// IsParsed will try to parse the return found string into -// map[string]string -// If found it will convert that to a map with all keys uppercased -// and any characters -func IsParsed(v any, trm ParsedMap) bool { - str := fmt.Sprint(v) - err := json.Unmarshal([]byte(str), &trm) - return err == nil -} - // keySeparatorLookup checks if the key contains // keySeparator character // If it does contain one then it tries to parse From ec8040c37e19ed58b8ac7b284b0ad2758b3e1292 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Thu, 23 Oct 2025 06:39:37 +0100 Subject: [PATCH 05/19] fix: make it right now let's refactor a bit --- configmanager.go | 56 ++++++++++---- configmanager_test.go | 83 ++++++++++++++++---- internal/lexer/lexer.go | 21 +++-- internal/parser/parser.go | 62 ++++++++++++--- internal/parser/parser_test.go | 137 +++++++++++++++++++++++++++------ pkg/generator/generator.go | 64 ++------------- pkg/generator/generatorvars.go | 59 ++++++++++++++ 7 files changed, 351 insertions(+), 131 deletions(-) create mode 100644 pkg/generator/generatorvars.go diff --git a/configmanager.go b/configmanager.go index de0c2c5..7c512f7 100644 --- a/configmanager.go +++ b/configmanager.go @@ -5,11 +5,16 @@ import ( "encoding/json" "errors" "fmt" + "io" + "regexp" "slices" "strings" "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v2/internal/lexer" + "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v2/internal/parser" "github.com/DevLabFoundry/configmanager/v2/pkg/generator" "github.com/a8m/envsubst" "gopkg.in/yaml.v3" @@ -27,25 +32,35 @@ type generateAPI interface { type ConfigManager struct { Config *config.GenVarsConfig generator generateAPI + logger log.ILogger } // New returns an initialised instance of ConfigManager // Uses default config for: // -// ``` -// outputPath = "" -// keySeparator = "|" -// tokenSeparator = "://" -// ``` +// outputPath = "" +// keySeparator = "|" +// tokenSeparator = "://" +// +// # Calling cm.Config.WithXXX() will overwrite the generator config // -// Calling cm.Config.WithXXX() will overwrite the generator config +// Default logger will log to io.Discard +// Attach your own if you need via +// +// WithLogger(l log.ILogger) *ConfigManager func New(ctx context.Context) *ConfigManager { cm := &ConfigManager{} cm.Config = config.NewConfig() cm.generator = generator.NewGenerator(ctx).WithConfig(cm.Config) + cm.logger = log.New(io.Discard) return cm } +func (c *ConfigManager) WithLogger(l log.ILogger) *ConfigManager { + c.logger = l + return c +} + // GeneratorConfig // Returns the gettable generator config func (c *ConfigManager) GeneratorConfig() *config.GenVarsConfig { @@ -86,6 +101,7 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) return "", fmt.Errorf("%w\n%v", ErrEnvSubst, err) } } + m, err := c.retrieve(FindTokens(input)) if err != nil { @@ -95,23 +111,29 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) return replaceString(m, input), nil } -func (c *ConfigManager) DiscoverTokens(input string) []string { - tokens := []string{} - // termChar := `[^\'\"\s\n\\\,\:\@]` // :\@\?\/ - // (?:[^\'\"\\\,\:\@\&\s]|(?:[^/?]))+ - for k := range config.VarPrefix { - re := regexp.MustCompile(regexp.QuoteMeta(string(k)+c.Config.TokenSeparator()) + `(?:[^'"\\,:@&\s/]|/(?:[^?]|$))+`) - // re := regexp.MustCompile(regexp.QuoteMeta(string(k)+c.Config.TokenSeparator()) + `(?:[^\'\"\s\n\\\,:@&]|(?:[^/?]))+`) - matches := re.FindAllString(input, -1) - tokens = append(tokens, matches...) +var ErrTokenDiscovery = errors.New("failed to discover tokens") + +func (c *ConfigManager) DiscoverTokens(input string) ([]config.ParsedTokenConfig, error) { + lexerSource := lexer.Source{FileName: "", FullPath: "", Input: input} + l := lexer.New(lexerSource, *c.Config) + p := parser.New(l, c.Config).WithLogger(c.logger) + parsed, errs := p.Parse() + if len(errs) > 0 { + return nil, fmt.Errorf("%w in input (%s) with errors: %q", ErrTokenDiscovery, input[0:min(len(input), 25)], errs) } - return tokens + + pt := []config.ParsedTokenConfig{} + for _, prsdToken := range parsed { + pt = append(pt, prsdToken.ParsedToken) + } + return pt, nil } // FindTokens extracts all replaceable tokens // from a given input string // -// Deprecated: FindTokens relies on Regex. Use DiscoverTokens +// Deprecated: FindTokens relies on Regex. +// Use func (c *ConfigManager) DiscoverTokens(input string) []*config.ParsedTokenConfig func FindTokens(input string) []string { tokens := []string{} for k := range config.VarPrefix { diff --git a/configmanager_test.go b/configmanager_test.go index 1088b9f..ea57805 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "reflect" + "slices" "sort" "testing" @@ -493,7 +494,7 @@ func TestFindTokens(t *testing.T) { }, "all implementations": { `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - secretsmgr: AWSSECRETS#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123] + secretsmgr: AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123] gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[] som othere strufsd @@ -501,7 +502,7 @@ func TestFindTokens(t *testing.T) { []string{ "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSSECRETS#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123]", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123]", "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[]"}, }, @@ -526,34 +527,84 @@ func Test_ConfigManager_DiscoverTokens(t *testing.T) { expect []string }{ "multiple tokens in single string": { - `Lorem_Ipsum: AWSPARAMSTR:///path/config|foo.user:AWSPARAMSTR:///path/config|password@AWSPARAMSTR:///path/config|foo.endpoint:AWSPARAMSTR:///path/config|foo.port/?someQ=AWSPARAMSTR:///path/queryparam[version=123]|p1&anotherQ=false`, + `Lorem_Ipsum: AWSPARAMSTR:///path/config|foo.user:AWSPARAMSTR:///path/config|password@AWSPARAMSTR:///path/config|foo.endpoint:AWSPARAMSTR:///path/config|foo.port/?someQ=AWSPARAMSTR:///path/queryparam|p1[version=123]&anotherQ=false`, "://", []string{ "AWSPARAMSTR:///path/config|foo.user", "AWSPARAMSTR:///path/config|password", "AWSPARAMSTR:///path/config|foo.endpoint", "AWSPARAMSTR:///path/config|foo.port", - "AWSPARAMSTR:///path/queryparam[version=123]|p1"}, + "AWSPARAMSTR:///path/queryparam|p1[version=123]"}, }, "# tokens in single string": { - `Lorem_Ipsum: AWSPARAMSTR#/path/config|foo.user:AWSPARAMSTR#/path/config|password@AWSPARAMSTR#/path/config|foo.endpoint:AWSPARAMSTR#/path/config|foo.port/?someQ=AWSPARAMSTR#/path/queryparam[version=123]|p1&anotherQ=false`, + `Lorem_Ipsum: AWSPARAMSTR#/path/config|foo.user:AWSPARAMSTR#/path/config|password@AWSPARAMSTR#/path/config|foo.endpoint:AWSPARAMSTR#/path/config|foo.port/?someQ=AWSPARAMSTR#/path/queryparam|p1[version=123]&anotherQ=false`, "#", []string{ "AWSPARAMSTR#/path/config|foo.user", "AWSPARAMSTR#/path/config|password", "AWSPARAMSTR#/path/config|foo.endpoint", "AWSPARAMSTR#/path/config|foo.port", - "AWSPARAMSTR#/path/queryparam[version=123]|p1"}, + "AWSPARAMSTR#/path/queryparam|p1[version=123]"}, }, "without leading slash and path like name # tokens in single string": { - `Lorem_Ipsum: AWSPARAMSTR#path_config|foo.user:AWSPARAMSTR#path_config|password@AWSPARAMSTR#path_config|foo.endpoint:AWSPARAMSTR#path_config|foo.port/?someQ=AWSPARAMSTR#path_queryparam[version=123]|p1&anotherQ=false`, + `Lorem_Ipsum: AWSPARAMSTR#path_config|foo.user:AWSPARAMSTR#path_config|password@AWSPARAMSTR#path_config|foo.endpoint:AWSPARAMSTR#path_config|foo.port/?someQ=AWSPARAMSTR#path_queryparam|p1[version=123]&anotherQ=false`, "#", []string{ "AWSPARAMSTR#path_config|foo.user", "AWSPARAMSTR#path_config|password", "AWSPARAMSTR#path_config|foo.endpoint", "AWSPARAMSTR#path_config|foo.port", - "AWSPARAMSTR#path_queryparam[version=123]|p1"}, + "AWSPARAMSTR#path_queryparam|p1[version=123]"}, + }, + // Ensures all previous test cases pass as well + "extract from text correctly": { + `Where does it come from? + Contrary to popular belief, + Lorem Ipsum is AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1 <= in middle of sentencenot simply random text. + It has roots in a piece of classical Latin literature from 45 + BC, making it over 2000 years old. Richard McClintock, a Latin professor at + Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, c + onsectetur, from a Lorem Ipsum passage , at the end of line => AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4 + and going through the cites of the word in c + lassical literature, discovered the undoubtable source. Lorem Ipsum comes from secti + ons in singles =>'AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2'1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) + in doubles => "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3" + by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular + during the :=> embedded in text RenaissanceAWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5 embedded in text <=: + The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.`, + "://", + []string{ + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5", + }, + }, + "unknown implementation not picked up": { + `foo: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + bar: AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + unknown: GCPPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + unknown: GCPSECRETS#/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, + "://", + []string{ + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]"}, + }, + "all implementations": { + `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + secretsmgr: AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + som othere strufsd + azkv: AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, + "://", + []string{ + "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]", + "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj"}, }, } for name, tt := range ttests { @@ -561,16 +612,18 @@ func Test_ConfigManager_DiscoverTokens(t *testing.T) { config.VarPrefix = map[config.ImplementationPrefix]bool{"AWSPARAMSTR": true} c := configmanager.New(context.TODO()) c.Config.WithTokenSeparator(tt.separator) - got := c.DiscoverTokens(tt.input) - sort.Strings(got) - sort.Strings(tt.expect) + got, err := c.DiscoverTokens(tt.input) + if err != nil { + t.Fatal(err) + } if len(got) != len(tt.expect) { - t.Errorf("wrong length - got %d, want %d", len(got), len(tt.expect)) + t.Errorf("wrong nmber of tokens resolved\ngot (%d) want (%d)", len(got), len(tt.expect)) } - - if !reflect.DeepEqual(got, tt.expect) { - t.Errorf("input=(%q)\n\ngot: %v\n\nwant: %v", tt.input, got, tt.expect) + for _, v := range got { + if !slices.Contains(tt.expect, v.String()) { + t.Errorf("got (%s) not found in expected slice (%v)", v, tt.expect) + } } }) } diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index 25a9be2..e785727 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -12,9 +12,11 @@ var nonText = map[string]bool{ // separators " ": true, "\n": true, "\r": true, "\t": true, "=": true, ".": true, ",": true, "|": true, "?": true, "/": true, "@": true, ":": true, - "]": true, "[": true, + "]": true, "[": true, "'": true, "\"": true, // initial chars of potential identifiers - // TODO: when a new implementation is added we should add it here + // this forces the lexer to not treat at as TEXT + // and enter the switch statement of the state machine + // NOTE: when a new implementation is added we should add it here // AWS|AZure... "A": true, // VAULT (HashiCorp) @@ -66,9 +68,10 @@ func (l *Lexer) NextToken() config.Token { // identify the dynamically selected key separator case l.keySeparator: tok = config.Token{Type: config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, Literal: string(l.ch)} + // Specific cases for BEGIN_CONFIGMANAGER_TOKEN possibilities case 'A': - // AWS store types if l.peekChar() == 'W' { + // AWS store types l.readChar() if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.SecretMgrPrefix, config.ParamStorePrefix}, "AW"); found { tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} @@ -76,8 +79,8 @@ func (l *Lexer) NextToken() config.Token { // it is not a marker AW as text tok = config.Token{Type: config.TEXT, Literal: "AW"} } - // Azure Store Types } else if l.peekChar() == 'Z' { + // Azure Store Types l.readChar() if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.AzKeyVaultSecretsPrefix, config.AzTableStorePrefix, config.AzAppConfigPrefix}, "AZ"); found { tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} @@ -89,6 +92,7 @@ func (l *Lexer) NextToken() config.Token { tok = config.Token{Type: config.TEXT, Literal: "A"} } case 'G': + // GCP TOKENS if l.peekChar() == 'C' { l.readChar() if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.GcpSecretsPrefix}, "GC"); found { @@ -101,18 +105,20 @@ func (l *Lexer) NextToken() config.Token { tok = config.Token{Type: config.TEXT, Literal: "G"} } case 'V': + // HASHI VAULT Tokens if l.peekChar() == 'A' { l.readChar() if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.HashicorpVaultPrefix}, "VA"); found { tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} } else { - // it is not a marker AW as text + // it is not a marker VA as text tok = config.Token{Type: config.TEXT, Literal: "VA"} } } else { tok = config.Token{Type: config.TEXT, Literal: "V"} } case 'U': + // UNKNOWN if l.peekChar() == 'N' { l.readChar() if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.UnknownPrefix}, "UN"); found { @@ -151,6 +157,10 @@ func (l *Lexer) NextToken() config.Token { tok = config.Token{Type: config.AT_SIGN, Literal: "@"} case ':': tok = config.Token{Type: config.COLON, Literal: ":"} + case '"': + tok = config.Token{Type: config.DOUBLE_QUOTE, Literal: "\""} + case '\'': + tok = config.Token{Type: config.SINGLE_QUOTE, Literal: "'"} case '\n': l.line = l.line + 1 l.column = 0 // reset column count @@ -164,7 +174,6 @@ func (l *Lexer) NextToken() config.Token { // case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', // 'B', 'C', 'D', 'E', 'F', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'W', 'X', 'Y', 'Z', // 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z': - default: if isText(l.ch) { tok.Literal = l.readText() diff --git a/internal/parser/parser.go b/internal/parser/parser.go index c7561a7..b70f702 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -13,8 +13,8 @@ import ( "github.com/a8m/envsubst" ) -func wrapErr(file string, line, position int, etyp error) error { - return fmt.Errorf("\n - [%s:%d:%d] %w", file, line, position, etyp) +func wrapErr(incompleteToken string, line, position int, etyp error) error { + return fmt.Errorf("\n- token: (%s) on line: %d column: %d] %w", incompleteToken, line, position, etyp) } var ( @@ -124,8 +124,10 @@ func (p *Parser) peekTokenIs(t config.TokenType) bool { func (p *Parser) peekTokenIsEnd() bool { endTokens := map[config.TokenType]bool{ config.AT_SIGN: true, config.QUESTION_MARK: true, config.COLON: true, - config.DOUBLE_QUOTE: true, config.SINGLE_QUOTE: true, - config.NEW_LINE: true, config.SLASH_QUESTION_MARK: true, + config.SLASH_QUESTION_MARK: true, config.EOF: true, + // traditional ends of tokens + config.DOUBLE_QUOTE: true, config.SINGLE_QUOTE: true, config.SPACE: true, + config.NEW_LINE: true, } return endTokens[p.peekToken.Type] } @@ -147,8 +149,18 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // should exit the loop if no end doc tag found notFoundEnd := true + // stop on end of file for !p.peekTokenIs(config.EOF) { + // This is the target state when there is an optional token wrapping + // {{ IMP://path }} + if p.peekTokenIs(config.END_CONFIGMANAGER_TOKEN) { + notFoundEnd = false + fullToken += p.curToken.Literal + sanitizedToken += p.curToken.Literal + stmt.EndToken = p.curToken + break + } // when next token is another token // i.e. the tokens are adjacent @@ -162,6 +174,9 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // reached the end of a potential token if p.peekTokenIsEnd() { + // when the next token is EOF + // we want set the current token as both the full and sanitized + // the current lexer token is the entire configmanager token notFoundEnd = false fullToken += p.curToken.Literal sanitizedToken += p.curToken.Literal @@ -175,10 +190,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // everything is token path until (if any key separator exists) // check key separator this marks the end of a normal token path if p.currentTokenIs(config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR) { - // advance to next token i.e. start of the path separator - p.nextToken() if err := p.buildKeyPathSeparator(configManagerToken); err != nil { - p.errors = append(p.errors, err) + p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) return nil } notFoundEnd = false @@ -190,19 +203,31 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // check metadata there can be a metadata bracket `[key=val,k1=v2]` if p.currentTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { if err := p.buildMetadata(configManagerToken); err != nil { - p.errors = append(p.errors, err) + p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) return nil } notFoundEnd = false break } + sanitizedToken += p.curToken.Literal fullToken += p.curToken.Literal + + // when the next token is EOF + // we want set the current token + // else it would be lost once the parser is advanced below p.nextToken() + if p.peekTokenIs(config.EOF) { + notFoundEnd = false + fullToken += p.curToken.Literal + sanitizedToken += p.curToken.Literal + stmt.EndToken = p.curToken + break + } } if notFoundEnd { - p.errors = append(p.errors, wrapErr(currentToken.Source.File, currentToken.Line, currentToken.Column, ErrNoEndTagFound)) + p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, ErrNoEndTagFound)) return nil } @@ -214,9 +239,20 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // buildKeyPathSeparator already advanced to the first token func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenConfig) error { + // advance to next token i.e. post the path separator + p.nextToken() keyPath := "" + if p.peekTokenIs(config.EOF) { + // if the next token EOF we set the path as current token and exit + // otherwise we would never hit the below loop + configManagerToken.WithKeyPath(p.curToken.Literal) + return nil + } for !p.peekTokenIs(config.EOF) { if p.peekTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { + // add current token to the keysPath and move onto the metadata + keyPath += p.curToken.Literal + p.nextToken() if err := p.buildMetadata(configManagerToken); err != nil { return err } @@ -230,17 +266,19 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon keyPath += p.curToken.Literal p.nextToken() } - // p.nextToken() configManagerToken.WithKeyPath(keyPath) return nil } +var ErrMetadataEmpty = errors.New("emtpy metadata") + // buildMetadata adds metadata to the ParsedTokenConfig func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) error { - // inLoop := true - // errNoClose := fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) metadata := "" found := false + if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { + return fmt.Errorf("%w, metadata brackets must include at least one set of key=value pairs", ErrMetadataEmpty) + } p.nextToken() for !p.peekTokenIs(config.EOF) { if p.peekTokenIsEnd() { diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 66ef3c4..4bf5d19 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -20,25 +20,44 @@ func Test_ParserBlocks(t *testing.T) { // prefix,path,keyLookup expected [][3]string }{ - "tokens touching each other in source": {`foo stuyfsdfsf -foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo -other text her -BAR=something - `, [][3]string{ - {string(config.ParamStorePrefix), "/path", "key"}, - {string(config.SecretMgrPrefix), "/foo", ""}, - }}, - "full URL of tokens": {`foo stuyfsdfsf -foo=proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2 -# some comment -BAR=something - `, [][3]string{ - {string(config.ParamStorePrefix), "/config", "user"}, - {string(config.SecretMgrPrefix), "/creds", "password"}, - {string(config.ParamStorePrefix), "/config", "endpoint"}, - {string(config.ParamStorePrefix), "/config", "port"}, - {string(config.ParamStorePrefix), "/config", "qp2"}, - }}, + "tokens touching each other in source": { + `foo stuyfsdfsf + foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo + other text her + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", "key"}, + {string(config.SecretMgrPrefix), "/foo", ""}, + }}, + "full URL of tokens": { + `foo stuyfsdfsf + foo=proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2 + # some comment + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/config", "user"}, + {string(config.SecretMgrPrefix), "/creds", "password"}, + {string(config.ParamStorePrefix), "/config", "endpoint"}, + {string(config.ParamStorePrefix), "/config", "port"}, + {string(config.ParamStorePrefix), "/config", "qp2"}, + }, + }, + "touching EOF single token": { + `AWSPARAMSTR:///config|qp2`, + [][3]string{ + {string(config.ParamStorePrefix), "/config", "qp2"}, + }, + }, + "touching EOF multi token": { + `proto://AWSPARAMSTR:///config|user:AWSSECRETS:///creds|password@AWSPARAMSTR:///config|endpoint:AWSPARAMSTR:///config|port/?queryParam1=123&queryParam2=AWSPARAMSTR:///config|qp2`, + [][3]string{ + {string(config.ParamStorePrefix), "/config", "user"}, + {string(config.SecretMgrPrefix), "/creds", "password"}, + {string(config.ParamStorePrefix), "/config", "endpoint"}, + {string(config.ParamStorePrefix), "/config", "port"}, + {string(config.ParamStorePrefix), "/config", "qp2"}, + }, + }, } for name, tt := range ttests { @@ -67,15 +86,26 @@ BAR=something } } -func Test_Parse_should_faile_when_no_metadata_end_tag_found(t *testing.T) { +func Test_Parse_should_fail_on_metadata(t *testing.T) { ttests := map[string]struct { - input string + input string + errTyp error }{ - "without keysPath": { + "when _end_tag_found without keysPath": { `AWSSECRETS:///foo[version=1.2.3`, + parser.ErrNoEndTagFound, }, - "with keysPath": { + "when _end_tag_found with keysPath": { `AWSSECRETS:///foo|path.one[version=1.2.3`, + parser.ErrNoEndTagFound, + }, + "when no metadata has been supplied": { + `AWSSECRETS:///foo|path.one[]`, + parser.ErrMetadataEmpty, + }, + "when no metadata has been supplied - without key path": { + `AWSSECRETS:///foo[]`, + parser.ErrMetadataEmpty, }, } for name, tt := range ttests { @@ -88,7 +118,7 @@ func Test_Parse_should_faile_when_no_metadata_end_tag_found(t *testing.T) { if len(errs) != 1 { t.Fatalf("unexpected number of errors\n got: %v, wanted: 1", errs) } - if !errors.Is(errs[0], parser.ErrNoEndTagFound) { + if !errors.Is(errs[0], tt.errTyp) { t.Errorf("unexpected error type\n got: %T, wanted: %T", errs, parser.ErrNoEndTagFound) } }) @@ -141,6 +171,10 @@ func Test_Parse_ParseMetadata(t *testing.T) { `AWSSECRETS:///foo|path.one[version=1.2.3]`, &store.SecretsMgrConfig{}, }, + "nestled in text": { + `someQ=AWSPARAMSTR:///path/queryparam|p1[version=1.2.3]&anotherQ`, + &store.SecretsMgrConfig{}, + }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { @@ -165,6 +199,59 @@ func Test_Parse_ParseMetadata(t *testing.T) { } } +func Test_Parse_Path_Keys_WithParsedMetadat(t *testing.T) { + + ttests := map[string]struct { + input string + typ *store.SecretsMgrConfig + wantSanitizedPath string + wantKeyPath string + }{ + "without keysPath": { + `AWSSECRETS:///foo[version=1.2.3]`, + &store.SecretsMgrConfig{}, + "/foo", "", + }, + "with keysPath": { + `AWSSECRETS:///foo|path.one[version=1.2.3]`, + &store.SecretsMgrConfig{}, + "/foo", "path.one", + }, + "nestled in text": { + `someQ=AWSPARAMSTR:///path/queryparam|p1[version=1.2.3]&anotherQ`, + &store.SecretsMgrConfig{}, + "/path/queryparam", "p1", + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + lexerSource.Input = tt.input + cfg := config.NewConfig() + l := lexer.New(lexerSource, *cfg) + p := parser.New(l, cfg).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + t.Fatalf("%v", errs) + } + + for _, p := range parsed { + if p.ParsedToken.StoreToken() != tt.wantSanitizedPath { + t.Errorf("got %s want %s", p.ParsedToken.StoreToken(), tt.wantSanitizedPath) + } + if p.ParsedToken.LookupKeys() != tt.wantKeyPath { + t.Errorf("got %s want %s", p.ParsedToken.LookupKeys(), tt.wantKeyPath) + } + if err := p.ParsedToken.ParseMetadata(tt.typ); err != nil { + t.Fatal(err) + } + if tt.typ.Version != "1.2.3" { + t.Errorf("got %v wanted 1.2.3", tt.typ.Version) + } + } + }) + } +} + func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBlock, tokenType config.ImplementationPrefix, tokenValue, keysLookupPath string) bool { t.Helper() if stmtBlock.ParsedToken.Prefix() != tokenType { @@ -178,7 +265,7 @@ func testHelperGenDocBlock(t *testing.T, stmtBlock parser.ConfigManagerTokenBloc } if stmtBlock.ParsedToken.LookupKeys() != keysLookupPath { - t.Errorf("token LookupKeys. got=%s, wanted=%s", stmtBlock.ParsedToken.LookupKeys(), tokenValue) + t.Errorf("token LookupKeys. got=%s, wanted=%s", stmtBlock.ParsedToken.LookupKeys(), keysLookupPath) return false } diff --git a/pkg/generator/generator.go b/pkg/generator/generator.go index 500ab4b..14cc85c 100644 --- a/pkg/generator/generator.go +++ b/pkg/generator/generator.go @@ -95,63 +95,13 @@ func (c *GenVars) Config() *config.GenVarsConfig { return &c.config } -// ParsedMap is the internal working object definition and -// the return type if results are not flushed to file -type ParsedMap map[string]any - -func (pm ParsedMap) MapKeys() (keys []string) { - for k := range pm { - keys = append(keys, k) - } - return -} - -type tokenMapSafe struct { - mu *sync.Mutex - tokenMap ParsedMap -} - -func (tms *tokenMapSafe) getTokenMap() ParsedMap { - tms.mu.Lock() - defer tms.mu.Unlock() - return tms.tokenMap -} - -func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { - tms.mu.Lock() - defer tms.mu.Unlock() - // NOTE: still use the metadata in the key - // there could be different versions / labels for the same token and hence different values - // However the JSONpath look up - tms.tokenMap[key.String()] = keySeparatorLookup(key, val) -} - -type rawTokenMap struct { - mu sync.Mutex - tokenMap map[string]*config.ParsedTokenConfig -} - -func newRawTokenMap() *rawTokenMap { - return &rawTokenMap{mu: sync.Mutex{}, tokenMap: map[string]*config.ParsedTokenConfig{}} -} - -func (rtm *rawTokenMap) addToken(name string, parsedToken *config.ParsedTokenConfig) { - rtm.mu.Lock() - defer rtm.mu.Unlock() - rtm.tokenMap[name] = parsedToken -} - -func (rtm *rawTokenMap) mapOfToken() map[string]*config.ParsedTokenConfig { - rtm.mu.Lock() - defer rtm.mu.Unlock() - return rtm.tokenMap -} - // Generate generates a k/v map of the tokens with their corresponding secret/paramstore values // the standard pattern of a token should follow a path like string +// +// Called only from a slice of tokens func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { - rtm := newRawTokenMap() + rtm := NewRawTokenConfig() for _, token := range tokens { lexerSource := lexer.Source{FileName: token, FullPath: "", Input: token} l := lexer.New(lexerSource, c.config) @@ -162,7 +112,7 @@ func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { continue } for _, prsdToken := range parsed { - rtm.addToken(token, &prsdToken.ParsedToken) + rtm.AddToken(token, &prsdToken.ParsedToken) } } // pass in default initialised retrieveStrategy @@ -187,8 +137,10 @@ func IsParsed(v any, trm ParsedMap) bool { // initiates groutines with fixed size channel map // to capture responses and errors // generates ParsedMap which includes -func (c *GenVars) generate(rawMap *rawTokenMap) error { - rtm := rawMap.mapOfToken() +// +// TODO: change this slightly +func (c *GenVars) generate(rawMap *RawTokenConfig) error { + rtm := rawMap.RawTokenMap() if len(rtm) < 1 { c.Logger.Debug("no replaceable tokens found in input") return nil diff --git a/pkg/generator/generatorvars.go b/pkg/generator/generatorvars.go new file mode 100644 index 0000000..d77bcf8 --- /dev/null +++ b/pkg/generator/generatorvars.go @@ -0,0 +1,59 @@ +package generator + +import ( + "sync" + + "github.com/DevLabFoundry/configmanager/v2/internal/config" +) + +// ParsedMap is the internal working object definition and +// the return type if results are not flushed to file +type ParsedMap map[string]any + +func (pm ParsedMap) MapKeys() (keys []string) { + for k := range pm { + keys = append(keys, k) + } + return +} + +type RawTokenConfig struct { + mu *sync.Mutex + tokenMap map[string]*config.ParsedTokenConfig +} + +func NewRawTokenConfig() *RawTokenConfig { + return &RawTokenConfig{mu: &sync.Mutex{}, tokenMap: map[string]*config.ParsedTokenConfig{}} +} + +func (rtm *RawTokenConfig) AddToken(name string, parsedToken *config.ParsedTokenConfig) { + rtm.mu.Lock() + defer rtm.mu.Unlock() + rtm.tokenMap[name] = parsedToken +} + +func (rtm *RawTokenConfig) RawTokenMap() map[string]*config.ParsedTokenConfig { + rtm.mu.Lock() + defer rtm.mu.Unlock() + return rtm.tokenMap +} + +type tokenMapSafe struct { + mu *sync.Mutex + tokenMap ParsedMap +} + +func (tms *tokenMapSafe) getTokenMap() ParsedMap { + tms.mu.Lock() + defer tms.mu.Unlock() + return tms.tokenMap +} + +func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { + tms.mu.Lock() + defer tms.mu.Unlock() + // NOTE: still use the metadata in the key + // there could be different versions / labels for the same token and hence different values + // However the JSONpath look up + tms.tokenMap[key.String()] = keySeparatorLookup(key, val) +} From 0c712c79416e8246b738edc188f9e03e3feecdb3 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Thu, 23 Oct 2025 06:47:58 +0100 Subject: [PATCH 06/19] fix: add more notes in buildConfigManagerTokenFromBlocks --- internal/parser/parser.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/parser/parser.go b/internal/parser/parser.go index b70f702..e2e2c3f 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -172,9 +172,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa break } - // reached the end of a potential token + // reached the end of token if p.peekTokenIsEnd() { - // when the next token is EOF // we want set the current token as both the full and sanitized // the current lexer token is the entire configmanager token notFoundEnd = false @@ -189,6 +188,8 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // // everything is token path until (if any key separator exists) // check key separator this marks the end of a normal token path + // + // keyLookup and Metadata are optional - is always specified in that order if p.currentTokenIs(config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR) { if err := p.buildKeyPathSeparator(configManagerToken); err != nil { p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) From e0a3fbcc3cc95dff0da04741b8ac1b739243a4d7 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Thu, 23 Oct 2025 06:59:08 +0100 Subject: [PATCH 07/19] fix: update tests prep a move to v3 --- cmd/configmanager/configmanager.go | 2 +- configmanager.go | 2 +- configmanager_test.go | 2 +- {pkg/generator => generator}/generator.go | 0 {pkg/generator => generator}/generator_test.go | 2 +- {pkg/generator => generator}/generatorvars.go | 0 internal/cmdutils/cmdutils.go | 2 +- internal/cmdutils/cmdutils_test.go | 2 +- internal/cmdutils/postprocessor.go | 2 +- internal/cmdutils/postprocessor_test.go | 2 +- internal/strategy/strategy_test.go | 3 ++- pkg/.gitkeep | 0 12 files changed, 10 insertions(+), 9 deletions(-) rename {pkg/generator => generator}/generator.go (100%) rename {pkg/generator => generator}/generator_test.go (99%) rename {pkg/generator => generator}/generatorvars.go (100%) delete mode 100644 pkg/.gitkeep diff --git a/cmd/configmanager/configmanager.go b/cmd/configmanager/configmanager.go index 01af646..36925e2 100644 --- a/cmd/configmanager/configmanager.go +++ b/cmd/configmanager/configmanager.go @@ -6,10 +6,10 @@ import ( "io" "github.com/DevLabFoundry/configmanager/v2" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" "github.com/spf13/cobra" ) diff --git a/configmanager.go b/configmanager.go index 7c512f7..6944088 100644 --- a/configmanager.go +++ b/configmanager.go @@ -11,11 +11,11 @@ import ( "slices" "strings" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/lexer" "github.com/DevLabFoundry/configmanager/v2/internal/log" "github.com/DevLabFoundry/configmanager/v2/internal/parser" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" "github.com/a8m/envsubst" "gopkg.in/yaml.v3" ) diff --git a/configmanager_test.go b/configmanager_test.go index ea57805..339a573 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -12,7 +12,7 @@ import ( "github.com/DevLabFoundry/configmanager/v2" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/go-test/deep" ) diff --git a/pkg/generator/generator.go b/generator/generator.go similarity index 100% rename from pkg/generator/generator.go rename to generator/generator.go diff --git a/pkg/generator/generator_test.go b/generator/generator_test.go similarity index 99% rename from pkg/generator/generator_test.go rename to generator/generator_test.go index bfb91b9..59dec7b 100644 --- a/pkg/generator/generator_test.go +++ b/generator/generator_test.go @@ -11,7 +11,7 @@ import ( "github.com/DevLabFoundry/configmanager/v2/internal/store" "github.com/DevLabFoundry/configmanager/v2/internal/strategy" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v2/generator" ) type mockGenerate struct { diff --git a/pkg/generator/generatorvars.go b/generator/generatorvars.go similarity index 100% rename from pkg/generator/generatorvars.go rename to generator/generatorvars.go diff --git a/internal/cmdutils/cmdutils.go b/internal/cmdutils/cmdutils.go index f1f71dc..71fb4e0 100644 --- a/internal/cmdutils/cmdutils.go +++ b/internal/cmdutils/cmdutils.go @@ -13,7 +13,7 @@ import ( "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/spf13/cobra" ) diff --git a/internal/cmdutils/cmdutils_test.go b/internal/cmdutils/cmdutils_test.go index 68e1ab4..0fd6591 100644 --- a/internal/cmdutils/cmdutils_test.go +++ b/internal/cmdutils/cmdutils_test.go @@ -8,11 +8,11 @@ import ( "strings" "testing" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" "github.com/DevLabFoundry/configmanager/v2/internal/config" log "github.com/DevLabFoundry/configmanager/v2/internal/log" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" "github.com/spf13/cobra" ) diff --git a/internal/cmdutils/postprocessor.go b/internal/cmdutils/postprocessor.go index 8419cc7..87fb0d1 100644 --- a/internal/cmdutils/postprocessor.go +++ b/internal/cmdutils/postprocessor.go @@ -5,8 +5,8 @@ import ( "io" "strings" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" ) // PostProcessor diff --git a/internal/cmdutils/postprocessor_test.go b/internal/cmdutils/postprocessor_test.go index 001ea68..3a4fa0b 100644 --- a/internal/cmdutils/postprocessor_test.go +++ b/internal/cmdutils/postprocessor_test.go @@ -5,10 +5,10 @@ import ( "strings" "testing" + "github.com/DevLabFoundry/configmanager/v2/generator" "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" "github.com/DevLabFoundry/configmanager/v2/internal/config" "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" ) func postprocessorHelper(t *testing.T) { diff --git a/internal/strategy/strategy_test.go b/internal/strategy/strategy_test.go index 0f642bd..12eb895 100644 --- a/internal/strategy/strategy_test.go +++ b/internal/strategy/strategy_test.go @@ -96,7 +96,8 @@ func Test_CustomStrategyFuncMap_add_own(t *testing.T) { t.Run(name, func(t *testing.T) { called := 0 genVarsConf := config.NewConfig() - token, _ := config.NewToken("AZTABLESTORE://mountPath/token", *genVarsConf) + token, _ := config.NewToken(config.AzTableStorePrefix, *config.NewConfig()) + token.WithSanitizedToken("mountPath/token") var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { m := &mockGenerate{"AZTABLESTORE://mountPath/token", "bar", nil} diff --git a/pkg/.gitkeep b/pkg/.gitkeep deleted file mode 100644 index e69de29..0000000 From eadebcdc58aefc3d46d949235223aa6f738c452a Mon Sep 17 00:00:00 2001 From: dnitsch Date: Thu, 23 Oct 2025 07:01:11 +0100 Subject: [PATCH 08/19] !BREAKING CHANGE: update to package to v3 +semver: BREAKING +semver: MAJOR --- Dockerfile | 2 +- cmd/configmanager/configmanager.go | 10 +++++----- cmd/configmanager/configmanager_test.go | 4 ++-- cmd/configmanager/fromfileinput.go | 2 +- cmd/main.go | 4 ++-- configmanager.go | 10 +++++----- configmanager_test.go | 8 ++++---- docs/examples.md | 8 ++++---- eirctl.yaml | 2 +- examples/examples.go | 2 +- generator/generator.go | 10 +++++----- generator/generator_test.go | 14 +++++++------- generator/generatorvars.go | 2 +- go.mod | 2 +- internal/cmdutils/cmdutils.go | 6 +++--- internal/cmdutils/cmdutils_test.go | 10 +++++----- internal/cmdutils/postprocessor.go | 4 ++-- internal/cmdutils/postprocessor_test.go | 8 ++++---- internal/config/config_test.go | 4 ++-- internal/lexer/lexer.go | 2 +- internal/lexer/lexer_test.go | 4 ++-- internal/log/log_test.go | 4 ++-- internal/parser/parser.go | 6 +++--- internal/parser/parser_test.go | 10 +++++----- internal/store/azappconf.go | 4 ++-- internal/store/azappconf_test.go | 8 ++++---- internal/store/azkeyvault.go | 4 ++-- internal/store/azkeyvault_test.go | 8 ++++---- internal/store/aztablestorage.go | 4 ++-- internal/store/aztablestorage_test.go | 8 ++++---- internal/store/gcpsecrets.go | 4 ++-- internal/store/gcpsecrets_test.go | 8 ++++---- internal/store/hashivault.go | 4 ++-- internal/store/hashivault_test.go | 8 ++++---- internal/store/paramstore.go | 4 ++-- internal/store/paramstore_test.go | 8 ++++---- internal/store/secretsmanager.go | 4 ++-- internal/store/secretsmanager_test.go | 8 ++++---- internal/store/store.go | 2 +- internal/strategy/strategy.go | 6 +++--- internal/strategy/strategy_test.go | 10 +++++----- 41 files changed, 120 insertions(+), 120 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8dd4e1a..5871513 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app COPY ./ /app RUN CGO_ENABLED=0 go build -mod=readonly -buildvcs=false \ - -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \ + -ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Version=${Version}\" -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Revision=${Revision}\" -extldflags -static" \ -o bin/configmanager cmd/main.go FROM docker.io/alpine:3 diff --git a/cmd/configmanager/configmanager.go b/cmd/configmanager/configmanager.go index 36925e2..d0f64e9 100644 --- a/cmd/configmanager/configmanager.go +++ b/cmd/configmanager/configmanager.go @@ -5,11 +5,11 @@ import ( "fmt" "io" - "github.com/DevLabFoundry/configmanager/v2" - "github.com/DevLabFoundry/configmanager/v2/generator" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/spf13/cobra" ) diff --git a/cmd/configmanager/configmanager_test.go b/cmd/configmanager/configmanager_test.go index fce284a..5089ad3 100644 --- a/cmd/configmanager/configmanager_test.go +++ b/cmd/configmanager/configmanager_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - cmd "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + cmd "github.com/DevLabFoundry/configmanager/v3/cmd/configmanager" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) type cmdTestInput struct { diff --git a/cmd/configmanager/fromfileinput.go b/cmd/configmanager/fromfileinput.go index 4b8e915..f531033 100644 --- a/cmd/configmanager/fromfileinput.go +++ b/cmd/configmanager/fromfileinput.go @@ -3,7 +3,7 @@ package cmd import ( "fmt" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" "github.com/spf13/cobra" ) diff --git a/cmd/main.go b/cmd/main.go index ee4999a..4f10c03 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,8 +4,8 @@ import ( "context" "os" - cfgmgr "github.com/DevLabFoundry/configmanager/v2/cmd/configmanager" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + cfgmgr "github.com/DevLabFoundry/configmanager/v3/cmd/configmanager" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) func main() { diff --git a/configmanager.go b/configmanager.go index 6944088..e6700f3 100644 --- a/configmanager.go +++ b/configmanager.go @@ -11,11 +11,11 @@ import ( "slices" "strings" - "github.com/DevLabFoundry/configmanager/v2/generator" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/lexer" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/parser" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/parser" "github.com/a8m/envsubst" "gopkg.in/yaml.v3" ) diff --git a/configmanager_test.go b/configmanager_test.go index 339a573..a32510a 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -9,10 +9,10 @@ import ( "sort" "testing" - "github.com/DevLabFoundry/configmanager/v2" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/generator" + "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/go-test/deep" ) diff --git a/docs/examples.md b/docs/examples.md index 73313c8..07634fa 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -84,8 +84,8 @@ import ( "context" "fmt" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" - "github.com/DevLabFoundry/configmanager/v2" + "github.com/DevLabFoundry/configmanager/v3/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3" ) func main() { @@ -128,8 +128,8 @@ import ( "log" "os" - "github.com/DevLabFoundry/configmanager/v2" - "github.com/DevLabFoundry/configmanager/v2/pkg/generator" + "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/pkg/generator" ) var ( diff --git a/eirctl.yaml b/eirctl.yaml index fd5349c..15d1a96 100644 --- a/eirctl.yaml +++ b/eirctl.yaml @@ -53,7 +53,7 @@ tasks: - | mkdir -p .deps unset GOTOOLCHAIN - ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Version=${VERSION}\" -X \"github.com/DevLabFoundry/configmanager/v2/cmd/configmanager.Revision=${REVISION}\" -extldflags -static" + ldflags="-s -w -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Version=${VERSION}\" -X \"github.com/DevLabFoundry/configmanager/v3/cmd/configmanager.Revision=${REVISION}\" -extldflags -static" GOPATH=/eirctl/.deps GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} CGO_ENABLED=0 go build -mod=readonly -buildvcs=false -ldflags="$ldflags" \ -o ./dist/configmanager-${BUILD_GOOS}-${BUILD_GOARCH}${BUILD_SUFFIX} ./cmd echo "---" diff --git a/examples/examples.go b/examples/examples.go index b15016a..b21ed36 100644 --- a/examples/examples.go +++ b/examples/examples.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "github.com/DevLabFoundry/configmanager/v2" + "github.com/DevLabFoundry/configmanager/v3" ) const DO_STUFF_WITH_VALS_HERE = "connstring:user@%v:host=%s/someschema..." diff --git a/generator/generator.go b/generator/generator.go index 14cc85c..0da95e1 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -9,11 +9,11 @@ import ( "strconv" "sync" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/lexer" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/parser" - "github.com/DevLabFoundry/configmanager/v2/internal/strategy" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/parser" + "github.com/DevLabFoundry/configmanager/v3/internal/strategy" "github.com/spyzhov/ajson" ) diff --git a/generator/generator_test.go b/generator/generator_test.go index 59dec7b..9d5f66c 100644 --- a/generator/generator_test.go +++ b/generator/generator_test.go @@ -6,12 +6,12 @@ import ( "fmt" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/strategy" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" - "github.com/DevLabFoundry/configmanager/v2/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/strategy" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/generator" ) type mockGenerate struct { @@ -177,7 +177,7 @@ func Test_IsParsed(t *testing.T) { // "strings" // "testing" -// "github.com/DevLabFoundry/configmanager/v2/internal/testutils" +// "github.com/DevLabFoundry/configmanager/v3/internal/testutils" // ) // var ( diff --git a/generator/generatorvars.go b/generator/generatorvars.go index d77bcf8..ed2af9f 100644 --- a/generator/generatorvars.go +++ b/generator/generatorvars.go @@ -3,7 +3,7 @@ package generator import ( "sync" - "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/config" ) // ParsedMap is the internal working object definition and diff --git a/go.mod b/go.mod index d4718c9..0ab4448 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/DevLabFoundry/configmanager/v2 +module github.com/DevLabFoundry/configmanager/v3 go 1.25.1 diff --git a/internal/cmdutils/cmdutils.go b/internal/cmdutils/cmdutils.go index 71fb4e0..87e233a 100644 --- a/internal/cmdutils/cmdutils.go +++ b/internal/cmdutils/cmdutils.go @@ -11,9 +11,9 @@ import ( "os" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/spf13/cobra" ) diff --git a/internal/cmdutils/cmdutils_test.go b/internal/cmdutils/cmdutils_test.go index 0fd6591..29cd6ea 100644 --- a/internal/cmdutils/cmdutils_test.go +++ b/internal/cmdutils/cmdutils_test.go @@ -8,11 +8,11 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/generator" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - log "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + log "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/spf13/cobra" ) diff --git a/internal/cmdutils/postprocessor.go b/internal/cmdutils/postprocessor.go index 87fb0d1..77ef0c2 100644 --- a/internal/cmdutils/postprocessor.go +++ b/internal/cmdutils/postprocessor.go @@ -5,8 +5,8 @@ import ( "io" "strings" - "github.com/DevLabFoundry/configmanager/v2/generator" - "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/config" ) // PostProcessor diff --git a/internal/cmdutils/postprocessor_test.go b/internal/cmdutils/postprocessor_test.go index 3a4fa0b..ac1a30c 100644 --- a/internal/cmdutils/postprocessor_test.go +++ b/internal/cmdutils/postprocessor_test.go @@ -5,10 +5,10 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/generator" - "github.com/DevLabFoundry/configmanager/v2/internal/cmdutils" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/generator" + "github.com/DevLabFoundry/configmanager/v3/internal/cmdutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func postprocessorHelper(t *testing.T) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 488e1b8..c85253c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,8 +3,8 @@ package config_test import ( "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func Test_SelfName(t *testing.T) { diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index e785727..41d89dc 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -4,7 +4,7 @@ package lexer import ( - "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/config" ) // nonText characters captures all character sets that are _not_ assignable to TEXT diff --git a/internal/lexer/lexer_test.go b/internal/lexer/lexer_test.go index 5ab297b..79f945a 100644 --- a/internal/lexer/lexer_test.go +++ b/internal/lexer/lexer_test.go @@ -3,8 +3,8 @@ package lexer_test import ( "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" ) func Test_Lexer_NextToken(t *testing.T) { diff --git a/internal/log/log_test.go b/internal/log/log_test.go index be24a28..10922fa 100644 --- a/internal/log/log_test.go +++ b/internal/log/log_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func Test_LogInfo(t *testing.T) { diff --git a/internal/parser/parser.go b/internal/parser/parser.go index e2e2c3f..1de07e4 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -6,9 +6,9 @@ import ( "os" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/lexer" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/a8m/envsubst" ) diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 4bf5d19..4fc9692 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -5,11 +5,11 @@ import ( "os" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/lexer" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/parser" - "github.com/DevLabFoundry/configmanager/v2/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/lexer" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/parser" + "github.com/DevLabFoundry/configmanager/v3/internal/store" ) var lexerSource = lexer.Source{FileName: "bar", FullPath: "/foo/bar"} diff --git a/internal/store/azappconf.go b/internal/store/azappconf.go index 92adab5..b12f501 100644 --- a/internal/store/azappconf.go +++ b/internal/store/azappconf.go @@ -11,8 +11,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) // appConfApi diff --git a/internal/store/azappconf_test.go b/internal/store/azappconf_test.go index eff0872..922f04e 100644 --- a/internal/store/azappconf_test.go +++ b/internal/store/azappconf_test.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - logger "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + logger "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func azAppConfCommonChecker(t *testing.T, key string, expectedKey string, expectLabel string, opts *azappconfig.GetSettingOptions) { diff --git a/internal/store/azkeyvault.go b/internal/store/azkeyvault.go index bd2a2a4..51f1047 100644 --- a/internal/store/azkeyvault.go +++ b/internal/store/azkeyvault.go @@ -8,8 +8,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) type kvApi interface { diff --git a/internal/store/azkeyvault_test.go b/internal/store/azkeyvault_test.go index f4562ab..9497096 100644 --- a/internal/store/azkeyvault_test.go +++ b/internal/store/azkeyvault_test.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func Test_azSplitToken(t *testing.T) { diff --git a/internal/store/aztablestorage.go b/internal/store/aztablestorage.go index c9bb275..3326a31 100644 --- a/internal/store/aztablestorage.go +++ b/internal/store/aztablestorage.go @@ -12,8 +12,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" ) var ErrIncorrectlyStructuredToken = errors.New("incorrectly structured token") diff --git a/internal/store/aztablestorage_test.go b/internal/store/aztablestorage_test.go index 64d58b4..50a9c62 100644 --- a/internal/store/aztablestorage_test.go +++ b/internal/store/aztablestorage_test.go @@ -9,10 +9,10 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/data/aztables" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" ) func azTableStoreCommonChecker(t *testing.T, partitionKey, rowKey, expectedPartitionKey, expectedRowKey string) { diff --git a/internal/store/gcpsecrets.go b/internal/store/gcpsecrets.go index 67cacec..0a78bb9 100644 --- a/internal/store/gcpsecrets.go +++ b/internal/store/gcpsecrets.go @@ -6,8 +6,8 @@ import ( gcpsecrets "cloud.google.com/go/secretmanager/apiv1" gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/googleapis/gax-go/v2" ) diff --git a/internal/store/gcpsecrets_test.go b/internal/store/gcpsecrets_test.go index d328650..471a238 100644 --- a/internal/store/gcpsecrets_test.go +++ b/internal/store/gcpsecrets_test.go @@ -9,10 +9,10 @@ import ( "testing" gcpsecretspb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/googleapis/gax-go/v2" ) diff --git a/internal/store/hashivault.go b/internal/store/hashivault.go index 6a505e2..84008e9 100644 --- a/internal/store/hashivault.go +++ b/internal/store/hashivault.go @@ -8,8 +8,8 @@ import ( "strconv" "strings" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" vault "github.com/hashicorp/vault/api" auth "github.com/hashicorp/vault/api/auth/aws" diff --git a/internal/store/hashivault_test.go b/internal/store/hashivault_test.go index 90e35f7..4077ea8 100644 --- a/internal/store/hashivault_test.go +++ b/internal/store/hashivault_test.go @@ -6,10 +6,10 @@ import ( "os" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" vault "github.com/hashicorp/vault/api" ) diff --git a/internal/store/paramstore.go b/internal/store/paramstore.go index 5e75d5f..493a49e 100644 --- a/internal/store/paramstore.go +++ b/internal/store/paramstore.go @@ -3,8 +3,8 @@ package store import ( "context" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/aws/aws-sdk-go-v2/aws" awsConf "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ssm" diff --git a/internal/store/paramstore_test.go b/internal/store/paramstore_test.go index 47537cf..ec73c59 100644 --- a/internal/store/paramstore_test.go +++ b/internal/store/paramstore_test.go @@ -6,10 +6,10 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/aws/aws-sdk-go-v2/service/ssm/types" ) diff --git a/internal/store/secretsmanager.go b/internal/store/secretsmanager.go index 1f764d4..d49c808 100644 --- a/internal/store/secretsmanager.go +++ b/internal/store/secretsmanager.go @@ -3,8 +3,8 @@ package store import ( "context" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/aws/aws-sdk-go-v2/aws" awsconf "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" diff --git a/internal/store/secretsmanager_test.go b/internal/store/secretsmanager_test.go index 30f9849..39bd475 100644 --- a/internal/store/secretsmanager_test.go +++ b/internal/store/secretsmanager_test.go @@ -6,10 +6,10 @@ import ( "strings" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" ) diff --git a/internal/store/store.go b/internal/store/store.go index 42adf5b..00e0c4a 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -3,7 +3,7 @@ package store import ( "errors" - "github.com/DevLabFoundry/configmanager/v2/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/config" ) const implementationNetworkErr string = "implementation %s error: %v for token: %s" diff --git a/internal/strategy/strategy.go b/internal/strategy/strategy.go index a5b4981..92b5c3b 100644 --- a/internal/strategy/strategy.go +++ b/internal/strategy/strategy.go @@ -9,9 +9,9 @@ import ( "fmt" "sync" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" ) var ErrTokenInvalid = errors.New("invalid token - cannot get prefix") diff --git a/internal/strategy/strategy_test.go b/internal/strategy/strategy_test.go index 12eb895..b09548f 100644 --- a/internal/strategy/strategy_test.go +++ b/internal/strategy/strategy_test.go @@ -7,11 +7,11 @@ import ( "os" "testing" - "github.com/DevLabFoundry/configmanager/v2/internal/config" - log "github.com/DevLabFoundry/configmanager/v2/internal/log" - "github.com/DevLabFoundry/configmanager/v2/internal/store" - "github.com/DevLabFoundry/configmanager/v2/internal/strategy" - "github.com/DevLabFoundry/configmanager/v2/internal/testutils" + "github.com/DevLabFoundry/configmanager/v3/internal/config" + log "github.com/DevLabFoundry/configmanager/v3/internal/log" + "github.com/DevLabFoundry/configmanager/v3/internal/store" + "github.com/DevLabFoundry/configmanager/v3/internal/strategy" + "github.com/DevLabFoundry/configmanager/v3/internal/testutils" "github.com/go-test/deep" ) From 38b69aacf1797be5bb0fdcf638cbda28cd003dd1 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Fri, 31 Oct 2025 16:25:27 +0000 Subject: [PATCH 09/19] fix: update go version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0ab4448..fe3973c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/DevLabFoundry/configmanager/v3 -go 1.25.1 +go 1.25.3 require ( cloud.google.com/go/secretmanager v1.15.0 From 87efeb3e420fd04dc80700dc72ce995d0c080679 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Fri, 31 Oct 2025 19:12:38 +0000 Subject: [PATCH 10/19] +semver: breaking fix: add tests +semver: breaking breaking: update the interface of the configmanager API --- .github/workflows/build.yml | 6 +- .github/workflows/release.yml | 6 +- configmanager.go | 146 +----- configmanager_test.go | 442 ++----------------- eirctl.yaml | 11 +- examples/examples.go | 84 ++-- generator/generator.go | 234 +++++----- generator/generator_test.go | 564 ++++++++---------------- generator/generatorvars.go | 73 ++- internal/cmdutils/cmdutils.go | 11 +- internal/cmdutils/cmdutils_test.go | 20 +- internal/cmdutils/postprocessor.go | 12 +- internal/cmdutils/postprocessor_test.go | 10 +- internal/config/config.go | 33 +- internal/parser/parser.go | 12 + internal/store/azappconf.go | 2 +- internal/store/azappconf_test.go | 4 +- internal/store/azkeyvault.go | 2 +- internal/store/azkeyvault_test.go | 2 +- internal/store/aztablestorage.go | 2 +- internal/store/aztablestorage_test.go | 6 +- internal/store/gcpsecrets.go | 2 +- internal/store/gcpsecrets_test.go | 2 +- internal/store/hashivault.go | 2 +- internal/store/hashivault_test.go | 2 +- internal/store/paramstore.go | 2 +- internal/store/paramstore_test.go | 2 +- internal/store/secretsmanager.go | 2 +- internal/store/secretsmanager_test.go | 113 +++-- internal/store/store.go | 4 +- internal/strategy/strategy.go | 82 ++-- internal/strategy/strategy_test.go | 15 +- 32 files changed, 680 insertions(+), 1230 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebccfcd..946bfc5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,11 +30,11 @@ jobs: git config user.email ${{ github.actor }}-ci@gha.org git config user.name ${{ github.actor }} - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v3.0.0 + uses: gittools/actions/gitversion/setup@v4.1.0 with: - versionSpec: "5.x" + versionSpec: "6.x" - name: Set SemVer Version - uses: gittools/actions/gitversion/execute@v3.0.0 + uses: gittools/actions/gitversion/execute@v4.1.0 id: gitversion - name: echo VERSIONS diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ddc903..2faca47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,11 +31,11 @@ jobs: git config user.email ${{ github.actor }}-ci@gha.org git config user.name ${{ github.actor }} - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v3.0.0 + uses: gittools/actions/gitversion/setup@v4.1.0 with: - versionSpec: '5.x' + versionSpec: '6.x' - name: Set SemVer Version - uses: gittools/actions/gitversion/execute@v3.0.0 + uses: gittools/actions/gitversion/execute@v4.1.0 id: gitversion release: diff --git a/configmanager.go b/configmanager.go index e6700f3..f3f6134 100644 --- a/configmanager.go +++ b/configmanager.go @@ -2,22 +2,17 @@ package configmanager import ( "context" - "encoding/json" "errors" "fmt" "io" - "regexp" "slices" "strings" "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/DevLabFoundry/configmanager/v3/internal/config" - "github.com/DevLabFoundry/configmanager/v3/internal/lexer" "github.com/DevLabFoundry/configmanager/v3/internal/log" - "github.com/DevLabFoundry/configmanager/v3/internal/parser" "github.com/a8m/envsubst" - "gopkg.in/yaml.v3" ) const ( @@ -26,7 +21,7 @@ const ( // generateAPI type generateAPI interface { - Generate(tokens []string) (generator.ParsedMap, error) + Generate(tokens []string) (generator.ReplacedToken, error) } type ConfigManager struct { @@ -75,25 +70,15 @@ func (c *ConfigManager) WithGenerator(generator generateAPI) *ConfigManager { // Retrieve gets a rawMap from a set implementation // will be empty if no matches found -func (c *ConfigManager) Retrieve(tokens []string) (generator.ParsedMap, error) { - return c.retrieve(tokens) -} - -func (c *ConfigManager) retrieve(tokens []string) (generator.ParsedMap, error) { +func (c *ConfigManager) Retrieve(tokens []string) (generator.ReplacedToken, error) { return c.generator.Generate(tokens) } var ErrEnvSubst = errors.New("envsubst enabled and errored on") -// RetrieveWithInputReplaced parses given input against all possible token strings -// using regex to grab a list of found tokens in the given string and returns the replaced string -func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) { +// RetrieveReplacedString parses given input against all possible token strings +func (c *ConfigManager) RetrieveReplacedString(input string) (string, error) { // replaces all env vars using strict mode of no unset and no empty - // - // NOTE: this happens before the FindTokens is called - // currently it uses a regex, and envsubst uses a more robust lexer => parser mechanism - // - // NOTE: configmanager needs an own lexer => parser to allow for easier modification extension in the future if c.GeneratorConfig().EnvSubstEnabled() { var err error input, err = envsubst.StringRestrictedNoDigit(input, true, true, false) @@ -102,7 +87,8 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) } } - m, err := c.retrieve(FindTokens(input)) + // calling the same Generate method with the input as single item in a slice + m, err := c.generator.Generate([]string{input}) if err != nil { return "", err @@ -111,40 +97,14 @@ func (c *ConfigManager) RetrieveWithInputReplaced(input string) (string, error) return replaceString(m, input), nil } -var ErrTokenDiscovery = errors.New("failed to discover tokens") - -func (c *ConfigManager) DiscoverTokens(input string) ([]config.ParsedTokenConfig, error) { - lexerSource := lexer.Source{FileName: "", FullPath: "", Input: input} - l := lexer.New(lexerSource, *c.Config) - p := parser.New(l, c.Config).WithLogger(c.logger) - parsed, errs := p.Parse() - if len(errs) > 0 { - return nil, fmt.Errorf("%w in input (%s) with errors: %q", ErrTokenDiscovery, input[0:min(len(input), 25)], errs) - } - - pt := []config.ParsedTokenConfig{} - for _, prsdToken := range parsed { - pt = append(pt, prsdToken.ParsedToken) - } - return pt, nil -} - -// FindTokens extracts all replaceable tokens -// from a given input string -// -// Deprecated: FindTokens relies on Regex. -// Use func (c *ConfigManager) DiscoverTokens(input string) []*config.ParsedTokenConfig -func FindTokens(input string) []string { - tokens := []string{} - for k := range config.VarPrefix { - matches := regexp.MustCompile(regexp.QuoteMeta(string(k))+`.(`+TERMINATING_CHAR+`+)`).FindAllString(input, -1) - tokens = append(tokens, matches...) - } - return tokens +// RetrieveReplacedBytes is functionally identical RetrieveReplacedString +func (c *ConfigManager) RetrieveReplacedBytes(input []byte) ([]byte, error) { + r, err := c.RetrieveReplacedString(string(input)) + return []byte(r), err } // replaceString fills tokens in a provided input with their actual secret/config values -func replaceString(inputMap generator.ParsedMap, inputString string) string { +func replaceString(inputMap generator.ReplacedToken, inputString string) string { oldNew := []string(nil) // ordered values by index @@ -155,7 +115,7 @@ func replaceString(inputMap generator.ParsedMap, inputString string) string { return replacer.Replace(inputString) } -func orderedKeysList(inputMap generator.ParsedMap) []string { +func orderedKeysList(inputMap generator.ReplacedToken) []string { mkeys := inputMap.MapKeys() // order map by keys length so that when passed to the // replacer it will replace the longest first @@ -165,85 +125,3 @@ func orderedKeysList(inputMap generator.ParsedMap) []string { slices.Sort(mkeys) return mkeys } - -// RetrieveMarshalledJson -// -// It marshalls an input pointer value of a type with appropriate struct tags in JSON -// marshalls it into a string and runs the appropriate token replacement. -// and fills the same pointer value with the replaced fields. -// -// This is useful for when you have another tool or framework already passing you a known type. -// e.g. a CRD Spec in kubernetes - where you POSTed the json/yaml spec with tokens in it -// but now want to use them with tokens replaced for values in a stateless way. -// -// Enables you to store secrets in CRD Specs and other metadata your controller can use -func (cm *ConfigManager) RetrieveMarshalledJson(input any) error { - - // marshall type into a []byte - // with tokens in a string like object - rawBytes, err := json.Marshal(input) - if err != nil { - return err - } - // run the replacement of tokens for values - replacedString, err := cm.RetrieveWithInputReplaced(string(rawBytes)) - if err != nil { - return err - } - // replace the original pointer value with replaced tokens - if err := json.Unmarshal([]byte(replacedString), input); err != nil { - return err - } - return nil -} - -// RetrieveUnmarshalledFromJson -// It accepts an already marshalled byte slice and pointer to the value type. -// It fills the type with the replaced -func (c *ConfigManager) RetrieveUnmarshalledFromJson(input []byte, output any) error { - replaced, err := c.RetrieveWithInputReplaced(string(input)) - if err != nil { - return err - } - if err := json.Unmarshal([]byte(replaced), output); err != nil { - return err - } - return nil -} - -// RetrieveMarshalledYaml -// -// Same as RetrieveMarshalledJson -func (cm *ConfigManager) RetrieveMarshalledYaml(input any) error { - - // marshall type into a []byte - // with tokens in a string like object - rawBytes, err := yaml.Marshal(input) - if err != nil { - return err - } - // run the replacement of tokens for values - replacedString, err := cm.RetrieveWithInputReplaced(string(rawBytes)) - if err != nil { - return err - } - // replace the original pointer value with replaced tokens - if err := yaml.Unmarshal([]byte(replacedString), input); err != nil { - return err - } - return nil -} - -// RetrieveUnmarshalledFromYaml -// -// Same as RetrieveUnmarshalledFromJson -func (c *ConfigManager) RetrieveUnmarshalledFromYaml(input []byte, output any) error { - replaced, err := c.RetrieveWithInputReplaced(string(input)) - if err != nil { - return err - } - if err := yaml.Unmarshal([]byte(replaced), output); err != nil { - return err - } - return nil -} diff --git a/configmanager_test.go b/configmanager_test.go index a32510a..1321232 100644 --- a/configmanager_test.go +++ b/configmanager_test.go @@ -2,29 +2,29 @@ package configmanager_test import ( "context" + "encoding/json" "fmt" "os" "reflect" - "slices" - "sort" "testing" "github.com/DevLabFoundry/configmanager/v3" + "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/DevLabFoundry/configmanager/v3/internal/config" "github.com/DevLabFoundry/configmanager/v3/internal/testutils" - "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/go-test/deep" + "gopkg.in/yaml.v3" ) type mockGenerator struct { - generate func(tokens []string) (generator.ParsedMap, error) + generate func(tokens []string) (generator.ReplacedToken, error) } -func (m *mockGenerator) Generate(tokens []string) (generator.ParsedMap, error) { +func (m *mockGenerator) Generate(tokens []string) (generator.ReplacedToken, error) { if m.generate != nil { return m.generate(tokens) } - pm := generator.ParsedMap{} + pm := generator.ReplacedToken{} pm["FOO#/test"] = "val1" pm["ANOTHER://bar/quz"] = "fux" pm["ZODTHER://bar/quz"] = "xuf" @@ -64,15 +64,15 @@ func Test_Retrieve_from_token_list(t *testing.T) { } } -func Test_retrieveWithInputReplaced(t *testing.T) { +func Test_retrieveReplacedBytes(t *testing.T) { tests := map[string]struct { name string - input string + input []byte genvar *mockGenerator expect string }{ "strYaml": { - input: ` + input: []byte(` space: preserved indents: preserved arr: [ "FOO#/test" ] @@ -80,7 +80,7 @@ space: preserved arr: - "FOO#/test" - ANOTHER://bar/quz -`, +`), genvar: &mockGenerator{}, expect: ` space: preserved @@ -93,11 +93,11 @@ space: preserved `, }, "strToml": { - input: ` + input: []byte(` // TOML [[somestuff]] key = "FOO#/test" -`, +`), genvar: &mockGenerator{}, expect: ` // TOML @@ -106,14 +106,14 @@ key = "val1" `, }, "strTomlWithoutQuotes": { - input: ` + input: []byte(` // TOML [[somestuff]] key = FOO#/test,FOO#/test-FOO#/test key2 = FOO#/test key3 = FOO#/test key4 = FOO#/test -`, +`), genvar: &mockGenerator{}, expect: ` // TOML @@ -125,7 +125,7 @@ key4 = val1 `, }, "strTomlWithoutMultiline": { - input: ` + input: []byte(` export FOO='FOO#/test' export FOO1=FOO#/test export FOO2="FOO#/test" @@ -135,7 +135,7 @@ export FOO4=FOO#/test [[section]] foo23 = FOO#/test -`, +`), genvar: &mockGenerator{}, expect: ` export FOO='val1' @@ -150,7 +150,7 @@ foo23 = val1 `, }, "escaped input": { - input: `"{\"patchPayloadTemplate\":\"{\\\"password\\\":\\\"FOO#/test\\\",\\\"passwordConfirm\\\":\\\"FOO#/test\\\"}\\n\"}"`, + input: []byte(`"{\"patchPayloadTemplate\":\"{\\\"password\\\":\\\"FOO#/test\\\",\\\"passwordConfirm\\\":\\\"FOO#/test\\\"}\\n\"}"`), genvar: &mockGenerator{}, expect: `"{\"patchPayloadTemplate\":\"{\\\"password\\\":\\\"val1\\\",\\\"passwordConfirm\\\":\\\"val1\\\"}\\n\"}"`, }, @@ -160,11 +160,11 @@ foo23 = val1 t.Run(tt.name, func(t *testing.T) { cm := configmanager.New(context.TODO()) cm.WithGenerator(tt.genvar) - got, err := cm.RetrieveWithInputReplaced(tt.input) + got, err := cm.RetrieveReplacedBytes([]byte(tt.input)) if err != nil { t.Errorf("failed with %v", err) } - if got != tt.expect { + if string(got) != string(tt.expect) { t.Errorf(testutils.TestPhrase, got, tt.expect) } }) @@ -198,7 +198,7 @@ func Test_replaceString_with_envsubst(t *testing.T) { cm := configmanager.New(context.TODO()) cm.WithGenerator(tt.genvar) cm.Config.WithEnvSubst(true) - got, err := cm.RetrieveWithInputReplaced(tt.input) + got, err := cm.RetrieveReplacedString(tt.input) if err != nil { t.Errorf("failed with %v", err) } @@ -264,8 +264,8 @@ var marshallTests = map[string]struct { }, generator: func(t *testing.T) *mockGenerator { m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) + m.generate = func(tokens []string) (generator.ReplacedToken, error) { + pm := make(generator.ReplacedToken) pm[testTokenAWS] = "baz" return pm, nil } @@ -284,8 +284,8 @@ var marshallTests = map[string]struct { }, generator: func(t *testing.T) *mockGenerator { m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) + m.generate = func(tokens []string) (generator.ReplacedToken, error) { + pm := make(generator.ReplacedToken) pm[testTokenAWS] = "baz" return pm, nil } @@ -294,30 +294,44 @@ var marshallTests = map[string]struct { }, } -func Test_RetrieveMarshalledJson(t *testing.T) { +func Test_RetrieveBytes_MarshalledJson(t *testing.T) { for name, tt := range marshallTests { t.Run(name, func(t *testing.T) { c := configmanager.New(context.TODO()) c.Config.WithTokenSeparator("://") c.WithGenerator(tt.generator(t)) - input := &tt.testType - err := c.RetrieveMarshalledJson(input) - MarhsalledHelper(t, err, input, &tt.expect) + b, err := json.Marshal(tt.testType) + if err != nil { + t.Fatal(err) + } + got, err := c.RetrieveReplacedBytes(b) + output := testNestedStruct{} + json.Unmarshal(got, &output) + MarhsalledHelper(t, err, &output, &tt.expect) }) } } -func Test_YamlRetrieveMarshalled(t *testing.T) { +// func Example_RetrieveReplacedBytesMarshalledJSON(t *testing.T) { +// return +// } + +func Test_RetrieveBytes_MarshalledYaml(t *testing.T) { for name, tt := range marshallTests { t.Run(name, func(t *testing.T) { c := configmanager.New(context.TODO()) c.Config.WithTokenSeparator("://") c.WithGenerator(tt.generator(t)) - input := &tt.testType - err := c.RetrieveMarshalledYaml(input) - MarhsalledHelper(t, err, input, &tt.expect) + b, err := yaml.Marshal(tt.testType) + if err != nil { + t.Fatal(err) + } + got, err := c.RetrieveReplacedBytes(b) + output := testNestedStruct{} + yaml.Unmarshal(got, &output) + MarhsalledHelper(t, err, &output, &tt.expect) }) } } @@ -332,370 +346,6 @@ func MarhsalledHelper(t *testing.T, err error, input, expectOut any) { } } -func Test_YamlRetrieveUnmarshalled(t *testing.T) { - ttests := map[string]struct { - input []byte - expect testNestedStruct - generator func(t *testing.T) *mockGenerator - }{ - "happy path complex struct complete": { - input: []byte(`foo: AWSSECRETS:///bar/foo -bar: quz -lol: - bla: booo - another: - number: 1235 - float: 123.09`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{ - Bla: "booo", - Another: testAnotherNEst{ - Number: 1235, - Float: 123.09, - }, - }, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - "complex struct - missing fields": { - input: []byte(`foo: AWSSECRETS:///bar/foo -bar: quz`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{}, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(tt.generator(t)) - output := &testNestedStruct{} - err := c.RetrieveUnmarshalledFromYaml(tt.input, output) - MarhsalledHelper(t, err, output, &tt.expect) - }) - } -} - -func Test_JsonRetrieveUnmarshalled(t *testing.T) { - tests := map[string]struct { - input []byte - expect testNestedStruct - generator func(t *testing.T) *mockGenerator - }{ - "happy path complex struct complete": { - input: []byte(`{"foo":"AWSSECRETS:///bar/foo","bar":"quz","lol":{"bla":"booo","another":{"number":1235,"float":123.09}}}`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{ - Bla: "booo", - Another: testAnotherNEst{ - Number: 1235, - Float: 123.09, - }, - }, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - "complex struct - missing fields": { - input: []byte(`{"foo":"AWSSECRETS:///bar/foo","bar":"quz"}`), - expect: testNestedStruct{ - Foo: "baz", - Bar: "quz", - Lol: testLol{}, - }, - generator: func(t *testing.T) *mockGenerator { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - pm := make(generator.ParsedMap) - pm[testTokenAWS] = "baz" - return pm, nil - } - return m - }, - }, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(tt.generator(t)) - output := &testNestedStruct{} - err := c.RetrieveUnmarshalledFromJson(tt.input, output) - MarhsalledHelper(t, err, output, &tt.expect) - }) - } -} - -func TestFindTokens(t *testing.T) { - ttests := map[string]struct { - input string - expect []string - }{ - "extract from text correctly": { - `Where does it come from? - Contrary to popular belief, - Lorem Ipsum is AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj <= in middle of sentencenot simply random text. - It has roots in a piece of classical Latin literature from 45 - BC, making it over 2000 years old. Richard McClintock, a Latin professor at - Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, c - onsectetur, from a Lorem Ipsum passage , at the end of line => AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - and going through the cites of the word in c - lassical literature, discovered the undoubtable source. Lorem Ipsum comes from secti - ons in singles =>'AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj'1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) - in doubles => "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj" - by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular - during the :=> embedded in text RenaissanceAWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[] embedded in text <=: - The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.`, - []string{ - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[]"}, - }, - "unknown implementation not picked up": { - `foo: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - bar: AWSPARAMSTR#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123] - unknown: GCPPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, - []string{ - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR#bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123]"}, - }, - "all implementations": { - `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - secretsmgr: AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123] - gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[] - som othere strufsd - azkv: AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, - []string{ - "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version:123]", - "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[]"}, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - got := configmanager.FindTokens(tt.input) - sort.Strings(got) - sort.Strings(tt.expect) - - if !reflect.DeepEqual(got, tt.expect) { - t.Errorf("input=(%q)\n\ngot: %v\n\nwant: %v", tt.input, got, tt.expect) - } - }) - } -} - -func Test_ConfigManager_DiscoverTokens(t *testing.T) { - ttests := map[string]struct { - input string - separator string - expect []string - }{ - "multiple tokens in single string": { - `Lorem_Ipsum: AWSPARAMSTR:///path/config|foo.user:AWSPARAMSTR:///path/config|password@AWSPARAMSTR:///path/config|foo.endpoint:AWSPARAMSTR:///path/config|foo.port/?someQ=AWSPARAMSTR:///path/queryparam|p1[version=123]&anotherQ=false`, - "://", - []string{ - "AWSPARAMSTR:///path/config|foo.user", - "AWSPARAMSTR:///path/config|password", - "AWSPARAMSTR:///path/config|foo.endpoint", - "AWSPARAMSTR:///path/config|foo.port", - "AWSPARAMSTR:///path/queryparam|p1[version=123]"}, - }, - "# tokens in single string": { - `Lorem_Ipsum: AWSPARAMSTR#/path/config|foo.user:AWSPARAMSTR#/path/config|password@AWSPARAMSTR#/path/config|foo.endpoint:AWSPARAMSTR#/path/config|foo.port/?someQ=AWSPARAMSTR#/path/queryparam|p1[version=123]&anotherQ=false`, - "#", - []string{ - "AWSPARAMSTR#/path/config|foo.user", - "AWSPARAMSTR#/path/config|password", - "AWSPARAMSTR#/path/config|foo.endpoint", - "AWSPARAMSTR#/path/config|foo.port", - "AWSPARAMSTR#/path/queryparam|p1[version=123]"}, - }, - "without leading slash and path like name # tokens in single string": { - `Lorem_Ipsum: AWSPARAMSTR#path_config|foo.user:AWSPARAMSTR#path_config|password@AWSPARAMSTR#path_config|foo.endpoint:AWSPARAMSTR#path_config|foo.port/?someQ=AWSPARAMSTR#path_queryparam|p1[version=123]&anotherQ=false`, - "#", - []string{ - "AWSPARAMSTR#path_config|foo.user", - "AWSPARAMSTR#path_config|password", - "AWSPARAMSTR#path_config|foo.endpoint", - "AWSPARAMSTR#path_config|foo.port", - "AWSPARAMSTR#path_queryparam|p1[version=123]"}, - }, - // Ensures all previous test cases pass as well - "extract from text correctly": { - `Where does it come from? - Contrary to popular belief, - Lorem Ipsum is AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1 <= in middle of sentencenot simply random text. - It has roots in a piece of classical Latin literature from 45 - BC, making it over 2000 years old. Richard McClintock, a Latin professor at - Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, c - onsectetur, from a Lorem Ipsum passage , at the end of line => AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4 - and going through the cites of the word in c - lassical literature, discovered the undoubtable source. Lorem Ipsum comes from secti - ons in singles =>'AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2'1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) - in doubles => "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3" - by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular - during the :=> embedded in text RenaissanceAWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5 embedded in text <=: - The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.`, - "://", - []string{ - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5", - }, - }, - "unknown implementation not picked up": { - `foo: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - bar: AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] - unknown: GCPPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - unknown: GCPSECRETS#/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, - "://", - []string{ - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]"}, - }, - "all implementations": { - `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - secretsmgr: AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] - gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj - som othere strufsd - azkv: AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, - "://", - []string{ - "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]", - "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", - "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj"}, - }, - } - for name, tt := range ttests { - t.Run(name, func(t *testing.T) { - config.VarPrefix = map[config.ImplementationPrefix]bool{"AWSPARAMSTR": true} - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator(tt.separator) - got, err := c.DiscoverTokens(tt.input) - if err != nil { - t.Fatal(err) - } - - if len(got) != len(tt.expect) { - t.Errorf("wrong nmber of tokens resolved\ngot (%d) want (%d)", len(got), len(tt.expect)) - } - for _, v := range got { - if !slices.Contains(tt.expect, v.String()) { - t.Errorf("got (%s) not found in expected slice (%v)", v, tt.expect) - } - } - }) - } -} - -func Test_YamlRetrieveMarshalled_errored_in_generator(t *testing.T) { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return nil, fmt.Errorf("failed to generate a parsedMap") - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - input := &testNestedStruct{} - err := c.RetrieveMarshalledYaml(input) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - -func Test_YamlRetrieveMarshalled_errored_in_marshal(t *testing.T) { - t.Skip() - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return generator.ParsedMap{}, nil - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - err := c.RetrieveMarshalledYaml(&struct { - A int - B map[string]int `yaml:",inline"` - }{1, map[string]int{"a": 2}}) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - -func Test_JsonRetrieveMarshalled_errored_in_generator(t *testing.T) { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return nil, fmt.Errorf("failed to generate a parsedMap") - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - input := &testNestedStruct{} - err := c.RetrieveMarshalledJson(input) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - -func Test_JsonRetrieveMarshalled_errored_in_marshal(t *testing.T) { - m := &mockGenerator{} - m.generate = func(tokens []string) (generator.ParsedMap, error) { - return generator.ParsedMap{}, nil - } - c := configmanager.New(context.TODO()) - c.Config.WithTokenSeparator("://") - c.WithGenerator(m) - // input := &testNestedStruct{} - err := c.RetrieveMarshalledJson(nil) - if err != nil { - } else { - t.Errorf(testutils.TestPhrase, nil, "err") - } -} - // config tests func Test_Generator_Config_(t *testing.T) { ttests := map[string]struct { diff --git a/eirctl.yaml b/eirctl.yaml index 15d1a96..f9e4f9c 100644 --- a/eirctl.yaml +++ b/eirctl.yaml @@ -12,17 +12,20 @@ contexts: name: mirror.gcr.io/bash:5.0.18-alpine3.22 pipelines: - gha:unit:test: + unit:test: - pipeline: test:unit env: ROOT_PKG_NAME: github.com/DevLabFoundry + + gha:unit:test: + - pipeline: unit:test - task: sonar:coverage:prep - depends_on: test:unit + depends_on: unit:test show_coverage: - - pipeline: test:unit + - pipeline: unit:test - task: show:coverage - depends_on: test:unit + depends_on: unit:test build:bin: - task: clean diff --git a/examples/examples.go b/examples/examples.go index b21ed36..379a84d 100644 --- a/examples/examples.go +++ b/examples/examples.go @@ -46,7 +46,7 @@ spec: secret_val: AWSSECRETS#/customfoo/secret-val owner: test_10016@example.com ` - pm, err := cm.RetrieveWithInputReplaced(exampleK8sCrdMarshalled) + pm, err := cm.RetrieveReplacedString(exampleK8sCrdMarshalled) if err != nil { panic(err) @@ -69,7 +69,7 @@ func SpecConfigTokenReplace[T any](inputType T) (*T, error) { // use custom token separator cm.Config.WithTokenSeparator("://") - replaced, err := cm.RetrieveWithInputReplaced(string(rawBytes)) + replaced, err := cm.RetrieveReplacedBytes(rawBytes) if err != nil { return nil, err } @@ -79,57 +79,29 @@ func SpecConfigTokenReplace[T any](inputType T) (*T, error) { return outType, nil } -// Example -func exampleRetrieveYamlUnmarshalled() { - - type config struct { - DbHost string `yaml:"dbhost"` - Username string `yaml:"user"` - Password string `yaml:"pass"` - } - configMarshalled := ` -user: AWSPARAMSTR:///int-test/pocketbase/config|user -pass: AWSPARAMSTR:///int-test/pocketbase/config|pwd -dbhost: AWSPARAMSTR:///int-test/pocketbase/config|host -` - - appConf := &config{} - cm := configmanager.New(context.TODO()) - // use custom token separator inline with future releases - cm.Config.WithTokenSeparator("://") - err := cm.RetrieveUnmarshalledFromYaml([]byte(configMarshalled), appConf) - if err != nil { - panic(err) - } - fmt.Println(appConf.DbHost) - fmt.Println(appConf.Username) - fmt.Println(appConf.Password) -} - -// ### exampleRetrieveYamlMarshalled -func exampleRetrieveYamlMarshalled() { - type config struct { - DbHost string `yaml:"dbhost"` - Username string `yaml:"user"` - Password string `yaml:"pass"` - } - - appConf := &config{ - DbHost: "AWSPARAMSTR:///int-test/pocketbase/config|host", - Username: "AWSPARAMSTR:///int-test/pocketbase/config|user", - Password: "AWSPARAMSTR:///int-test/pocketbase/config|pwd", - } - - cm := configmanager.New(context.TODO()) - cm.Config.WithTokenSeparator("://") - err := cm.RetrieveMarshalledYaml(appConf) - if err != nil { - panic(err) - } - if appConf.DbHost == "AWSPARAMSTR:///int-test/pocketbase/config|host" { - panic(fmt.Errorf("value of DbHost should have been replaced with a value from token")) - } - fmt.Println(appConf.DbHost) - fmt.Println(appConf.Username) - fmt.Println(appConf.Password) -} +// // Example +// func exampleRetrieveYamlUnmarshalled() { + +// type config struct { +// DbHost string `yaml:"dbhost"` +// Username string `yaml:"user"` +// Password string `yaml:"pass"` +// } +// configMarshalled := ` +// user: AWSPARAMSTR:///int-test/pocketbase/config|user +// pass: AWSPARAMSTR:///int-test/pocketbase/config|pwd +// dbhost: AWSPARAMSTR:///int-test/pocketbase/config|host +// ` + +// appConf := &config{} +// cm := configmanager.New(context.TODO()) +// // use custom token separator inline with future releases +// cm.Config.WithTokenSeparator("://") +// err := cm.RetrieveUnmarshalledFromYaml([]byte(configMarshalled), appConf) +// if err != nil { +// panic(err) +// } +// fmt.Println(appConf.DbHost) +// fmt.Println(appConf.Username) +// fmt.Println(appConf.Password) +// } diff --git a/generator/generator.go b/generator/generator.go index 0da95e1..237c6a6 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -3,10 +3,11 @@ package generator import ( "context" "encoding/json" + "errors" "fmt" "io" "os" - "strconv" + "strings" "sync" "github.com/DevLabFoundry/configmanager/v3/internal/config" @@ -14,7 +15,6 @@ import ( "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/DevLabFoundry/configmanager/v3/internal/parser" "github.com/DevLabFoundry/configmanager/v3/internal/strategy" - "github.com/spyzhov/ajson" ) // GenVars is the main struct holding the @@ -28,10 +28,6 @@ type GenVars struct { strategy strategy.StrategyFuncMap ctx context.Context config config.GenVarsConfig - // rawMap is the internal object that holds the values - // of original token => retrieved value - decrypted in plain text - // with a mutex RW locker - rawMap tokenMapSafe //ParsedMap } type Opts func(*GenVars) @@ -45,15 +41,10 @@ func NewGenerator(ctx context.Context, opts ...Opts) *GenVars { } func newGenVars(ctx context.Context, opts ...Opts) *GenVars { - m := make(ParsedMap) conf := config.NewConfig() g := &GenVars{ Logger: log.New(io.Discard), - rawMap: tokenMapSafe{ - tokenMap: m, - mu: &sync.Mutex{}, - }, - ctx: ctx, + ctx: ctx, // return using default config config: *conf, } @@ -99,126 +90,169 @@ func (c *GenVars) Config() *config.GenVarsConfig { // the standard pattern of a token should follow a path like string // // Called only from a slice of tokens -func (c *GenVars) Generate(tokens []string) (ParsedMap, error) { +func (c *GenVars) Generate(tokens []string) (ReplacedToken, error) { - rtm := NewRawTokenConfig() - for _, token := range tokens { - lexerSource := lexer.Source{FileName: token, FullPath: "", Input: token} - l := lexer.New(lexerSource, c.config) - p := parser.New(l, &c.config).WithLogger(log.New(os.Stderr)) - parsed, errs := p.Parse() - if len(errs) > 0 { - c.Logger.Info(fmt.Sprintf("%v", errs)) - continue - } - for _, prsdToken := range parsed { - rtm.AddToken(token, &prsdToken.ParsedToken) - } + ntm, err := c.DiscoverTokens(strings.Join(tokens, "\n")) + if err != nil { + return nil, err } + // pass in default initialised retrieveStrategy // input should be - if err := c.generate(rtm); err != nil { + rt, err := c.generate(ntm) + if err != nil { return nil, err } - return c.rawMap.getTokenMap(), nil + return rt, nil +} + +var ErrTokenDiscovery = errors.New("failed to discover tokens") + +// DiscoverToken generates a k/v map of the tokens with their corresponding secret/paramstore values +// the standard pattern of a token should follow a path like string +// +// Called only from a slice of tokens +func (c *GenVars) DiscoverTokens(text string) (NormalizedTokenSafe, error) { + + rtm := NewRawTokenConfig() + + lexerSource := lexer.Source{FileName: text[0:min(len(text), 20)], FullPath: "", Input: text} + l := lexer.New(lexerSource, c.config) + p := parser.New(l, &c.config).WithLogger(log.New(os.Stderr)) + parsed, errs := p.Parse() + if len(errs) > 0 { + return NormalizedTokenSafe{}, fmt.Errorf("%w in input (%s) with errors: %q", ErrTokenDiscovery, text[0:min(len(text), 25)], errs) + } + for _, prsdToken := range parsed { + rtm.AddToken(prsdToken.ParsedToken.String(), &prsdToken.ParsedToken) + } + return c.NormalizeRawToken(rtm), nil } // IsParsed will try to parse the return found string into // map[string]string // If found it will convert that to a map with all keys uppercased // and any characters -func IsParsed(v any, trm ParsedMap) bool { +func IsParsed(v any, trm ReplacedToken) bool { str := fmt.Sprint(v) err := json.Unmarshal([]byte(str), &trm) return err == nil } -// generate checks if any tokens found -// initiates groutines with fixed size channel map -// to capture responses and errors -// generates ParsedMap which includes +// generate initiates waitGroup to handle 1 or more normalized network calls concurrently to the underlying stores // -// TODO: change this slightly -func (c *GenVars) generate(rawMap *RawTokenConfig) error { - rtm := rawMap.RawTokenMap() - if len(rtm) < 1 { +// Captures the response/error in TokenResponse struct +// It then denormalizes the NormalizedTokenSafe back to a ReplacedToken map +// which stores the values for each token to be returned to the caller +func (c *GenVars) generate(ntm NormalizedTokenSafe) (ReplacedToken, error) { + if len(ntm.normalizedTokenMap) < 1 { c.Logger.Debug("no replaceable tokens found in input") - return nil + return nil, nil } - tokenCount := len(rtm) - outCh := make(chan *strategy.TokenResponse, tokenCount) - - // TODO: initialise the singleton serviceContainer - // pass into each goroutine - for _, parsedToken := range rtm { - token := parsedToken // safe closure capture - // take value from config allocation on a per iteration basis - go func() { - s := strategy.New(c.config, c.Logger, strategy.WithStrategyFuncMap(c.strategy)) - storeStrategy, err := s.SelectImplementation(c.ctx, token) + wg := &sync.WaitGroup{} + + s := strategy.New(c.config, c.Logger, strategy.WithStrategyFuncMap(c.strategy)) + + // safe read of normalized token map + // this will ensure that we are minimizing + // the number of network calls to each underlying store + for _, prsdTkn := range ntm.GetMap() { + if len(prsdTkn.parsedTokens) == 0 { + // TODO: err type this + return nil, fmt.Errorf("no tokens assigned to parsedTokens slice") + } + token := prsdTkn.parsedTokens[0] + wg.Go(func() { + prsdTkn.resp = &strategy.TokenResponse{} + storeStrategy, err := s.GetImplementation(c.ctx, token) if err != nil { - outCh <- &strategy.TokenResponse{Err: err} + prsdTkn.resp.Err = err return } - outCh <- s.RetrieveByToken(c.ctx, storeStrategy, token) - }() + prsdTkn.resp = strategy.ExchangeToken(storeStrategy, token) + }) } - // Fan-in: receive results with pure select - received := 0 - for received < tokenCount { - select { - case cr := <-outCh: - if cr == nil { - continue // defensive (shouldn't happen) - } - c.Logger.Debug("cro: %+v", cr) - if cr.Err != nil { - c.Logger.Debug("cr.err %v, for token: %s", cr.Err, cr.Key()) - } else { - c.rawMap.addKeyVal(cr.Key(), cr.Value()) - } - received++ - case <-c.ctx.Done(): - c.Logger.Debug("context done: %v", c.ctx.Err()) - return c.ctx.Err() // propagate context error (cancel/timeout) + wg.Wait() + + // now we fan out the normalized value to ReplacedToken map + // this will ensure all found tokens will have a value assigned to them + replacedToken := make(ReplacedToken) + for _, r := range ntm.GetMap() { + if r == nil { + // defensive as this shouldn't happen + continue + } + if r.resp.Err != nil { + c.Logger.Debug("cr.err %v, for token: %s", r.resp.Err, r.resp.Key().String()) + continue + } + for _, originalToken := range r.parsedTokens { + replacedToken[originalToken.String()] = keySeparatorLookup(originalToken, r.resp.Value()) } } - return nil -} - -// keySeparatorLookup checks if the key contains -// keySeparator character -// If it does contain one then it tries to parse -func keySeparatorLookup(key *config.ParsedTokenConfig, val string) string { - // key has separator - k := key.LookupKeys() - if k == "" { - // c.logger.Info("no keyseparator found") - return val - } + return replacedToken, nil +} - keys, err := ajson.JSONPath([]byte(val), fmt.Sprintf("$..%s", k)) - if err != nil { - // c.logger.Debug("unable to parse as json object %v", err.Error()) - return val - } +// NormalizedToken represents the struct after all the possible tokens +// were merged into the lowest commmon denominator. +// The idea is to minimize the number of networks calls to the underlying `store` Implementations +// +// The merging is based on the implemenentation and sanitized token being the same, +// if the token contains metadata then it must be +// +// # Merging strategy +// +// Same Prefix + Same SanitisedToken && No Metadata +type NormalizedToken struct { + // all the tokens that can be used to do a replacement + parsedTokens []*config.ParsedTokenConfig + // will be assigned post generate + resp *strategy.TokenResponse + // // configToken is the last assigned full config in the loop if multip + // configToken *config.ParsedTokenConfig +} - if len(keys) == 1 { - v := keys[0] - if v.Type() == ajson.String { - str, err := strconv.Unquote(fmt.Sprintf("%v", v)) - if err != nil { - // c.logger.Debug("unable to unquote value: %v returning as is", v) - return fmt.Sprintf("%v", v) +func (n *NormalizedToken) WithParsedToken(v *config.ParsedTokenConfig) *NormalizedToken { + n.parsedTokens = append(n.parsedTokens, v) + return n +} + +// NormalizedTokenSafe is the map of lowest common denominators +// by token.Keypathless or token.String (full token) if metadata is included +type NormalizedTokenSafe struct { + mu *sync.Mutex + normalizedTokenMap map[string]*NormalizedToken +} + +func (n NormalizedTokenSafe) GetMap() map[string]*NormalizedToken { + n.mu.Lock() + defer n.mu.Unlock() + return n.normalizedTokenMap +} + +func (c *GenVars) NormalizeRawToken(rtm *RawTokenConfig) NormalizedTokenSafe { + ntm := NormalizedTokenSafe{mu: &sync.Mutex{}, normalizedTokenMap: make(map[string]*NormalizedToken)} + + for _, r := range rtm.RawTokenMap() { + // if a string contains we need to store it uniquely + // future improvements might group all the metadata values together + if len(r.Metadata()) > 0 { + if n, found := ntm.normalizedTokenMap[r.String()]; found { + n.WithParsedToken(r) + continue } - return str + ntm.normalizedTokenMap[r.String()] = (&NormalizedToken{}).WithParsedToken(r) + continue } - return fmt.Sprintf("%v", v) + if n, found := ntm.normalizedTokenMap[r.Keypathless()]; found { + n.WithParsedToken(r) + continue + } + ntm.normalizedTokenMap[r.Keypathless()] = (&NormalizedToken{}).WithParsedToken(r) + continue } - - // c.logger.Info("no value found in json using path expression") - return "" + return ntm } diff --git a/generator/generator_test.go b/generator/generator_test.go index 9d5f66c..e20b2f4 100644 --- a/generator/generator_test.go +++ b/generator/generator_test.go @@ -4,14 +4,15 @@ import ( "bytes" "context" "fmt" + "slices" "testing" + "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/DevLabFoundry/configmanager/v3/internal/config" "github.com/DevLabFoundry/configmanager/v3/internal/log" "github.com/DevLabFoundry/configmanager/v3/internal/store" "github.com/DevLabFoundry/configmanager/v3/internal/strategy" "github.com/DevLabFoundry/configmanager/v3/internal/testutils" - "github.com/DevLabFoundry/configmanager/v3/generator" ) type mockGenerate struct { @@ -21,7 +22,7 @@ type mockGenerate struct { func (m *mockGenerate) SetToken(s *config.ParsedTokenConfig) { } -func (m *mockGenerate) Token() (s string, e error) { +func (m *mockGenerate) Value() (s string, e error) { return m.value, m.err } @@ -162,7 +163,7 @@ func Test_IsParsed(t *testing.T) { } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - typ := generator.ParsedMap{} + typ := generator.ReplacedToken{} got := generator.IsParsed(tt.val, typ) if got != tt.isParsed { t.Errorf(testutils.TestPhraseWithContext, "unexpected IsParsed", got, tt.isParsed) @@ -171,378 +172,199 @@ func Test_IsParsed(t *testing.T) { } } -// import ( -// "context" -// "fmt" -// "strings" -// "testing" - -// "github.com/DevLabFoundry/configmanager/v3/internal/testutils" -// ) - -// var ( -// customts = "___" -// customop = "/foo" -// standardop = "./app.env" -// standardts = "#" -// ) - -// type fixture struct { -// t *testing.T -// c *GenVars -// rs *retrieveStrategy -// } - -// func newFixture(t *testing.T) *fixture { -// f := &fixture{} -// f.t = t -// return f -// } - -// func (f *fixture) configGenVars(op, ts string) { -// conf := NewConfig().WithOutputPath(op).WithTokenSeparator(ts) -// gv := NewGenerator().WithConfig(conf) -// f.rs = newRetrieveStrategy(NewDefatultStrategy(), *conf) -// f.c = gv -// } - -// func TestGenVarsWithConfig(t *testing.T) { - -// f := newFixture(t) - -// f.configGenVars(customop, customts) -// if f.c.config.outpath != customop { -// f.t.Errorf(testutils.TestPhrase, f.c.config.outpath, customop) -// } -// if f.c.config.tokenSeparator != customts { -// f.t.Errorf(testutils.TestPhrase, f.c.config.tokenSeparator, customts) -// } -// } - -// func TestStripPrefixNormal(t *testing.T) { -// ttests := map[string]struct { -// prefix ImplementationPrefix -// token string -// keySeparator string -// tokenSeparator string -// f *fixture -// expect string -// }{ -// "standard azkv": {AzKeyVaultSecretsPrefix, "AZKVSECRET://vault1/secret2", "|", "://", newFixture(t), "vault1/secret2"}, -// "standard hashivault": {HashicorpVaultPrefix, "VAULT://vault1/secret2", "|", "://", newFixture(t), "vault1/secret2"}, -// "custom separator hashivault": {HashicorpVaultPrefix, "VAULT#vault1/secret2", "|", "#", newFixture(t), "vault1/secret2"}, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// tt.f.configGenVars(tt.keySeparator, tt.tokenSeparator) -// got := tt.f.rs.stripPrefix(tt.token, tt.prefix) -// if got != tt.expect { -// t.Errorf(testutils.TestPhrase, got, tt.expect) -// } -// }) -// } -// } - -// func Test_stripPrefix(t *testing.T) { -// f := newFixture(t) -// f.configGenVars(standardop, standardts) -// tests := []struct { -// name string -// token string -// prefix ImplementationPrefix -// expect string -// }{ -// { -// name: "simple", -// token: fmt.Sprintf("%s#/test/123", SecretMgrPrefix), -// prefix: SecretMgrPrefix, -// expect: "/test/123", -// }, -// { -// name: "key appended", -// token: fmt.Sprintf("%s#/test/123|key", ParamStorePrefix), -// prefix: ParamStorePrefix, -// expect: "/test/123", -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := f.rs.stripPrefix(tt.token, tt.prefix) -// if tt.expect != got { -// t.Errorf(testutils.TestPhrase, tt.expect, got) -// } -// }) -// } -// } - -// func Test_NormaliseMap(t *testing.T) { -// f := newFixture(t) -// f.configGenVars(standardop, standardts) -// tests := []struct { -// name string -// gv *GenVars -// input map[string]any -// expected string -// }{ -// { -// name: "foo->FOO", -// gv: f.c, -// input: map[string]any{"foo": "bar"}, -// expected: "FOO", -// }, -// { -// name: "num->NUM", -// gv: f.c, -// input: map[string]any{"num": 123}, -// expected: "NUM", -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := f.c.envVarNormalize(tt.input) -// for k := range got { -// if k != tt.expected { -// t.Errorf(testutils.TestPhrase, tt.expected, k) -// } -// } -// }) -// } -// } - -// func Test_KeyLookup(t *testing.T) { -// f := newFixture(t) -// f.configGenVars(standardop, standardts) +func TestGenVars_NormalizeRawToken(t *testing.T) { -// tests := []struct { -// name string -// gv *GenVars -// val string -// key string -// expect string -// }{ -// { -// name: "lowercase key found in str val", -// gv: f.c, -// key: `something|key`, -// val: `{"key": "11235"}`, -// expect: "11235", -// }, -// { -// name: "lowercase key found in numeric val", -// gv: f.c, -// key: `something|key`, -// val: `{"key": 11235}`, -// expect: "11235", -// }, -// { -// name: "lowercase nested key found in numeric val", -// gv: f.c, -// key: `something|key.test`, -// val: `{"key":{"bar":"foo","test":12345}}`, -// expect: "12345", -// }, -// { -// name: "uppercase key found in val", -// gv: f.c, -// key: `something|KEY`, -// val: `{"KEY": "upposeres"}`, -// expect: "upposeres", -// }, -// { -// name: "uppercase nested key found in val", -// gv: f.c, -// key: `something|KEY.TEST`, -// val: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: "upposeres", -// }, -// { -// name: "no key found in val", -// gv: f.c, -// key: `something`, -// val: `{"key": "notfound"}`, -// expect: `{"key": "notfound"}`, -// }, -// { -// name: "nested key not found", -// gv: f.c, -// key: `something|KEY.KEY`, -// val: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: "", -// }, -// { -// name: "incorrect json", -// gv: f.c, -// key: "something|key", -// val: `"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: `"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// }, -// { -// name: "no key provided", -// gv: f.c, -// key: "something", -// val: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// expect: `{"KEY":{"BAR":"FOO","TEST":"upposeres"}}`, -// }, -// { -// name: "return json object", -// gv: f.c, -// key: "something|key.test", -// val: `{"key":{"bar":"foo","test": {"key": "default"}}}`, -// expect: `{"key": "default"}`, -// }, -// { -// name: "unescapable string", -// gv: f.c, -// key: "something|key.test", -// val: `{"key":{"bar":"foo","test":"\\\"upposeres\\\""}}`, -// expect: `\"upposeres\"`, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := f.c.keySeparatorLookup(tt.key, tt.val) -// if got != tt.expect { -// t.Errorf(testutils.TestPhrase, got, tt.expect) -// } -// }) -// } -// } + t.Run("multiple tokens", func(t *testing.T) { + g := generator.NewGenerator(context.TODO()) -// type mockRetrieve struct //func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) chanResp -// { -// r func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp -// s func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) -// } + input := `GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|a + GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|b + GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|c + AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key1 + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key2 + AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj` + want := []string{"GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj"} + got, err := g.DiscoverTokens(input) + if err != nil { + t.Fatal(err) + } + if len(got.GetMap()) != len(want) { + t.Errorf("got %v wanted %d", len(got.GetMap()), len(want)) + } + for key := range got.GetMap() { + if !slices.Contains(want, key) { + t.Errorf("got %s, wanted to be included in %v", key, want) + } + } + }) +} -// func (m mockRetrieve) RetrieveByToken(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return m.r(ctx, impl, prefix, in) -// } -// func (m mockRetrieve) SelectImplementation(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return m.s(ctx, prefix, in, config) -// } +func Test_ConfigManager_DiscoverTokens(t *testing.T) { + ttests := map[string]struct { + input string + separator string + expect []string + }{ + "multiple tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR:///path/config|foo.user:AWSPARAMSTR:///path/config|password@AWSPARAMSTR:///path/config|foo.endpoint:AWSPARAMSTR:///path/config|foo.port/?someQ=AWSPARAMSTR:///path/queryparam|p1[version=123]&anotherQ=false`, + "://", + []string{ + "AWSPARAMSTR:///path/config", + // "AWSPARAMSTR:///path/config|password", + // "AWSPARAMSTR:///path/config|foo.endpoint", + // "AWSPARAMSTR:///path/config|foo.port", + "AWSPARAMSTR:///path/queryparam|p1[version=123]"}, + }, + "# tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR#/path/config|foo.user:AWSPARAMSTR#/path/config|password@AWSPARAMSTR#/path/config|foo.endpoint:AWSPARAMSTR#/path/config|foo.port/?someQ=AWSPARAMSTR#/path/queryparam|p1[version=123]&anotherQ=false`, + "#", + []string{ + "AWSPARAMSTR#/path/config", + // "AWSPARAMSTR#/path/config|password", + // "AWSPARAMSTR#/path/config|foo.endpoint", + // "AWSPARAMSTR#/path/config|foo.port", + "AWSPARAMSTR#/path/queryparam|p1[version=123]"}, + }, + "without leading slash and path like name # tokens in single string": { + `Lorem_Ipsum: AWSPARAMSTR#path_config|foo.user:AWSPARAMSTR#path_config|password@AWSPARAMSTR#path_config|foo.endpoint:AWSPARAMSTR#path_config|foo.port/?someQ=AWSPARAMSTR#path_queryparam|p1[version=123]&anotherQ=false`, + "#", + []string{ + "AWSPARAMSTR#path_config", + // "AWSPARAMSTR#path_config|password", + // "AWSPARAMSTR#path_config|foo.endpoint", + // "AWSPARAMSTR#path_config|foo.port", + "AWSPARAMSTR#path_queryparam|p1[version=123]"}, + }, + // Ensures all previous test cases pass as well + "extract from text correctly": { + `Where does it come from? + Contrary to popular belief, + Lorem Ipsum is AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1 <= in middle of sentencenot simply random text. + It has roots in a piece of classical Latin literature from 45 + BC, making it over 2000 years old. Richard McClintock, a Latin professor at + Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, c + onsectetur, from a Lorem Ipsum passage , at the end of line => AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4 + and going through the cites of the word in c + lassical literature, discovered the undoubtable source. Lorem Ipsum comes from secti + ons in singles =>'AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2'1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) + in doubles => "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3" + by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular + during the :=> embedded in text RenaissanceAWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5 embedded in text <=: + The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32.`, + "://", + []string{ + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl1", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl2", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl3", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl4", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsfl5", + }, + }, + "unknown implementation not picked up": { + `foo: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + bar: AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + unknown: GCPPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + unknown: GCPSECRETS#/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, + "://", + []string{ + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]"}, + }, + "all implementations": { + `param: AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + secretsmgr: AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123] + gcp: GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + vault: VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + som othere strufsd + azkv: AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj`, + "://", + []string{ + "GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]", + "AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", + "VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj"}, + }, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + config.VarPrefix = map[config.ImplementationPrefix]bool{"AWSPARAMSTR": true} + g := generator.NewGenerator(context.TODO()) + g.Config().WithTokenSeparator(tt.separator) + gdt, err := g.DiscoverTokens(tt.input) + if err != nil { + t.Fatal(err) + } + got := gdt.GetMap() -// type mockImpl struct { -// token, value string -// err error -// } + if len(got) != len(tt.expect) { + t.Errorf("wrong nmber of tokens resolved\ngot (%d) want (%d)", len(got), len(tt.expect)) + } + // for _, v := range got { + // if !slices.Contains(tt.expect, v.String()) { + // t.Errorf("got (%s) not found in expected slice (%v)", v, tt.expect) + // } + // } + }) + } +} -// func (m *mockImpl) tokenVal(rs *retrieveStrategy) (s string, e error) { -// return m.value, m.err -// } -// func (m *mockImpl) setTokenVal(s string) { -// m.token = s -// } +func Test_Generate_EnsureRaceFree(t *testing.T) { + g := generator.NewGenerator(context.TODO()) + + input := ` +fg +dfg gdfgfdGCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj +GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|a +GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|b +GCPSECRETS:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|c +ddsffds AWSPARAMSTR:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj + 'AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj[version=123]' + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key1 + AWSSECRETS://bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj|key2 + AZKVSECRET:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj gdf gdfgdf + dfg gdf gdf gdf + fdg dgf dgf + VAULT:///djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj . dfg dfgdf dfg fddf` + + g.WithStrategyMap(strategy.StrategyFuncMap{ + config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"a":"bar","b":{"key2":"val"},"c":123}`, nil} + return m, nil + }, + config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"a":"bar","b":{"key2":"val"},"c":123}`, nil} + return m, nil + }, + config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"bar/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"key1":"bar","key2":"val","c":123}`, nil} + return m, nil + }, + config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"key1":"bar","key2":"val","c":123}`, nil} + return m, nil + }, + config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + m := &mockGenerate{"/djsfsdkjvfjkhfdvibdfinjdsfnjvdsflj", `{"key1":"bar","key2":"val","c":123}`, nil} + return m, nil + }, + }) -// func Test_generate_rawmap_of_tokens_mapped_to_values(t *testing.T) { -// ttests := map[string]struct { -// rawMap func(t *testing.T) map[string]string -// rs func(t *testing.T) retrieveIface -// expectMap func() map[string]string -// }{ -// "success": { -// func(t *testing.T) map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// func(t *testing.T) retrieveIface { -// return mockRetrieve{ -// r: func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return ChanResp{ -// err: nil, -// value: "bar", -// } -// }, -// s: func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return &mockImpl{"foo", "bar", nil}, nil -// }} -// }, -// func() map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// }, -// // as the method swallows errors at the moment this is not very useful -// "error in implementation": { -// func(t *testing.T) map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// func(t *testing.T) retrieveIface { -// return mockRetrieve{ -// r: func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return ChanResp{ -// err: fmt.Errorf("unable to retrieve"), -// } -// }, -// s: func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return &mockImpl{"foo", "bar", nil}, nil -// }} -// }, -// func() map[string]string { -// rm := make(map[string]string) -// return rm -// }, -// }, -// "error in imp selection": { -// func(t *testing.T) map[string]string { -// rm := make(map[string]string) -// rm["foo"] = "bar" -// return rm -// }, -// func(t *testing.T) retrieveIface { -// return mockRetrieve{ -// r: func(ctx context.Context, impl genVarsStrategy, prefix ImplementationPrefix, in string) ChanResp { -// return ChanResp{ -// err: fmt.Errorf("unable to retrieve"), -// } -// }, -// s: func(ctx context.Context, prefix ImplementationPrefix, in string, config GenVarsConfig) (genVarsStrategy, error) { -// return nil, fmt.Errorf("implementation not found for input string: %s", in) -// }} -// }, -// func() map[string]string { -// rm := make(map[string]string) -// return rm -// }, -// }, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// generator := newGenVars() -// generator.generate(tt.rawMap(t), tt.rs(t)) -// got := generator.RawMap() -// if len(got) != len(tt.expectMap()) { -// t.Errorf(testutils.TestPhraseWithContext, "generated raw map did not match", len(got), len(tt.expectMap())) -// } -// }) -// } -// } + got, err := g.Generate([]string{input}) + if err != nil { + t.Fatal(err) + } + if len(got) != 10 { + t.Errorf("got %v wanted %d", len(got), 10) + } -// func TestGenerate(t *testing.T) { -// ttests := map[string]struct { -// tokens func(t *testing.T) []string -// expectLength int -// }{ -// "success without correct prefix": { -// func(t *testing.T) []string { -// return []string{"WRONGIMPL://bar-vault/token1", "AZKVNOTSECRET://bar-vault/token1"} -// }, -// 0, -// }, -// } -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// generator := newGenVars() -// pm, err := generator.Generate(tt.tokens(t)) -// if err != nil { -// t.Errorf(testutils.TestPhrase, err.Error(), nil) -// } -// if len(pm) < tt.expectLength { -// t.Errorf(testutils.TestPhrase, len(pm), tt.expectLength) -// } -// }) -// } -// } +} diff --git a/generator/generatorvars.go b/generator/generatorvars.go index ed2af9f..79a56ae 100644 --- a/generator/generatorvars.go +++ b/generator/generatorvars.go @@ -1,22 +1,27 @@ package generator import ( + "fmt" + "strconv" "sync" "github.com/DevLabFoundry/configmanager/v3/internal/config" + "github.com/spyzhov/ajson" ) -// ParsedMap is the internal working object definition and +// ReplacedToken is the internal working object definition and // the return type if results are not flushed to file -type ParsedMap map[string]any +type ReplacedToken map[string]any -func (pm ParsedMap) MapKeys() (keys []string) { +func (pm ReplacedToken) MapKeys() (keys []string) { for k := range pm { keys = append(keys, k) } return } +// RawTokenConfig represents the map of +// discovered tokens via the lexer/parser type RawTokenConfig struct { mu *sync.Mutex tokenMap map[string]*config.ParsedTokenConfig @@ -38,22 +43,52 @@ func (rtm *RawTokenConfig) RawTokenMap() map[string]*config.ParsedTokenConfig { return rtm.tokenMap } -type tokenMapSafe struct { - mu *sync.Mutex - tokenMap ParsedMap -} +// type tokenMapSafe struct { +// mu *sync.Mutex +// tokenMap ReplacedToken +// } -func (tms *tokenMapSafe) getTokenMap() ParsedMap { - tms.mu.Lock() - defer tms.mu.Unlock() - return tms.tokenMap -} +// func (tms *tokenMapSafe) getTokenMap() ReplacedToken { +// tms.mu.Lock() +// defer tms.mu.Unlock() +// return tms.tokenMap +// } + +// func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { +// tms.mu.Lock() +// defer tms.mu.Unlock() +// // NOTE: still use the metadata in the key +// // there could be different versions / labels for the same token and hence different values +// // However the JSONpath look up +// tms.tokenMap[key.String()] = keySeparatorLookup(key, val) +// } + +// keySeparatorLookup checks if the key contains +// keySeparator character +// If it does contain one then it tries to parse +func keySeparatorLookup(token *config.ParsedTokenConfig, val string) string { + k := token.LookupKeys() + if k == "" { + return val + } + + keys, err := ajson.JSONPath([]byte(val), fmt.Sprintf("$..%s", k)) + if err != nil { + return val + } + + if len(keys) == 1 { + v := keys[0] + if v.Type() == ajson.String { + str, err := strconv.Unquote(fmt.Sprintf("%v", v)) + if err != nil { + return fmt.Sprintf("%v", v) + } + return str + } + + return fmt.Sprintf("%v", v) + } -func (tms *tokenMapSafe) addKeyVal(key *config.ParsedTokenConfig, val string) { - tms.mu.Lock() - defer tms.mu.Unlock() - // NOTE: still use the metadata in the key - // there could be different versions / labels for the same token and hence different values - // However the JSONpath look up - tms.tokenMap[key.String()] = keySeparatorLookup(key, val) + return "" } diff --git a/internal/cmdutils/cmdutils.go b/internal/cmdutils/cmdutils.go index 87e233a..b86b292 100644 --- a/internal/cmdutils/cmdutils.go +++ b/internal/cmdutils/cmdutils.go @@ -11,15 +11,16 @@ import ( "os" "strings" + "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/DevLabFoundry/configmanager/v3/internal/config" "github.com/DevLabFoundry/configmanager/v3/internal/log" - "github.com/DevLabFoundry/configmanager/v3/generator" "github.com/spf13/cobra" ) type configManagerIface interface { - RetrieveWithInputReplaced(input string) (string, error) - Retrieve(tokens []string) (generator.ParsedMap, error) + RetrieveReplacedBytes(input []byte) ([]byte, error) + RetrieveReplacedString(input string) (string, error) + Retrieve(tokens []string) (generator.ReplacedToken, error) GeneratorConfig() *config.GenVarsConfig } @@ -111,13 +112,13 @@ func (c *CmdUtils) generateStrOutFromInput(input io.Reader, writer io.Writer) er return err } - str, err := c.configManager.RetrieveWithInputReplaced(string(b)) + replacedBytes, err := c.configManager.RetrieveReplacedBytes(b) if err != nil { return err } pp := &PostProcessor{} - return pp.StrToFile(writer, str) + return pp.StrToFile(writer, string(replacedBytes)) } type WriterCloserWrapper struct { diff --git a/internal/cmdutils/cmdutils_test.go b/internal/cmdutils/cmdutils_test.go index 29cd6ea..7ec1daf 100644 --- a/internal/cmdutils/cmdutils_test.go +++ b/internal/cmdutils/cmdutils_test.go @@ -17,7 +17,7 @@ import ( ) type mockCfgMgr struct { - parsedMap generator.ParsedMap + parsedMap generator.ReplacedToken err error parsedString string config *config.GenVarsConfig @@ -27,7 +27,13 @@ func (m mockCfgMgr) RetrieveWithInputReplaced(input string) (string, error) { return m.parsedString, m.err } -func (m mockCfgMgr) Retrieve(tokens []string) (generator.ParsedMap, error) { +func (m mockCfgMgr) RetrieveReplacedBytes(input []byte) ([]byte, error) { + return []byte(m.parsedString), m.err +} +func (m mockCfgMgr) RetrieveReplacedString(input string) (string, error) { + return m.parsedString, m.err +} +func (m mockCfgMgr) Retrieve(tokens []string) (generator.ReplacedToken, error) { return m.parsedMap, m.err } @@ -62,12 +68,12 @@ func cmdTestHelper(t *testing.T, err error, got []byte, expect []string) { func Test_GenerateFromCmd(t *testing.T) { ttests := map[string]struct { - mockMap generator.ParsedMap + mockMap generator.ReplacedToken tokens []string expect []string }{ "succeeds with 3 tokens": { - generator.ParsedMap{"FOO://bar/qusx": "aksujg", "FOO://bar/lorem": "", "FOO://bar/ducks": "sdhbjk0293"}, + generator.ReplacedToken{"FOO://bar/qusx": "aksujg", "FOO://bar/lorem": "", "FOO://bar/ducks": "sdhbjk0293"}, []string{"FOO://bar/qusx", "FOO://bar/lorem", "FOO://bar/ducks"}, []string{"export QUSX='aksujg'", "export LOREM=''", "export DUCKS='sdhbjk0293'"}, }, @@ -206,7 +212,7 @@ func Test_CmdUtils_Errors_on(t *testing.T) { t.Run("REtrieve from tokens in fetching ANY of the tokens", func(t *testing.T) { m := &mockCfgMgr{ config: config.NewConfig(), - parsedMap: generator.ParsedMap{}, + parsedMap: generator.ReplacedToken{}, err: fmt.Errorf("err in fetching tokens"), } @@ -221,7 +227,7 @@ func Test_CmdUtils_Errors_on(t *testing.T) { t.Run("REtrieve from tokens in fetching SOME of the tokens", func(t *testing.T) { m := &mockCfgMgr{ config: config.NewConfig(), - parsedMap: generator.ParsedMap{"IMNP://foo": "bar"}, + parsedMap: generator.ReplacedToken{"IMNP://foo": "bar"}, err: fmt.Errorf("err in fetching tokens"), } @@ -235,7 +241,7 @@ func Test_CmdUtils_Errors_on(t *testing.T) { t.Run("REtrieve from string in fetching SOME of the tokens", func(t *testing.T) { m := &mockCfgMgr{ config: config.NewConfig().WithOutputPath("stdout"), - parsedMap: generator.ParsedMap{"IMNP://foo": "bar"}, + parsedMap: generator.ReplacedToken{"IMNP://foo": "bar"}, parsedString: `bar `, err: fmt.Errorf("err in fetching tokens"), } diff --git a/internal/cmdutils/postprocessor.go b/internal/cmdutils/postprocessor.go index 77ef0c2..3b4a33b 100644 --- a/internal/cmdutils/postprocessor.go +++ b/internal/cmdutils/postprocessor.go @@ -13,7 +13,7 @@ import ( // processes the rawMap and outputs the result // depending on cmdline options type PostProcessor struct { - ProcessedMap generator.ParsedMap + ProcessedMap generator.ReplacedToken Config *config.GenVarsConfig outString []string } @@ -24,7 +24,7 @@ func (p *PostProcessor) ConvertToExportVar() []string { for k, v := range p.ProcessedMap { rawKeyToken := strings.Split(k, "/") // assumes a path like token was used topLevelKey := rawKeyToken[len(rawKeyToken)-1] - trm := generator.ParsedMap{} + trm := generator.ReplacedToken{} if parsedOk := generator.IsParsed(v, trm); parsedOk { // if is a map // try look up on key if separator defined @@ -32,21 +32,21 @@ func (p *PostProcessor) ConvertToExportVar() []string { p.exportVars(normMap) continue } - p.exportVars(generator.ParsedMap{topLevelKey: v}) + p.exportVars(generator.ReplacedToken{topLevelKey: v}) } return p.outString } // envVarNormalize -func (p *PostProcessor) envVarNormalize(pmap generator.ParsedMap) generator.ParsedMap { - normalizedMap := make(generator.ParsedMap) +func (p *PostProcessor) envVarNormalize(pmap generator.ReplacedToken) generator.ReplacedToken { + normalizedMap := make(generator.ReplacedToken) for k, v := range pmap { normalizedMap[p.normalizeKey(k)] = v } return normalizedMap } -func (p *PostProcessor) exportVars(exportMap generator.ParsedMap) { +func (p *PostProcessor) exportVars(exportMap generator.ReplacedToken) { for k, v := range exportMap { // NOTE: \n line ending is not totally cross platform diff --git a/internal/cmdutils/postprocessor_test.go b/internal/cmdutils/postprocessor_test.go index ac1a30c..5c18e23 100644 --- a/internal/cmdutils/postprocessor_test.go +++ b/internal/cmdutils/postprocessor_test.go @@ -17,14 +17,14 @@ func postprocessorHelper(t *testing.T) { } func Test_ConvertToExportVars(t *testing.T) { tests := map[string]struct { - rawMap generator.ParsedMap + rawMap generator.ReplacedToken expectStr string expectLength int }{ - "number included": {generator.ParsedMap{"foo": "BAR", "num": 123}, `export FOO='BAR'`, 2}, - "strings only": {generator.ParsedMap{"foo": "BAR", "num": "a123"}, `export FOO='BAR'`, 2}, - "numbers only": {generator.ParsedMap{"foo": 123, "num": 456}, `export FOO=123`, 2}, - "map inside response": {generator.ParsedMap{"map": `{"foo":"bar","baz":"qux"}`, "num": 123}, `export FOO='bar'`, 3}, + "number included": {generator.ReplacedToken{"foo": "BAR", "num": 123}, `export FOO='BAR'`, 2}, + "strings only": {generator.ReplacedToken{"foo": "BAR", "num": "a123"}, `export FOO='BAR'`, 2}, + "numbers only": {generator.ReplacedToken{"foo": 123, "num": 456}, `export FOO=123`, 2}, + "map inside response": {generator.ReplacedToken{"map": `{"foo":"bar","baz":"qux"}`, "num": 123}, `export FOO='bar'`, 3}, } for name, tt := range tests { diff --git a/internal/config/config.go b/internal/config/config.go index c66ad14..0631da5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -170,20 +170,6 @@ func (ptc *ParsedTokenConfig) WithSanitizedToken(v string) { ptc.sanitizedToken = v } -// depracated -func (ptc *ParsedTokenConfig) new() *ParsedTokenConfig { - // // order must be respected here - // // - // ptc.prefixLessToken = strings.Replace(ptc.fullToken, fmt.Sprintf("%s%s", ptc.prefix, ptc.tokenSeparator), "", 1) - - // // token without metadata and the string itself - // ptc.extractMetadataStr() - // // token without keys - // ptc.keysLookup() - // return ptc - return nil -} - func (t *ParsedTokenConfig) ParseMetadata(metadataTyp any) error { // crude json like builder from key/val tags // since we are only ever dealing with a string input @@ -220,14 +206,25 @@ func (t *ParsedTokenConfig) StoreToken() string { // Full returns the full Token path. // Including key separator and metadata values func (t *ParsedTokenConfig) String() string { + token := t.Metadaless() + if len(t.metadataStr) > 0 { + token += fmt.Sprintf("[%s]", t.metadataStr) + } + return token +} + +// Keypathless returns the token without the key and metadata attributes +// Token will include the ImplementationPrefix + token separator + path to item +func (t *ParsedTokenConfig) Keypathless() string { + token := fmt.Sprintf("%s%s%s", t.prefix, t.tokenSeparator, t.sanitizedToken) + return token +} + +func (t *ParsedTokenConfig) Metadaless() string { token := fmt.Sprintf("%s%s%s", t.prefix, t.tokenSeparator, t.sanitizedToken) if len(t.keysPath) > 0 { token += t.keySeparator + t.keysPath } - - if len(t.metadataStr) > 0 { - token += fmt.Sprintf("[%s]", t.metadataStr) - } return token } diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 1de07e4..2f1486e 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -266,6 +266,12 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon } keyPath += p.curToken.Literal p.nextToken() + if p.peekTokenIs(config.EOF) { + // check if the next token is EOF once advanced + // if it is we want to consume current token else it will be skipped + keyPath += p.curToken.Literal + break + } } configManagerToken.WithKeyPath(keyPath) return nil @@ -283,6 +289,12 @@ func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) err p.nextToken() for !p.peekTokenIs(config.EOF) { if p.peekTokenIsEnd() { + if p.currentTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { + metadata += p.curToken.Literal + found = true + p.nextToken() + break + } return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) } if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { diff --git a/internal/store/azappconf.go b/internal/store/azappconf.go index b12f501..c35f538 100644 --- a/internal/store/azappconf.go +++ b/internal/store/azappconf.go @@ -82,7 +82,7 @@ func (implmt *AzAppConf) SetToken(token *config.ParsedTokenConfig) {} // label can be specified // From this point then normal rules of configmanager apply, // including keySeperator and lookup. -func (imp *AzAppConf) Token() (string, error) { +func (imp *AzAppConf) Value() (string, error) { imp.logger.Info("Concrete implementation AzAppConf") imp.logger.Info("AzAppConf Token: %s", imp.token.String()) diff --git a/internal/store/azappconf_test.go b/internal/store/azappconf_test.go index 922f04e..17da526 100644 --- a/internal/store/azappconf_test.go +++ b/internal/store/azappconf_test.go @@ -132,7 +132,7 @@ func Test_AzAppConf_Success(t *testing.T) { } impl.WithSvc(tt.mockClient(t)) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) @@ -182,7 +182,7 @@ func Test_AzAppConf_Error(t *testing.T) { t.Fatal("failed to init AZAPPCONF") } impl.WithSvc(tt.mockClient(t)) - if _, err := impl.Token(); !errors.Is(err, tt.expect) { + if _, err := impl.Value(); !errors.Is(err, tt.expect) { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) } }) diff --git a/internal/store/azkeyvault.go b/internal/store/azkeyvault.go index 51f1047..781b066 100644 --- a/internal/store/azkeyvault.go +++ b/internal/store/azkeyvault.go @@ -72,7 +72,7 @@ func (s *KvScrtStore) WithSvc(svc kvApi) { // setToken already happens in AzureKVClient in the constructor func (implmt *KvScrtStore) SetToken(token *config.ParsedTokenConfig) {} -func (imp *KvScrtStore) Token() (string, error) { +func (imp *KvScrtStore) Value() (string, error) { imp.logger.Info("Concrete implementation AzKeyVault Secret") imp.logger.Info("AzKeyVault Token: %s", imp.token.String()) diff --git a/internal/store/azkeyvault_test.go b/internal/store/azkeyvault_test.go index 9497096..35b5c7d 100644 --- a/internal/store/azkeyvault_test.go +++ b/internal/store/azkeyvault_test.go @@ -202,7 +202,7 @@ func TestAzKeyVault(t *testing.T) { } impl.WithSvc(tt.mockClient(t)) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/aztablestorage.go b/internal/store/aztablestorage.go index 3326a31..eedef16 100644 --- a/internal/store/aztablestorage.go +++ b/internal/store/aztablestorage.go @@ -83,7 +83,7 @@ func (implmt *AzTableStore) SetToken(token *config.ParsedTokenConfig) {} // // From this point then normal rules of configmanager apply, // including keySeperator and lookup. -func (imp *AzTableStore) Token() (string, error) { +func (imp *AzTableStore) Value() (string, error) { imp.logger.Info("AzTableSTore Token: %s", imp.token.String()) imp.logger.Info("Concrete implementation AzTableSTore") diff --git a/internal/store/aztablestorage_test.go b/internal/store/aztablestorage_test.go index 50a9c62..50273fc 100644 --- a/internal/store/aztablestorage_test.go +++ b/internal/store/aztablestorage_test.go @@ -97,7 +97,7 @@ func Test_AzTableStore_Success(t *testing.T) { impl.WithSvc(tt.mockClient(t)) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) @@ -186,7 +186,7 @@ func Test_azstorage_with_value_property(t *testing.T) { impl.WithSvc(tt.mockClient(t)) - got, err := impl.Token() + got, err := impl.Value() if err != nil { t.Fatalf(testutils.TestPhrase, err.Error(), nil) } @@ -250,7 +250,7 @@ func Test_AzTableStore_Error(t *testing.T) { } impl.WithSvc(tt.mockClient(t)) - if _, err := impl.Token(); !errors.Is(err, tt.expect) { + if _, err := impl.Value(); !errors.Is(err, tt.expect) { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) } }) diff --git a/internal/store/gcpsecrets.go b/internal/store/gcpsecrets.go index 0a78bb9..07c43e2 100644 --- a/internal/store/gcpsecrets.go +++ b/internal/store/gcpsecrets.go @@ -53,7 +53,7 @@ func (imp *GcpSecrets) SetToken(token *config.ParsedTokenConfig) { imp.config = storeConf } -func (imp *GcpSecrets) Token() (string, error) { +func (imp *GcpSecrets) Value() (string, error) { // Close client currently as new one would be created per iteration defer func() { _ = imp.close() diff --git a/internal/store/gcpsecrets_test.go b/internal/store/gcpsecrets_test.go index 471a238..5f859ba 100644 --- a/internal/store/gcpsecrets_test.go +++ b/internal/store/gcpsecrets_test.go @@ -168,7 +168,7 @@ func Test_GetGcpSecretVarHappy(t *testing.T) { impl.WithSvc(tt.mockClient(t)) impl.SetToken(tt.token()) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { diff --git a/internal/store/hashivault.go b/internal/store/hashivault.go index 84008e9..558c9b2 100644 --- a/internal/store/hashivault.go +++ b/internal/store/hashivault.go @@ -111,7 +111,7 @@ func (imp *VaultStore) SetToken(token *config.ParsedTokenConfig) {} // getTokenValue implements the underlying techonology // token retrieval and returns a stringified version // of the secret -func (imp *VaultStore) Token() (string, error) { +func (imp *VaultStore) Value() (string, error) { imp.logger.Info("%s", "Concrete implementation HashiVault") imp.logger.Info("Getting Secret: %s", imp.token) diff --git a/internal/store/hashivault_test.go b/internal/store/hashivault_test.go index 4077ea8..ccb061d 100644 --- a/internal/store/hashivault_test.go +++ b/internal/store/hashivault_test.go @@ -319,7 +319,7 @@ func TestVaultScenarios(t *testing.T) { } impl.WithSvc(tt.mockClient(t)) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/paramstore.go b/internal/store/paramstore.go index 493a49e..aa45ace 100644 --- a/internal/store/paramstore.go +++ b/internal/store/paramstore.go @@ -52,7 +52,7 @@ func (imp *ParamStore) SetToken(token *config.ParsedTokenConfig) { imp.config = storeConf } -func (imp *ParamStore) Token() (string, error) { +func (imp *ParamStore) Value() (string, error) { imp.logger.Info("%s", "Concrete implementation ParameterStore") imp.logger.Info("ParamStore Token: %s", imp.token.String()) diff --git a/internal/store/paramstore_test.go b/internal/store/paramstore_test.go index ec73c59..db71b0f 100644 --- a/internal/store/paramstore_test.go +++ b/internal/store/paramstore_test.go @@ -117,7 +117,7 @@ func Test_GetParamStore(t *testing.T) { } impl.WithSvc(tt.mockClient(t)) impl.SetToken(tt.token()) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/secretsmanager.go b/internal/store/secretsmanager.go index d49c808..6744d8a 100644 --- a/internal/store/secretsmanager.go +++ b/internal/store/secretsmanager.go @@ -55,7 +55,7 @@ func (imp *SecretsMgr) SetToken(token *config.ParsedTokenConfig) { imp.config = storeConf } -func (imp *SecretsMgr) Token() (string, error) { +func (imp *SecretsMgr) Value() (string, error) { imp.logger.Info("Concrete implementation SecretsManager") imp.logger.Debug("SecretsManager Token: %s", imp.token.String()) diff --git a/internal/store/secretsmanager_test.go b/internal/store/secretsmanager_test.go index 39bd475..870bb75 100644 --- a/internal/store/secretsmanager_test.go +++ b/internal/store/secretsmanager_test.go @@ -2,6 +2,7 @@ package store_test import ( "context" + "fmt" "io" "strings" "testing" @@ -59,44 +60,78 @@ func Test_GetSecretMgr(t *testing.T) { }) }, }, - // "success with version": {"AWSSECRETS#/token/1[version=123]", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - // t.Helper() - // awsSecretsMgrGetChecker(t, params) - // return &secretsmanager.GetSecretValueOutput{ - // SecretString: &tsuccessSecret, - // }, nil - // }) - // }, config.NewConfig(), - // }, - // "success with binary": {"AWSSECRETS#/token/1", "|", "#", tsuccessSecret, func(t *testing.T) secretsMgrApi { - // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - // t.Helper() - // awsSecretsMgrGetChecker(t, params) - // return &secretsmanager.GetSecretValueOutput{ - // SecretBinary: []byte(tsuccessSecret), - // }, nil - // }) - // }, config.NewConfig(), - // }, - // "errored": {"AWSSECRETS#/token/1", "|", "#", "unable to retrieve secret", func(t *testing.T) secretsMgrApi { - // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - // t.Helper() - // awsSecretsMgrGetChecker(t, params) - // return nil, fmt.Errorf("unable to retrieve secret") - // }) - // }, config.NewConfig(), - // }, - // "ok but empty": {"AWSSECRETS#/token/1", "|", "#", "", func(t *testing.T) secretsMgrApi { - // return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { - // t.Helper() - // awsSecretsMgrGetChecker(t, params) - // return &secretsmanager.GetSecretValueOutput{ - // SecretString: nil, - // }, nil - // }) - // }, config.NewConfig(), - // }, + "success with version": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version=123") + return tkn + }, + tsuccessSecret, func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretString: &tsuccessSecret, + }, nil + }) + }, + }, + "success with binary": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + tsuccessSecret, func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretBinary: []byte(tsuccessSecret), + }, nil + }) + }, + }, + "errored": { + func() *config.ParsedTokenConfig { + // "AWSSECRETS#/token/1", "|", "#", + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "unable to retrieve secret", func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return nil, fmt.Errorf("unable to retrieve secret") + }) + }, + }, + "ok but empty": { + func() *config.ParsedTokenConfig { + // "AWSSECRETS#/token/1", "|", "#", + tkn, _ := config.NewToken(config.SecretMgrPrefix, *config.NewConfig().WithTokenSeparator("#")) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("version=123") + return tkn + }, + "", func(t *testing.T) mockSecretsApi { + return mockSecretsApi(func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + t.Helper() + awsSecretsMgrGetChecker(t, params) + return &secretsmanager.GetSecretValueOutput{ + SecretString: nil, + }, nil + }) + }, + }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { @@ -104,7 +139,7 @@ func Test_GetSecretMgr(t *testing.T) { impl.WithSvc(tt.mockClient(t)) impl.SetToken(tt.token()) - got, err := impl.Token() + got, err := impl.Value() if err != nil { if err.Error() != tt.expect { t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) diff --git a/internal/store/store.go b/internal/store/store.go index 00e0c4a..eceeb45 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -20,6 +20,8 @@ var ( // // Defined on the package for easier re-use across the program type Strategy interface { - Token() (s string, e error) + // Value retrieves the underlying value for the token + Value() (s string, e error) + // SetToken SetToken(s *config.ParsedTokenConfig) } diff --git a/internal/strategy/strategy.go b/internal/strategy/strategy.go index 92b5c3b..47c47ac 100644 --- a/internal/strategy/strategy.go +++ b/internal/strategy/strategy.go @@ -52,17 +52,21 @@ type strategyFnMap struct { mu sync.Mutex funcMap StrategyFuncMap } + type RetrieveStrategy struct { + // mu *sync.Mutex implementation store.Strategy config config.GenVarsConfig strategyFuncMap strategyFnMap } + type Opts func(*RetrieveStrategy) // New func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *RetrieveStrategy { rs := &RetrieveStrategy{ - config: config, + config: config, + // mu: &sync.Mutex{}, strategyFuncMap: strategyFnMap{mu: sync.Mutex{}, funcMap: defaultStrategyFuncMap(logger)}, } // overwrite or add any options/defaults set above @@ -79,26 +83,55 @@ func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *Retriev // NOTE: this may lead to eventual optional configurations by users func WithStrategyFuncMap(funcMap StrategyFuncMap) Opts { return func(rs *RetrieveStrategy) { + rs.strategyFuncMap.mu.Lock() + defer rs.strategyFuncMap.mu.Unlock() for prefix, implementation := range funcMap { - rs.strategyFuncMap.mu.Lock() - defer rs.strategyFuncMap.mu.Unlock() rs.strategyFuncMap.funcMap[config.ImplementationPrefix(prefix)] = implementation } } } -func (rs *RetrieveStrategy) setImplementation(strategy store.Strategy) { - rs.implementation = strategy -} +// GetImplementation is a factory method returning the concrete implementation for the retrieval of the token value +// i.e. facilitating the exchange of the supplied token for the underlying value +func (rs *RetrieveStrategy) GetImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + if token == nil { + return nil, fmt.Errorf("unable to get prefix, %w", ErrTokenInvalid) + } -func (rs *RetrieveStrategy) setTokenVal(s *config.ParsedTokenConfig) { - rs.implementation.SetToken(s) + if store, found := rs.strategyFuncMap.funcMap[token.Prefix()]; found { + return store(ctx, token) + } + + return nil, fmt.Errorf("implementation not found for input string: %s", token) } -func (rs *RetrieveStrategy) getTokenValue() (string, error) { - return rs.implementation.Token() +func ExchangeToken(s store.Strategy, token *config.ParsedTokenConfig) *TokenResponse { + cr := &TokenResponse{} + cr.Err = nil + cr.key = token + s.SetToken(token) + cr.value, cr.Err = s.Value() + return cr } +// func (rs *RetrieveStrategy) setImplementation(strategy store.Strategy) { +// rs.mu.Lock() +// defer rs.mu.Unlock() +// rs.implementation = strategy +// } + +// func (rs *RetrieveStrategy) setTokenVal(s *config.ParsedTokenConfig) { +// rs.mu.Lock() +// defer rs.mu.Unlock() +// rs.implementation.SetToken(s) +// } + +// func (rs *RetrieveStrategy) getTokenValue() (string, error) { +// rs.mu.Lock() +// defer rs.mu.Unlock() +// return rs.implementation.Token() +// } + type TokenResponse struct { value string key *config.ParsedTokenConfig @@ -112,32 +145,3 @@ func (tr *TokenResponse) Key() *config.ParsedTokenConfig { func (tr *TokenResponse) Value() string { return tr.value } - -// retrieveSpecificCh wraps around the specific strategy implementation -// and publishes results to a channel -func (rs *RetrieveStrategy) RetrieveByToken(ctx context.Context, impl store.Strategy, tokenConf *config.ParsedTokenConfig) *TokenResponse { - cr := &TokenResponse{} - cr.Err = nil - cr.key = tokenConf - rs.setImplementation(impl) - rs.setTokenVal(tokenConf) - s, err := rs.getTokenValue() - if err != nil { - cr.Err = err - return cr - } - cr.value = s - return cr -} - -func (rs *RetrieveStrategy) SelectImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - if token == nil { - return nil, fmt.Errorf("unable to get prefix, %w", ErrTokenInvalid) - } - - if store, found := rs.strategyFuncMap.funcMap[token.Prefix()]; found { - return store(ctx, token) - } - - return nil, fmt.Errorf("implementation not found for input string: %s", token) -} diff --git a/internal/strategy/strategy_test.go b/internal/strategy/strategy_test.go index b09548f..acae5e1 100644 --- a/internal/strategy/strategy_test.go +++ b/internal/strategy/strategy_test.go @@ -23,7 +23,7 @@ type mockGenerate struct { func (m mockGenerate) SetToken(s *config.ParsedTokenConfig) { } -func (m mockGenerate) Token() (s string, e error) { +func (m mockGenerate) Value() (s string, e error) { return m.value, m.err } @@ -69,18 +69,17 @@ func Test_Strategy_Retrieve_succeeds(t *testing.T) { } for name, tt := range ttests { t.Run(name, func(t *testing.T) { - rs := strategy.New(*tt.config, log.New(io.Discard)) token, _ := config.NewToken(tt.impPrefix, *tt.config) token.WithSanitizedToken(tt.token) - got := rs.RetrieveByToken(context.TODO(), tt.impl(t), token) + got := strategy.ExchangeToken(tt.impl(t), token) if got.Err != nil { t.Errorf(testutils.TestPhraseWithContext, "Token response errored", got.Err.Error(), tt.expect) } if got.Value() != tt.expect { t.Errorf(testutils.TestPhraseWithContext, "Value not correct", got.Value(), tt.expect) } - if got.Key().String() != tt.token { - t.Errorf(testutils.TestPhraseWithContext, "INcorrect Token returned in Key", got.Key().String(), tt.token) + if got.Key().StoreToken() != tt.token { + t.Errorf(testutils.TestPhraseWithContext, "Incorrect Token returned in Key", got.Key().StoreToken(), tt.token) } }) } @@ -107,8 +106,8 @@ func Test_CustomStrategyFuncMap_add_own(t *testing.T) { s := strategy.New(*genVarsConf, log.New(io.Discard), strategy.WithStrategyFuncMap(strategy.StrategyFuncMap{config.AzTableStorePrefix: custFunc})) - store, _ := s.SelectImplementation(context.TODO(), token) - _ = s.RetrieveByToken(context.TODO(), store, token) + store, _ := s.GetImplementation(context.TODO(), token) + _ = strategy.ExchangeToken(store, token) if called != 1 { t.Errorf(testutils.TestPhraseWithContext, "custom func not called", called, 1) @@ -273,7 +272,7 @@ func Test_SelectImpl_With(t *testing.T) { rs := strategy.New(*tt.config, log.New(io.Discard)) token, _ := config.NewToken(tt.impPrefix, *tt.config) token.WithSanitizedToken(tt.token) - got, err := rs.SelectImplementation(context.TODO(), token) + got, err := rs.GetImplementation(context.TODO(), token) if err != nil { if err.Error() != tt.expErr.Error() { From 329bf19be92c02a72191b79913d91e04ca2cba7c Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 09:07:23 +0000 Subject: [PATCH 11/19] fix: clean up --- internal/parser/parser.go | 52 +++------- internal/store/aztablestorage_test.go | 138 ++++++++++++++++---------- 2 files changed, 99 insertions(+), 91 deletions(-) diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 2f1486e..7bb3c26 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -4,13 +4,10 @@ import ( "errors" "fmt" "os" - "strings" "github.com/DevLabFoundry/configmanager/v3/internal/config" "github.com/DevLabFoundry/configmanager/v3/internal/lexer" "github.com/DevLabFoundry/configmanager/v3/internal/log" - - "github.com/a8m/envsubst" ) func wrapErr(incompleteToken string, line, position int, etyp error) error { @@ -74,38 +71,23 @@ func (p *Parser) WithLogger(logger log.ILogger) *Parser { // // The parser does not do a second pass and interprets the source from top to bottom func (p *Parser) Parse() ([]ConfigManagerTokenBlock, []error) { - genDocStms := []ConfigManagerTokenBlock{} + stmts := []ConfigManagerTokenBlock{} for !p.currentTokenIs(config.EOF) { if p.currentTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { - // parseGenDocBlocks will advance the token until - // it hits the END_DOC_GEN token + // continues to read the tokens until it hits an end token or errors configManagerToken, err := config.NewToken(p.curToken.ImpPrefix, *p.config) if err != nil { return nil, []error{err} } if stmt := p.buildConfigManagerTokenFromBlocks(configManagerToken); stmt != nil { - genDocStms = append(genDocStms, *stmt) + stmts = append(stmts, *stmt) } } p.nextToken() } - return genDocStms, p.errors -} - -// ExpandEnvVariables expands the env vars inside DocContent -// to their environment var values. -// -// Failing when a variable is either not set or set but empty. -func ExpandEnvVariables(input string, vars []string) (string, error) { - for _, v := range vars { - kv := strings.Split(v, "=") - key, value := kv[0], kv[1] // kv[1] will be an empty string = "" - os.Setenv(key, value) - } - - return envsubst.StringRestrictedNoDigit(input, true, true, false) + return stmts, p.errors } func (p *Parser) nextToken() { @@ -132,10 +114,7 @@ func (p *Parser) peekTokenIsEnd() bool { return endTokens[p.peekToken.Type] } -// buildConfigManagerTokenFromBlocks throws away all other content other -// than what is inside //+gendoc tags -// parses any annotation and creates GenDocBlock -// for later analysis +// buildConfigManagerTokenFromBlocks func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.ParsedTokenConfig) *ConfigManagerTokenBlock { currentToken := p.curToken stmt := &ConfigManagerTokenBlock{BeginToken: currentToken} @@ -147,20 +126,21 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // built as part of the below parser sanitizedToken := "" - // should exit the loop if no end doc tag found + // should exit the loop if no end tag found notFoundEnd := true // stop on end of file for !p.peekTokenIs(config.EOF) { - // This is the target state when there is an optional token wrapping - // {{ IMP://path }} - if p.peekTokenIs(config.END_CONFIGMANAGER_TOKEN) { - notFoundEnd = false - fullToken += p.curToken.Literal - sanitizedToken += p.curToken.Literal - stmt.EndToken = p.curToken - break - } + // // This is the target state when there is an optional token wrapping + // // e.g. `{{ IMP://path }}` + // // currently this is untestable + // if p.peekTokenIs(config.END_CONFIGMANAGER_TOKEN) { + // notFoundEnd = false + // fullToken += p.curToken.Literal + // sanitizedToken += p.curToken.Literal + // stmt.EndToken = p.curToken + // break + // } // when next token is another token // i.e. the tokens are adjacent diff --git a/internal/store/aztablestorage_test.go b/internal/store/aztablestorage_test.go index 50273fc..892ee9e 100644 --- a/internal/store/aztablestorage_test.go +++ b/internal/store/aztablestorage_test.go @@ -126,7 +126,6 @@ func Test_azstorage_with_value_property(t *testing.T) { tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") tkn.WithKeyPath("host") - tkn.WithMetadata("version:123]") return tkn }, "map[bool:true host:foo port:1234]", @@ -138,42 +137,58 @@ func Test_azstorage_with_value_property(t *testing.T) { }) }, }, - // "return value property with string only": { - // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", - // "foo.bar.com", - // func(t *testing.T) tableStoreApi { - // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - // t.Helper() - // resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} - // return resp, nil - // }) - // }, - // conf, - // }, - // "return value property with numeric only": { - // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", - // "1234", - // func(t *testing.T) tableStoreApi { - // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - // t.Helper() - // resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} - // return resp, nil - // }) - // }, - // conf, - // }, - // "return value property with boolean only": { - // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", - // "false", - // func(t *testing.T) tableStoreApi { - // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - // t.Helper() - // resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} - // return resp, nil - // }) - // }, - // conf, - // }, + "return value property with string only": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + // tkn.WithKeyPath("host") + // tkn.WithMetadata("version:123]") + return tkn + }, + "foo.bar.com", + func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":"foo.bar.com"}`)} + return resp, nil + }) + }, + }, + "return value property with numeric only": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + // tkn.WithKeyPath("host") + // tkn.WithMetadata("version:123]") + return tkn + }, + "1234", + func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":1234}`)} + return resp, nil + }) + }, + }, + "return value property with boolean only": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE:///test-account/table/partitionkey/rowKey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *conf) + tkn.WithSanitizedToken("/test-account/table/partitionkey/rowKey") + return tkn + }, + "false", + func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{Value: []byte(`{"value":false}`)} + return resp, nil + }) + }, + }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { @@ -221,25 +236,38 @@ func Test_AzTableStore_Error(t *testing.T) { }) }, }, - // "errored on service method call": {"AZTABLESTORE#/test-account/table/token/ok", ErrRetrieveFailed, func(t *testing.T) tableStoreApi { - // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - // t.Helper() - // resp := aztables.GetEntityResponse{} - // return resp, fmt.Errorf("network error") - // }) - // }, - // config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - // }, + "errored on service method call": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-account/table/token/ok", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-account/table/token/ok") + return tkn + }, + store.ErrRetrieveFailed, + func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, fmt.Errorf("network error") + }) + }, + }, - // "empty": {"AZTABLESTORE#/test-vault/token/1|somekey", ErrIncorrectlyStructuredToken, func(t *testing.T) tableStoreApi { - // return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { - // t.Helper() - // resp := aztables.GetEntityResponse{} - // return resp, nil - // }) - // }, - // config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#"), - // }, + "empty": { + func() *config.ParsedTokenConfig { + // "AZTABLESTORE#/test-vault/token/1|somekey", + tkn, _ := config.NewToken(config.AzKeyVaultSecretsPrefix, *config.NewConfig().WithKeySeparator("|").WithTokenSeparator("#")) + tkn.WithSanitizedToken("/test-vault/token/1|somekey") + return tkn + }, + store.ErrIncorrectlyStructuredToken, func(t *testing.T) mockAzTableStoreApi { + return mockAzTableStoreApi(func(ctx context.Context, partitionKey string, rowKey string, options *aztables.GetEntityOptions) (aztables.GetEntityResponse, error) { + t.Helper() + resp := aztables.GetEntityResponse{} + return resp, nil + }) + }, + }, } for name, tt := range tests { From f224b74323cea3ca7d527c3a96b287135e0c626f Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 16:17:35 +0000 Subject: [PATCH 12/19] fix: change defaults in CLI for token separator --- cmd/configmanager/configmanager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/configmanager/configmanager.go b/cmd/configmanager/configmanager.go index d0f64e9..1eed6cb 100644 --- a/cmd/configmanager/configmanager.go +++ b/cmd/configmanager/configmanager.go @@ -46,8 +46,8 @@ func NewRootCmd(logger log.ILogger) *Root { //channelOut, channelErr io.Writer } rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.verbose, "verbose", "v", false, "Verbosity level") - rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "#", "Separator to use to mark concrete store and the key within it") - rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS#/token/map|key1") + rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.tokenSeparator, "token-separator", "s", "://", "Separator to use to mark concrete store and the key within it") + rc.Cmd.PersistentFlags().StringVarP(&rc.rootFlags.keySeparator, "key-separator", "k", "|", "Separator to use to mark a key look up in a map. e.g. AWSSECRETS:///token/map|key1") rc.Cmd.PersistentFlags().BoolVarP(&rc.rootFlags.enableEnvSubst, "enable-envsubst", "e", false, "Enable envsubst on input. This will fail on any unset or empty variables") addSubCmds(rc) return rc From abcd85c7a7ebe0e88bfe9e57e272f4f56f950e85 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 17:49:47 +0000 Subject: [PATCH 13/19] fix: set version remove UNKNOWN from lexer --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 4 +- internal/lexer/lexer.go | 17 +----- internal/lexer/lexer_test.go | 6 +- internal/store/paramstore_test.go | 98 +++++++++++++++++++------------ 5 files changed, 67 insertions(+), 60 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 946bfc5..0af6870 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: set-version: runs-on: ubuntu-latest container: - image: mcr.microsoft.com/dotnet/sdk:6.0 + image: mcr.microsoft.com/dotnet/sdk:10.0 outputs: semVer: ${{ steps.gitversion.outputs.semVer }} steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2faca47..dd8c725 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.head_branch == 'master' && github.event.workflow_run.conclusion == 'success' }} container: - image: mcr.microsoft.com/dotnet/sdk:6.0 + image: mcr.microsoft.com/dotnet/sdk:10.0 outputs: semVer: ${{ steps.gitversion.outputs.semVer }} steps: @@ -66,7 +66,7 @@ jobs: generate_release_notes: true token: ${{ secrets.GITHUB_TOKEN }} files: ./dist/* - prerelease: false + prerelease: true - name: release library run: | diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index 41d89dc..4012555 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -23,8 +23,8 @@ var nonText = map[string]bool{ "V": true, // GCP "G": true, - // Unknown - "U": true, + // // Unknown + // "U": true, } type Source struct { @@ -117,19 +117,6 @@ func (l *Lexer) NextToken() config.Token { } else { tok = config.Token{Type: config.TEXT, Literal: "V"} } - case 'U': - // UNKNOWN - if l.peekChar() == 'N' { - l.readChar() - if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.UnknownPrefix}, "UN"); found { - tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} - } else { - // it is not a marker AW as text - tok = config.Token{Type: config.TEXT, Literal: "UN"} - } - } else { - tok = config.Token{Type: config.TEXT, Literal: "U"} - } case '=': tok = config.Token{Type: config.EQUALS, Literal: "="} case '.': diff --git a/internal/lexer/lexer_test.go b/internal/lexer/lexer_test.go index 79f945a..97f6ba0 100644 --- a/internal/lexer/lexer_test.go +++ b/internal/lexer/lexer_test.go @@ -33,9 +33,9 @@ META_INCLUDED=VAULT://baz/bar/123|key1.prop2[role=arn:aws:iam::1111111:role,vers {config.NEW_LINE, "\n"}, {config.TEXT, "MET"}, {config.TEXT, "A"}, - {config.TEXT, "_INCL"}, - {config.TEXT, "U"}, - {config.TEXT, "DED"}, + {config.TEXT, "_INCLUDED"}, + // {config.TEXT, "U"}, + // {config.TEXT, "DED"}, {config.EQUALS, "="}, {config.BEGIN_CONFIGMANAGER_TOKEN, "VAULT://"}, {config.TEXT, "baz"}, diff --git a/internal/store/paramstore_test.go b/internal/store/paramstore_test.go index db71b0f..8fc11d4 100644 --- a/internal/store/paramstore_test.go +++ b/internal/store/paramstore_test.go @@ -2,6 +2,7 @@ package store_test import ( "context" + "fmt" "io" "strings" "testing" @@ -14,11 +15,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ssm/types" ) -// var ( -// tsuccessParam = "someVal" -// tsuccessObj map[string]string = map[string]string{"AWSPARAMSTR#/token/1": "someVal"} -// ) - type mockParamApi func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) func (m mockParamApi) GetParameter(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { @@ -72,45 +68,69 @@ func Test_GetParamStore(t *testing.T) { }) }, }, - // "successVal with keyseparator": {"AWSPARAMSTR#/token/1|somekey", "|", "#", tsuccessParam, func(t *testing.T) paramStoreApi { - // return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - // t.Helper() - // awsParamtStoreCommonGetChecker(t, params) + "successVal with keyseparator": { + func() *config.ParsedTokenConfig { + // "AWSPARAMSTR#/token/1|somekey", + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("somekey") + tkn.WithMetadata("") + return tkn + }, + tsuccessParam, func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) - // if strings.Contains(*params.Name, "|somekey") { - // t.Errorf("incorrectly stripped key separator") - // } + if strings.Contains(*params.Name, "|somekey") { + t.Errorf("incorrectly stripped key separator") + } - // return &ssm.GetParameterOutput{ - // Parameter: &types.Parameter{Value: &tsuccessParam}, - // }, nil - // }) - // }, config.NewConfig(), - // }, - // "errored": {"AWSPARAMSTR#/token/1", "|", "#", "unable to retrieve", func(t *testing.T) paramStoreApi { - // return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - // t.Helper() - // awsParamtStoreCommonGetChecker(t, params) - // return nil, fmt.Errorf("unable to retrieve") - // }) - // }, config.NewConfig(), - // }, - // "nil to empty": {"AWSPARAMSTR#/token/1", "|", "#", "", func(t *testing.T) paramStoreApi { - // return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { - // t.Helper() - // awsParamtStoreCommonGetChecker(t, params) - // return &ssm.GetParameterOutput{ - // Parameter: &types.Parameter{Value: nil}, - // }, nil - // }) - // }, config.NewConfig(), - // }, + return &ssm.GetParameterOutput{ + Parameter: &types.Parameter{Value: &tsuccessParam}, + }, nil + }) + }, + }, + "errored": { + func() *config.ParsedTokenConfig { + // "AWSPARAMSTR#/token/1", + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "unable to retrieve", func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) + return nil, fmt.Errorf("unable to retrieve") + }) + }, + }, + "nil to empty": { + func() *config.ParsedTokenConfig { + // "AWSPARAMSTR#/token/1", + tkn, _ := config.NewToken(config.ParamStorePrefix, *config.NewConfig()) + tkn.WithSanitizedToken("/token/1") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "", func(t *testing.T) mockParamApi { + return mockParamApi(func(ctx context.Context, params *ssm.GetParameterInput, optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error) { + t.Helper() + awsParamtStoreCommonGetChecker(t, params) + return &ssm.GetParameterOutput{ + Parameter: &types.Parameter{Value: nil}, + }, nil + }) + }, + }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { - - // token, _ := config.NewToken(tt.token, *tt.config.WithTokenSeparator(tt.tokenSeparator).WithKeySeparator(tt.keySeparator)) - impl, err := store.NewParamStore(context.TODO(), log.New(io.Discard)) if err != nil { t.Errorf(testutils.TestPhrase, err.Error(), nil) From ca496401d12ad28712f5bd0be409318cfa031507 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 18:20:33 +0000 Subject: [PATCH 14/19] fix: tests --- generator/generator_test.go | 34 +- internal/store/hashivault_test.go | 831 +++++++++++++++++------------- 2 files changed, 484 insertions(+), 381 deletions(-) diff --git a/generator/generator_test.go b/generator/generator_test.go index e20b2f4..e48e546 100644 --- a/generator/generator_test.go +++ b/generator/generator_test.go @@ -26,19 +26,19 @@ func (m *mockGenerate) Value() (s string, e error) { return m.value, m.err } -func Test_Generate(t *testing.T) { +func TestGenerate(t *testing.T) { t.Run("succeeds with funcMap", func(t *testing.T) { var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"UNKNOWN://mountPath/token", "bar", nil} + m := &mockGenerate{"AWSPARAMSTR://mountPath/token", "bar", nil} return m, nil } g := generator.NewGenerator(context.TODO(), func(gv *generator.GenVars) { gv.Logger = log.New(&bytes.Buffer{}) }) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) - got, err := g.Generate([]string{"UNKNOWN://mountPath/token"}) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: custFunc}) + got, err := g.Generate([]string{"AWSPARAMSTR://mountPath/token"}) if err != nil { t.Fatal("errored on generate") @@ -50,13 +50,13 @@ func Test_Generate(t *testing.T) { t.Run("errors in retrieval and logs it out", func(t *testing.T) { var custFunc = func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - m := &mockGenerate{"UNKNOWN://mountPath/token", "bar", fmt.Errorf("failed to get value")} + m := &mockGenerate{"AWSPARAMSTR://mountPath/token", "bar", fmt.Errorf("failed to get value")} return m, nil } g := generator.NewGenerator(context.TODO()) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) - got, err := g.Generate([]string{"UNKNOWN://mountPath/token"}) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: custFunc}) + got, err := g.Generate([]string{"AWSPARAMSTR://mountPath/token"}) if err != nil { t.Fatal("errored on generate") @@ -73,8 +73,8 @@ func Test_Generate(t *testing.T) { } g := generator.NewGenerator(context.TODO()) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: custFunc}) - got, err := g.Generate([]string{"UNKNOWN://mountPath/token|key1.key2"}) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: custFunc}) + got, err := g.Generate([]string{"AWSPARAMSTR://mountPath/token|key1.key2"}) if err != nil { t.Fatal("errored on generate") @@ -82,13 +82,13 @@ func Test_Generate(t *testing.T) { if len(got) != 1 { t.Errorf(testutils.TestPhraseWithContext, "incorect number in a map", len(got), 0) } - if got["UNKNOWN://mountPath/token|key1.key2"] != "val" { - t.Errorf(testutils.TestPhraseWithContext, "incorrect value returned in parsedMap", got["UNKNOWN://mountPath/token|key1.key2"], "val") + if got["AWSPARAMSTR://mountPath/token|key1.key2"] != "val" { + t.Errorf(testutils.TestPhraseWithContext, "incorrect value returned in parsedMap", got["AWSPARAMSTR://mountPath/token|key1.key2"], "val") } }) } -func Test_generate_withKeys_lookup(t *testing.T) { +func TestGenerate_withKeys_lookup(t *testing.T) { ttests := map[string]struct { custFunc strategy.StrategyFunc token string @@ -99,7 +99,7 @@ func Test_generate_withKeys_lookup(t *testing.T) { m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":"val"}}`, nil} return m, nil }, - token: "UNKNOWN://mountPath/token|key1.key2", + token: "AWSPARAMSTR://mountPath/token|key1.key2", expectVal: "val", }, "retrieves number value correctly from a keylookup inside": { @@ -107,7 +107,7 @@ func Test_generate_withKeys_lookup(t *testing.T) { m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":123}}`, nil} return m, nil }, - token: "UNKNOWN://mountPath/token|key1.key2", + token: "AWSPARAMSTR://mountPath/token|key1.key2", expectVal: "123", }, "retrieves nothing as keylookup is incorrect": { @@ -115,7 +115,7 @@ func Test_generate_withKeys_lookup(t *testing.T) { m := &mockGenerate{"token", `{"foo":"bar","key1":{"key2":123}}`, nil} return m, nil }, - token: "UNKNOWN://mountPath/token|noprop", + token: "AWSPARAMSTR://mountPath/token|noprop", expectVal: "", }, "retrieves value as is due to incorrectly stored json in backing store": { @@ -123,14 +123,14 @@ func Test_generate_withKeys_lookup(t *testing.T) { m := &mockGenerate{"token", `foo":"bar","key1":{"key2":123}}`, nil} return m, nil }, - token: "UNKNOWN://mountPath/token|noprop", + token: "AWSPARAMSTR://mountPath/token|noprop", expectVal: `foo":"bar","key1":{"key2":123}}`, }, } for name, tt := range ttests { t.Run(name, func(t *testing.T) { g := generator.NewGenerator(context.TODO()) - g.WithStrategyMap(strategy.StrategyFuncMap{config.UnknownPrefix: tt.custFunc}) + g.WithStrategyMap(strategy.StrategyFuncMap{config.ParamStorePrefix: tt.custFunc}) got, err := g.Generate([]string{tt.token}) if err != nil { diff --git a/internal/store/hashivault_test.go b/internal/store/hashivault_test.go index ccb061d..8c9aed8 100644 --- a/internal/store/hashivault_test.go +++ b/internal/store/hashivault_test.go @@ -2,8 +2,12 @@ package store_test import ( "context" + "fmt" "io" + "net/http" + "net/http/httptest" "os" + "strings" "testing" "github.com/DevLabFoundry/configmanager/v3/internal/config" @@ -99,7 +103,8 @@ func TestVaultScenarios(t *testing.T) { tkn.WithKeyPath("") tkn.WithMetadata("") return tkn - }, `{"foo":"test2130-9sd-0ds"}`, + }, + `{"foo":"test2130-9sd-0ds"}`, func(t *testing.T) mockVaultApi { mv := mockVaultApi{} mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { @@ -120,189 +125,266 @@ func TestVaultScenarios(t *testing.T) { } }, }, - // "incorrect json": {"VAULT://secret___/foo", config.NewConfig(), `json: unsupported type: func() error`, - // func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "foo" { - // t.Errorf("got %v; want %s", secretPath, `foo`) - // } - // m := make(map[string]interface{}) - // m["error"] = func() error { return fmt.Errorf("ddodod") } - // return &vault.KVSecret{Data: m}, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "another return": { - // "VAULT://secret/engine1___/some/other/foo2", - // config.NewConfig(), - // `{"foo1":"test2130-9sd-0ds","foo2":"dsfsdf3454456"}`, - // func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - // } - // m := make(map[string]interface{}) - // m["foo1"] = "test2130-9sd-0ds" - // m["foo2"] = "dsfsdf3454456" - // return &vault.KVSecret{Data: m}, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "not found": {"VAULT://secret___/foo", config.NewConfig(), `secret not found`, - // func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "foo" { - // t.Errorf("got %v; want %s", secretPath, `foo`) - // } - // return nil, fmt.Errorf("secret not found") - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "403": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `client 403`, - // func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - // } - // return nil, fmt.Errorf("client 403") - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "found but empty": {"VAULT://secret___/some/other/foo2", config.NewConfig(), `{}`, func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) - // } - // m := make(map[string]interface{}) - // return &vault.KVSecret{Data: m}, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "found but nil returned": {"VAULT://secret___/some/other/foo2", config.NewConfig(), "", func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - // } - // return &vault.KVSecret{Data: nil}, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "version provided correctly": {"VAULT://secret___/some/other/foo2[version=1]", config.NewConfig(), `{"foo2":"dsfsdf3454456"}`, func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - // } - // m := make(map[string]interface{}) - // m["foo2"] = "dsfsdf3454456" - // return &vault.KVSecret{Data: m}, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "version provided but unable to parse": {"VAULT://secret___/some/other/foo2[version=1a]", config.NewConfig(), "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - // } - // return nil, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "129378y1231283") - // return func() { - // os.Clearenv() - // } - // }, - // }, - // "vault rate limit incorrect": { - // "VAULT://secret___/some/other/foo2", - // config.NewConfig(), - // `error encountered setting up default configuration: VAULT_RATE_LIMIT was provided but incorrectly formatted - // failed to initialize the client`, - // func(t *testing.T) hashiVaultApi { - // mv := mockVaultApi{} - // mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { - // t.Helper() - // if secretPath != "some/other/foo2" { - // t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) - // } - // return &vault.KVSecret{Data: nil}, nil - // } - // return mv - // }, - // func() func() { - // os.Setenv("VAULT_TOKEN", "") - // os.Setenv("VAULT_RATE_LIMIT", "wrong") - // return func() { - // os.Clearenv() - // } - // }, - // }, + "incorrect json": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/foo", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/foo") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `json: unsupported type: func() error`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "foo" { + t.Errorf("got %v; want %s", secretPath, `foo`) + } + m := make(map[string]interface{}) + m["error"] = func() error { return fmt.Errorf("ddodod") } + return &vault.KVSecret{Data: m}, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "another return": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret/engine1___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `{"foo1":"test2130-9sd-0ds","foo2":"dsfsdf3454456"}`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + } + m := make(map[string]interface{}) + m["foo1"] = "test2130-9sd-0ds" + m["foo2"] = "dsfsdf3454456" + return &vault.KVSecret{Data: m}, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "not found": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/foo", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/foo") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `secret not found`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "foo" { + t.Errorf("got %v; want %s", secretPath, `foo`) + } + return nil, fmt.Errorf("secret not found") + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "403": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `client 403`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + } + return nil, fmt.Errorf("client 403") + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "found but empty": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + // config.NewConfig(), + `{}`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf("got %v; want %s", secretPath, `some/other/foo2`) + } + m := make(map[string]interface{}) + return &vault.KVSecret{Data: m}, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "found but nil returned": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + "", + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return &vault.KVSecret{Data: nil}, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "version provided correctly": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2[version=1]", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1") + return tkn + }, + `{"foo2":"dsfsdf3454456"}`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + m := make(map[string]interface{}) + m["foo2"] = "dsfsdf3454456" + return &vault.KVSecret{Data: m}, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "version provided but unable to parse": { + func() *config.ParsedTokenConfig { + // "VAULT://secret___/some/other/foo2[version=1a]", + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1a") + return tkn + }, + "unable to parse version into an integer: strconv.Atoi: parsing \"1a\": invalid syntax", + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.gv = func(ctx context.Context, secretPath string, version int) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return nil, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "129378y1231283") + return func() { + os.Clearenv() + } + }, + }, + "vault rate limit incorrect": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("") + return tkn + }, + `error encountered setting up default configuration: VAULT_RATE_LIMIT was provided but incorrectly formatted +failed to initialize the client`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return &vault.KVSecret{Data: nil}, nil + } + return mv + }, + func() func() { + os.Setenv("VAULT_TOKEN", "") + os.Setenv("VAULT_RATE_LIMIT", "wrong") + return func() { + os.Clearenv() + } + }, + }, } for name, tt := range ttests { @@ -333,189 +415,210 @@ func TestVaultScenarios(t *testing.T) { } } -// func TestAwsIamAuth(t *testing.T) { -// ttests := map[string]struct { -// token string -// conf *config.GenVarsConfig -// expect string -// mockClient func(t *testing.T) hashiVaultApi -// mockHanlder func(t *testing.T) http.Handler -// setupEnv func(addr string) func() -// }{ -// "aws_iam auth no role specified": { -// "VAULT://secret___/some/other/foo2[version:1]", config.NewConfig(), -// "role provided is empty, EC2 auth not supported", -// func(t *testing.T) hashiVaultApi { -// mv := mockVaultApi{} -// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { -// t.Helper() -// if secretPath != "some/other/foo2" { -// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) -// } -// return &vault.KVSecret{Data: nil}, nil -// } -// return mv -// }, -// func(t *testing.T) http.Handler { -// return nil -// }, -// func(_ string) func() { -// os.Setenv("VAULT_TOKEN", "aws_iam") -// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") -// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") -// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") -// os.Setenv("AWS_REGION", "eu-west-1") -// return func() { -// os.Clearenv() -// } -// }, -// }, -// "aws_iam auth incorrectly formatted request": { -// "VAULT://secret___/some/other/foo2[version=1,iam_role=not_a_role]", config.NewConfig(), -// `unable to login to AWS auth method: unable to log in to auth method: unable to log in with AWS auth: Error making API request. - -// URL: PUT %s/v1/auth/aws/login -// Code: 400. Raw Message: - -// incorrect values supplied. failed to initialize the client`, -// func(t *testing.T) hashiVaultApi { -// mv := mockVaultApi{} -// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { -// t.Helper() -// if secretPath != "some/other/foo2" { -// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) -// } -// return &vault.KVSecret{Data: nil}, nil -// } -// return mv -// }, -// func(t *testing.T) http.Handler { -// mux := http.NewServeMux() -// mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { +func TestAwsIamAuth(t *testing.T) { + ttests := map[string]struct { + token func() *config.ParsedTokenConfig + expect string + mockClient func(t *testing.T) mockVaultApi + mockHanlder func(t *testing.T) http.Handler + setupEnv func(addr string) func() + }{ + "aws_iam auth no role specified": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1") + return tkn + }, + "role provided is empty, EC2 auth not supported", + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return &vault.KVSecret{Data: nil}, nil + } + return mv + }, + func(t *testing.T) http.Handler { + return nil + }, + func(_ string) func() { + os.Setenv("VAULT_TOKEN", "aws_iam") + os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") + os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") + os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") + os.Setenv("AWS_REGION", "eu-west-1") + return func() { + os.Clearenv() + } + }, + }, + "aws_iam auth incorrectly formatted request": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("version=1,iam_role=not_a_role") + return tkn + }, + `unable to login to AWS auth method: unable to log in to auth method: unable to log in with AWS auth: Error making API request. -// w.Header().Set("Content-Type", "application/json; charset=utf-8") -// w.WriteHeader(400) -// w.Write([]byte(`incorrect values supplied`)) -// }) -// return mux -// }, -// func(addr string) func() { -// os.Setenv("VAULT_TOKEN", "aws_iam") -// os.Setenv("VAULT_ADDR", addr) -// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") -// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") -// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") -// os.Setenv("AWS_REGION", "eu-west-1") -// return func() { -// os.Clearenv() -// } -// }, -// }, -// "aws_iam auth success": { -// "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), -// `{"foo2":"dsfsdf3454456"}`, -// func(t *testing.T) hashiVaultApi { -// mv := mockVaultApi{} -// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { -// t.Helper() -// if secretPath != "some/other/foo2" { -// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) -// } -// m := make(map[string]interface{}) -// m["foo2"] = "dsfsdf3454456" -// return &vault.KVSecret{Data: m}, nil -// } -// return mv -// }, -// func(t *testing.T) http.Handler { -// mux := http.NewServeMux() -// mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { +URL: PUT %s/v1/auth/aws/login +Code: 400. Raw Message: -// w.Header().Set("Content-Type", "application/json; charset=utf-8") -// w.Write([]byte(`{"auth":{"client_token": "fooresddfasdsasad"}}`)) -// }) -// return mux -// }, -// func(addr string) func() { -// os.Setenv("VAULT_TOKEN", "aws_iam") -// os.Setenv("VAULT_ADDR", addr) -// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") -// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") -// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") -// os.Setenv("AWS_REGION", "eu-west-1") -// return func() { -// os.Clearenv() -// } -// }, -// }, -// "aws_iam auth no token returned": { -// "VAULT://secret___/some/other/foo2[iam_role=arn:aws:iam::1111111:role/i-orchestration]", config.NewConfig(), -// `unable to login to AWS auth method: response did not return ClientToken, client token not set. failed to initialize the client`, -// func(t *testing.T) hashiVaultApi { -// mv := mockVaultApi{} -// mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { -// t.Helper() -// if secretPath != "some/other/foo2" { -// t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) -// } -// m := make(map[string]interface{}) -// m["foo2"] = "dsfsdf3454456" -// return &vault.KVSecret{Data: m}, nil -// } -// return mv -// }, -// func(t *testing.T) http.Handler { -// mux := http.NewServeMux() -// mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { +incorrect values supplied. failed to initialize the client`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + return &vault.KVSecret{Data: nil}, nil + } + return mv + }, + func(t *testing.T) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { -// w.Header().Set("Content-Type", "application/json; charset=utf-8") -// w.Write([]byte(`{"auth":{}}`)) -// }) -// return mux -// }, -// func(addr string) func() { -// os.Setenv("VAULT_TOKEN", "aws_iam") -// os.Setenv("VAULT_ADDR", addr) -// os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") -// os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") -// os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") -// os.Setenv("AWS_REGION", "eu-west-1") -// return func() { -// os.Clearenv() -// } -// }, -// }, -// } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(400) + w.Write([]byte(`incorrect values supplied`)) + }) + return mux + }, + func(addr string) func() { + os.Setenv("VAULT_TOKEN", "aws_iam") + os.Setenv("VAULT_ADDR", addr) + os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") + os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") + os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") + os.Setenv("AWS_REGION", "eu-west-1") + return func() { + os.Clearenv() + } + }, + }, + "aws_iam auth success": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("iam_role=arn:aws:iam::1111111:role/i-orchestration") + return tkn + }, + // + `{"foo2":"dsfsdf3454456"}`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + m := make(map[string]any) + m["foo2"] = "dsfsdf3454456" + return &vault.KVSecret{Data: m}, nil + } + return mv + }, + func(t *testing.T) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { -// for name, tt := range ttests { -// t.Run(name, func(t *testing.T) { -// // -// ts := httptest.NewServer(tt.mockHanlder(t)) -// tearDown := tt.setupEnv(ts.URL) -// defer tearDown() -// token, _ := config.NewToken(tt.token, *tt.conf) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Write([]byte(`{"auth":{"client_token": "fooresddfasdsasad"}}`)) + }) + return mux + }, + func(addr string) func() { + os.Setenv("VAULT_TOKEN", "aws_iam") + os.Setenv("VAULT_ADDR", addr) + os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") + os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") + os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") + os.Setenv("AWS_REGION", "eu-west-1") + return func() { + os.Clearenv() + } + }, + }, + "aws_iam auth no token returned": { + func() *config.ParsedTokenConfig { + tkn, _ := config.NewToken(config.HashicorpVaultPrefix, *config.NewConfig()) + tkn.WithSanitizedToken("secret___/some/other/foo2") + tkn.WithKeyPath("") + tkn.WithMetadata("iam_role=arn:aws:iam::1111111:role/i-orchestration") + return tkn + }, + `unable to login to AWS auth method: response did not return ClientToken, client token not set. failed to initialize the client`, + func(t *testing.T) mockVaultApi { + mv := mockVaultApi{} + mv.g = func(ctx context.Context, secretPath string) (*vault.KVSecret, error) { + t.Helper() + if secretPath != "some/other/foo2" { + t.Errorf(testutils.TestPhrase, secretPath, `some/other/foo2`) + } + m := make(map[string]interface{}) + m["foo2"] = "dsfsdf3454456" + return &vault.KVSecret{Data: m}, nil + } + return mv + }, + func(t *testing.T) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/v1/auth/aws/login", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Write([]byte(`{"auth":{}}`)) + }) + return mux + }, + func(addr string) func() { + os.Setenv("VAULT_TOKEN", "aws_iam") + os.Setenv("VAULT_ADDR", addr) + os.Setenv("AWS_ACCESS_KEY_ID", "1280qwed9u9nsc9fdsbv9gsfrd") + os.Setenv("AWS_SECRET_ACCESS_KEY", "SED)SDVfdv0jfds08sdfgu09sd943tj4fELH/") + os.Setenv("AWS_SESSION_TOKEN", "IQoJb3JpZ2luX2VjELH//////////wEaCWV1LXdlc3QtMiJIMEYCIQDPU6UGJ0...df.fdgdfg.dfg.gdf.dgf") + os.Setenv("AWS_REGION", "eu-west-1") + return func() { + os.Clearenv() + } + }, + }, + } -// impl, err := NewVaultStore(context.TODO(), token, log.New(io.Discard)) -// if err != nil { -// // WHAT A CRAP way to do this... -// if err.Error() != strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0] { -// t.Errorf(testutils.TestPhraseWithContext, "aws iam auth", err.Error(), strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0]) -// t.Fatalf("failed to init hashivault, %v", err.Error()) -// } -// return -// } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + // + ts := httptest.NewServer(tt.mockHanlder(t)) + tearDown := tt.setupEnv(ts.URL) + defer tearDown() + impl, err := store.NewVaultStore(context.TODO(), tt.token(), log.New(io.Discard)) + if err != nil { + // WHAT A CRAP way to do this... + if err.Error() != strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0] { + t.Errorf(testutils.TestPhraseWithContext, "aws iam auth", err.Error(), strings.Split(fmt.Sprintf(tt.expect, ts.URL), `%!`)[0]) + t.Fatalf("failed to init hashivault, %v", err.Error()) + } + return + } -// impl.svc = tt.mockClient(t) -// got, err := impl.Token() -// if err != nil { -// if err.Error() != tt.expect { -// t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) -// } -// return -// } -// if got != tt.expect { -// t.Errorf(testutils.TestPhrase, got, tt.expect) -// } -// }) -// } -// } + impl.WithSvc(tt.mockClient(t)) + got, err := impl.Value() + if err != nil { + if err.Error() != tt.expect { + t.Errorf(testutils.TestPhrase, err.Error(), tt.expect) + } + return + } + if got != tt.expect { + t.Errorf(testutils.TestPhrase, got, tt.expect) + } + }) + } +} From 88b98dbd1516a62661458cdd1b50e0b0005aafa3 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 18:29:20 +0000 Subject: [PATCH 15/19] fix: linter --- .github/workflows/build.yml | 2 ++ .github/workflows/release.yml | 2 ++ internal/lexer/lexer.go | 11 ++-------- internal/strategy/strategy.go | 39 +++++++---------------------------- 4 files changed, 14 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0af6870..e0aae4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,7 @@ permissions: jobs: set-version: + name: Set Version runs-on: ubuntu-latest container: image: mcr.microsoft.com/dotnet/sdk:10.0 @@ -44,6 +45,7 @@ jobs: test: runs-on: ubuntu-latest + name: Run Tests needs: set-version env: SEMVER: ${{ needs.set-version.outputs.semVer }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd8c725..405c112 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,7 @@ permissions: jobs: set-version: + name: Set Version runs-on: ubuntu-latest if: ${{ github.event.workflow_run.head_branch == 'master' && github.event.workflow_run.conclusion == 'success' }} container: @@ -39,6 +40,7 @@ jobs: id: gitversion release: + name: Release runs-on: ubuntu-latest needs: set-version env: diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index 4012555..9c3106a 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -17,14 +17,12 @@ var nonText = map[string]bool{ // this forces the lexer to not treat at as TEXT // and enter the switch statement of the state machine // NOTE: when a new implementation is added we should add it here - // AWS|AZure... + // AWS|AZure|GCP... "A": true, // VAULT (HashiCorp) "V": true, // GCP "G": true, - // // Unknown - // "U": true, } type Source struct { @@ -211,7 +209,7 @@ func (l *Lexer) setTextSeparatorToken() config.Token { return tok } -// peekIsBeginOfToken attempts to identify the gendoc keyword after 2 slashes +// peekIsBeginOfToken attempts to identify the possible token func (l *Lexer) peekIsBeginOfToken(possibleBeginToken []config.ImplementationPrefix, charsRead string) (bool, string, config.ImplementationPrefix) { for _, pbt := range possibleBeginToken { configToken := "" @@ -229,11 +227,6 @@ func (l *Lexer) peekIsBeginOfToken(possibleBeginToken []config.ImplementationPre return false, "", "" } -// peekIsEndOfToken -func (l *Lexer) peekIsEndOfToken() bool { - return false -} - // resetAfterPeek will go back specified amount on the cursor func (l *Lexer) resetAfterPeek(back int) { l.position = l.position - back diff --git a/internal/strategy/strategy.go b/internal/strategy/strategy.go index 47c47ac..7ed3b44 100644 --- a/internal/strategy/strategy.go +++ b/internal/strategy/strategy.go @@ -1,6 +1,4 @@ -// Package strategy is a strategy pattern wrapper around the store implementations -// -// NOTE: this may be refactored out into the store package directly +// Package strategy is a factory method wrapper around the backing store implementations package strategy import ( @@ -53,20 +51,17 @@ type strategyFnMap struct { funcMap StrategyFuncMap } -type RetrieveStrategy struct { - // mu *sync.Mutex - implementation store.Strategy +type Strategy struct { config config.GenVarsConfig strategyFuncMap strategyFnMap } -type Opts func(*RetrieveStrategy) +type Opts func(*Strategy) // New -func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *RetrieveStrategy { - rs := &RetrieveStrategy{ - config: config, - // mu: &sync.Mutex{}, +func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *Strategy { + rs := &Strategy{ + config: config, strategyFuncMap: strategyFnMap{mu: sync.Mutex{}, funcMap: defaultStrategyFuncMap(logger)}, } // overwrite or add any options/defaults set above @@ -82,7 +77,7 @@ func New(config config.GenVarsConfig, logger log.ILogger, opts ...Opts) *Retriev // Mainly used for testing // NOTE: this may lead to eventual optional configurations by users func WithStrategyFuncMap(funcMap StrategyFuncMap) Opts { - return func(rs *RetrieveStrategy) { + return func(rs *Strategy) { rs.strategyFuncMap.mu.Lock() defer rs.strategyFuncMap.mu.Unlock() for prefix, implementation := range funcMap { @@ -93,7 +88,7 @@ func WithStrategyFuncMap(funcMap StrategyFuncMap) Opts { // GetImplementation is a factory method returning the concrete implementation for the retrieval of the token value // i.e. facilitating the exchange of the supplied token for the underlying value -func (rs *RetrieveStrategy) GetImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { +func (rs *Strategy) GetImplementation(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { if token == nil { return nil, fmt.Errorf("unable to get prefix, %w", ErrTokenInvalid) } @@ -114,24 +109,6 @@ func ExchangeToken(s store.Strategy, token *config.ParsedTokenConfig) *TokenResp return cr } -// func (rs *RetrieveStrategy) setImplementation(strategy store.Strategy) { -// rs.mu.Lock() -// defer rs.mu.Unlock() -// rs.implementation = strategy -// } - -// func (rs *RetrieveStrategy) setTokenVal(s *config.ParsedTokenConfig) { -// rs.mu.Lock() -// defer rs.mu.Unlock() -// rs.implementation.SetToken(s) -// } - -// func (rs *RetrieveStrategy) getTokenValue() (string, error) { -// rs.mu.Lock() -// defer rs.mu.Unlock() -// return rs.implementation.Token() -// } - type TokenResponse struct { value string key *config.ParsedTokenConfig From 21049e98baef48cf36dc7fa5c800f393792f8fa0 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 19:00:58 +0000 Subject: [PATCH 16/19] fix: additional tests --- internal/config/config_test.go | 18 ++++++++++ internal/config/token.go | 15 -------- internal/lexer/lexer.go | 3 -- internal/parser/parser.go | 31 ++++------------- internal/parser/parser_test.go | 33 +++++++++++++++++- internal/strategy/strategy.go | 62 +++++++++++++++++----------------- 6 files changed, 88 insertions(+), 74 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c85253c..8fc33f0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -164,3 +164,21 @@ func Test_TokenParser_config(t *testing.T) { }) } } + +func TestLookupIdent(t *testing.T) { + ttests := map[string]struct { + char string + expect config.TokenType + }{ + "new line": {"\n", config.NEW_LINE}, + "dash": {"-", config.TEXT}, + } + for name, tt := range ttests { + t.Run(name, func(t *testing.T) { + got := config.LookupIdent(tt.char) + if got != tt.expect { + t.Errorf("got %v wanted %v", got, tt.expect) + } + }) + } +} diff --git a/internal/config/token.go b/internal/config/token.go index 7b1c2d1..00b13a6 100644 --- a/internal/config/token.go +++ b/internal/config/token.go @@ -78,18 +78,3 @@ func LookupIdent(ident string) TokenType { } return TEXT } - -// var typeMapper = map[string]TokenType{ -// "MESSAGE": MESSAGE, -// "OPERATION": OPERATION, -// "CHANNEL": CHANNEL, -// "INFO": INFO, -// "SERVER": SERVER, -// } - -// func LookupType(typ string) TokenType { -// if tok, ok := typeMapper[strings.ToUpper(typ)]; ok { -// return tok -// } -// return "" -// } diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index 9c3106a..684be2c 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -156,9 +156,6 @@ func (l *Lexer) NextToken() config.Token { case 0: tok.Literal = "" tok.Type = config.EOF - // case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - // 'B', 'C', 'D', 'E', 'F', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'W', 'X', 'Y', 'Z', - // 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z': default: if isText(l.ch) { tok.Literal = l.readText() diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 7bb3c26..505b984 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -55,11 +55,6 @@ func New(l *lexer.Lexer, c *config.GenVarsConfig) *Parser { return p } -func (p *Parser) WithEnvironment(environ []string) *Parser { - p.environ = environ - return p -} - func (p *Parser) WithLogger(logger log.ILogger) *Parser { p.log = nil //speed up GC p.log = logger @@ -126,9 +121,6 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // built as part of the below parser sanitizedToken := "" - // should exit the loop if no end tag found - notFoundEnd := true - // stop on end of file for !p.peekTokenIs(config.EOF) { // // This is the target state when there is an optional token wrapping @@ -145,7 +137,6 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // when next token is another token // i.e. the tokens are adjacent if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { - notFoundEnd = false fullToken += p.curToken.Literal sanitizedToken += p.curToken.Literal stmt.EndToken = p.curToken @@ -156,7 +147,6 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa if p.peekTokenIsEnd() { // we want set the current token as both the full and sanitized // the current lexer token is the entire configmanager token - notFoundEnd = false fullToken += p.curToken.Literal sanitizedToken += p.curToken.Literal stmt.EndToken = p.curToken @@ -175,7 +165,6 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) return nil } - notFoundEnd = false // keyPath would have built the keyPath and metadata if any break } @@ -187,7 +176,6 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) return nil } - notFoundEnd = false break } @@ -199,7 +187,6 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // else it would be lost once the parser is advanced below p.nextToken() if p.peekTokenIs(config.EOF) { - notFoundEnd = false fullToken += p.curToken.Literal sanitizedToken += p.curToken.Literal stmt.EndToken = p.curToken @@ -207,10 +194,10 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa } } - if notFoundEnd { - p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, ErrNoEndTagFound)) - return nil - } + // if notFoundEnd { + // p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, ErrNoEndTagFound)) + // return nil + // } configManagerToken.WithSanitizedToken(sanitizedToken) stmt.ParsedToken = *configManagerToken @@ -269,13 +256,8 @@ func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) err p.nextToken() for !p.peekTokenIs(config.EOF) { if p.peekTokenIsEnd() { - if p.currentTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { - metadata += p.curToken.Literal - found = true - p.nextToken() - break - } - return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) + // next token is an end of token but no closing `]` found + return fmt.Errorf("%w, metadata (%s) string has no closing", ErrNoEndTagFound, metadata) } if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { metadata += p.curToken.Literal @@ -289,6 +271,7 @@ func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) err configManagerToken.WithMetadata(metadata) if !found { + // hit the end of file and no end tag found return fmt.Errorf("%w, metadata string has no closing", ErrNoEndTagFound) } return nil diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 4fc9692..089d4f2 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -20,7 +20,7 @@ func Test_ParserBlocks(t *testing.T) { // prefix,path,keyLookup expected [][3]string }{ - "tokens touching each other in source": { + "tokens touching each other in source after key path": { `foo stuyfsdfsf foo=AWSPARAMSTR:///path|keyAWSSECRETS:///foo other text her @@ -42,6 +42,26 @@ func Test_ParserBlocks(t *testing.T) { {string(config.ParamStorePrefix), "/config", "qp2"}, }, }, + "tokens touching each other in source after metadata": { + `foo stuyfsdfsf + foo=AWSPARAMSTR:///path|key[meta=val]AWSSECRETS:///foo + other text her + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", "key"}, + {string(config.SecretMgrPrefix), "/foo", ""}, + }, + }, + "tokens touching each other in source": { + `foo stuyfsdfsf + foo=AWSPARAMSTR:///pathAWSSECRETS:///foo + other text her + BAR=something + `, [][3]string{ + {string(config.ParamStorePrefix), "/path", ""}, + {string(config.SecretMgrPrefix), "/foo", ""}, + }, + }, "touching EOF single token": { `AWSPARAMSTR:///config|qp2`, [][3]string{ @@ -99,6 +119,12 @@ func Test_Parse_should_fail_on_metadata(t *testing.T) { `AWSSECRETS:///foo|path.one[version=1.2.3`, parser.ErrNoEndTagFound, }, + "when _end_tag_found with keysPath in the middle": { + `AWSSECRETS:///foo|path.one[version=1.2.3 + more content here +`, + parser.ErrNoEndTagFound, + }, "when no metadata has been supplied": { `AWSSECRETS:///foo|path.one[]`, parser.ErrMetadataEmpty, @@ -134,6 +160,11 @@ func Test_Parse_should_pass_with_metadata_end_tag(t *testing.T) { `AWSSECRETS:///foo[version=1.2.3]`, `version=1.2.3`, }, + "without keysPath in the middle of content": { + `AWSSECRETS:///foo[version=1.2.3] +`, + `version=1.2.3`, + }, "with keysPath": { `AWSSECRETS:///foo|path.one[version=1.2.3]`, `version=1.2.3`, diff --git a/internal/strategy/strategy.go b/internal/strategy/strategy.go index 7ed3b44..ac19a30 100644 --- a/internal/strategy/strategy.go +++ b/internal/strategy/strategy.go @@ -20,37 +20,6 @@ type StrategyFunc func(ctx context.Context, token *config.ParsedTokenConfig) (st // StrategyFuncMap type StrategyFuncMap map[config.ImplementationPrefix]StrategyFunc -func defaultStrategyFuncMap(logger log.ILogger) map[config.ImplementationPrefix]StrategyFunc { - return map[config.ImplementationPrefix]StrategyFunc{ - config.AzTableStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewAzTableStore(ctx, token, logger) - }, - config.AzAppConfigPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewAzAppConf(ctx, token, logger) - }, - config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewGcpSecrets(ctx, logger) - }, - config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewSecretsMgr(ctx, logger) - }, - config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewParamStore(ctx, logger) - }, - config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewKvScrtStore(ctx, token, logger) - }, - config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { - return store.NewVaultStore(ctx, token, logger) - }, - } -} - -type strategyFnMap struct { - mu sync.Mutex - funcMap StrategyFuncMap -} - type Strategy struct { config config.GenVarsConfig strategyFuncMap strategyFnMap @@ -122,3 +91,34 @@ func (tr *TokenResponse) Key() *config.ParsedTokenConfig { func (tr *TokenResponse) Value() string { return tr.value } + +func defaultStrategyFuncMap(logger log.ILogger) StrategyFuncMap { + return map[config.ImplementationPrefix]StrategyFunc{ + config.AzTableStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewAzTableStore(ctx, token, logger) + }, + config.AzAppConfigPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewAzAppConf(ctx, token, logger) + }, + config.GcpSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewGcpSecrets(ctx, logger) + }, + config.SecretMgrPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewSecretsMgr(ctx, logger) + }, + config.ParamStorePrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewParamStore(ctx, logger) + }, + config.AzKeyVaultSecretsPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewKvScrtStore(ctx, token, logger) + }, + config.HashicorpVaultPrefix: func(ctx context.Context, token *config.ParsedTokenConfig) (store.Strategy, error) { + return store.NewVaultStore(ctx, token, logger) + }, + } +} + +type strategyFnMap struct { + mu sync.Mutex + funcMap StrategyFuncMap +} From dc5fa5e1568032f03ba8e9fb75acdd8db6549b4a Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 19:04:45 +0000 Subject: [PATCH 17/19] fix: add more implicit tests to lexer --- internal/lexer/lexer.go | 4 ++-- internal/parser/parser_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index 684be2c..a8f5d5a 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -17,7 +17,7 @@ var nonText = map[string]bool{ // this forces the lexer to not treat at as TEXT // and enter the switch statement of the state machine // NOTE: when a new implementation is added we should add it here - // AWS|AZure|GCP... + // AWS|AZure "A": true, // VAULT (HashiCorp) "V": true, @@ -96,7 +96,7 @@ func (l *Lexer) NextToken() config.Token { if found, literal, imp := l.peekIsBeginOfToken([]config.ImplementationPrefix{config.GcpSecretsPrefix}, "GC"); found { tok = config.Token{Type: config.BEGIN_CONFIGMANAGER_TOKEN, Literal: literal, ImpPrefix: imp} } else { - // it is not a marker AW as text + // it is not a marker - GC literal as text tok = config.Token{Type: config.TEXT, Literal: "GC"} } } else { diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 089d4f2..2a97c04 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -53,7 +53,7 @@ func Test_ParserBlocks(t *testing.T) { }, }, "tokens touching each other in source": { - `foo stuyfsdfsf + `foo stuyfsdfsf GCFOO VAbarAWbuX AZmore foo=AWSPARAMSTR:///pathAWSSECRETS:///foo other text her BAR=something From 4f9b43ff979cadd4cabcfac6bda282963f5f0cff Mon Sep 17 00:00:00 2001 From: dnitsch Date: Mon, 3 Nov 2025 19:20:43 +0000 Subject: [PATCH 18/19] fix: ineffassign linter errors --- internal/config/config.go | 4 +++ internal/parser/parser.go | 70 ++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 0631da5..d1d05ed 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -239,3 +239,7 @@ func (t *ParsedTokenConfig) Metadata() string { func (t *ParsedTokenConfig) Prefix() ImplementationPrefix { return t.prefix } + +func (t *ParsedTokenConfig) TokenSeparator() string { + return t.tokenSeparator +} diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 505b984..b785dc2 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -10,8 +10,8 @@ import ( "github.com/DevLabFoundry/configmanager/v3/internal/log" ) -func wrapErr(incompleteToken string, line, position int, etyp error) error { - return fmt.Errorf("\n- token: (%s) on line: %d column: %d] %w", incompleteToken, line, position, etyp) +func wrapErr(incompleteToken *config.ParsedTokenConfig, sanitized string, line, position int, etyp error) error { + return fmt.Errorf("\n- token: (%s%s%s) on line: %d column: %d] %w", incompleteToken.Prefix(), incompleteToken.TokenSeparator(), sanitized, line, position, etyp) } var ( @@ -26,13 +26,13 @@ type ConfigManagerTokenBlock struct { } type Parser struct { - l *lexer.Lexer - errors []error - log log.ILogger - curToken config.Token - peekToken config.Token - config *config.GenVarsConfig - environ []string + l *lexer.Lexer + errors []error + log log.ILogger + currentToken config.Token + peekToken config.Token + config *config.GenVarsConfig + environ []string } func New(l *lexer.Lexer, c *config.GenVarsConfig) *Parser { @@ -71,7 +71,7 @@ func (p *Parser) Parse() ([]ConfigManagerTokenBlock, []error) { for !p.currentTokenIs(config.EOF) { if p.currentTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { // continues to read the tokens until it hits an end token or errors - configManagerToken, err := config.NewToken(p.curToken.ImpPrefix, *p.config) + configManagerToken, err := config.NewToken(p.currentToken.ImpPrefix, *p.config) if err != nil { return nil, []error{err} } @@ -86,12 +86,12 @@ func (p *Parser) Parse() ([]ConfigManagerTokenBlock, []error) { } func (p *Parser) nextToken() { - p.curToken = p.peekToken + p.currentToken = p.peekToken p.peekToken = p.l.NextToken() } func (p *Parser) currentTokenIs(t config.TokenType) bool { - return p.curToken.Type == t + return p.currentToken.Type == t } func (p *Parser) peekTokenIs(t config.TokenType) bool { @@ -111,13 +111,12 @@ func (p *Parser) peekTokenIsEnd() bool { // buildConfigManagerTokenFromBlocks func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.ParsedTokenConfig) *ConfigManagerTokenBlock { - currentToken := p.curToken + currentToken := p.currentToken stmt := &ConfigManagerTokenBlock{BeginToken: currentToken} // move past current token p.nextToken() - fullToken := currentToken.Literal // built as part of the below parser sanitizedToken := "" @@ -137,19 +136,15 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // when next token is another token // i.e. the tokens are adjacent if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) { - fullToken += p.curToken.Literal - sanitizedToken += p.curToken.Literal - stmt.EndToken = p.curToken + sanitizedToken += p.currentToken.Literal + stmt.EndToken = p.currentToken break } // reached the end of token if p.peekTokenIsEnd() { - // we want set the current token as both the full and sanitized - // the current lexer token is the entire configmanager token - fullToken += p.curToken.Literal - sanitizedToken += p.curToken.Literal - stmt.EndToken = p.curToken + sanitizedToken += p.currentToken.Literal + stmt.EndToken = p.currentToken break } @@ -162,7 +157,7 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // keyLookup and Metadata are optional - is always specified in that order if p.currentTokenIs(config.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR) { if err := p.buildKeyPathSeparator(configManagerToken); err != nil { - p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) + p.errors = append(p.errors, wrapErr(configManagerToken, sanitizedToken, currentToken.Line, currentToken.Column, err)) return nil } // keyPath would have built the keyPath and metadata if any @@ -173,32 +168,25 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa // check metadata there can be a metadata bracket `[key=val,k1=v2]` if p.currentTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { if err := p.buildMetadata(configManagerToken); err != nil { - p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, err)) + p.errors = append(p.errors, wrapErr(configManagerToken, sanitizedToken, currentToken.Line, currentToken.Column, err)) return nil } break } - sanitizedToken += p.curToken.Literal - fullToken += p.curToken.Literal + sanitizedToken += p.currentToken.Literal // when the next token is EOF // we want set the current token // else it would be lost once the parser is advanced below p.nextToken() if p.peekTokenIs(config.EOF) { - fullToken += p.curToken.Literal - sanitizedToken += p.curToken.Literal - stmt.EndToken = p.curToken + sanitizedToken += p.currentToken.Literal + stmt.EndToken = p.currentToken break } } - // if notFoundEnd { - // p.errors = append(p.errors, wrapErr(fullToken, currentToken.Line, currentToken.Column, ErrNoEndTagFound)) - // return nil - // } - configManagerToken.WithSanitizedToken(sanitizedToken) stmt.ParsedToken = *configManagerToken @@ -213,13 +201,13 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon if p.peekTokenIs(config.EOF) { // if the next token EOF we set the path as current token and exit // otherwise we would never hit the below loop - configManagerToken.WithKeyPath(p.curToken.Literal) + configManagerToken.WithKeyPath(p.currentToken.Literal) return nil } for !p.peekTokenIs(config.EOF) { if p.peekTokenIs(config.BEGIN_META_CONFIGMANAGER_TOKEN) { // add current token to the keysPath and move onto the metadata - keyPath += p.curToken.Literal + keyPath += p.currentToken.Literal p.nextToken() if err := p.buildMetadata(configManagerToken); err != nil { return err @@ -228,15 +216,15 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon } // touching another token or end of token if p.peekTokenIs(config.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() { - keyPath += p.curToken.Literal + keyPath += p.currentToken.Literal break } - keyPath += p.curToken.Literal + keyPath += p.currentToken.Literal p.nextToken() if p.peekTokenIs(config.EOF) { // check if the next token is EOF once advanced // if it is we want to consume current token else it will be skipped - keyPath += p.curToken.Literal + keyPath += p.currentToken.Literal break } } @@ -260,12 +248,12 @@ func (p *Parser) buildMetadata(configManagerToken *config.ParsedTokenConfig) err return fmt.Errorf("%w, metadata (%s) string has no closing", ErrNoEndTagFound, metadata) } if p.peekTokenIs(config.END_META_CONFIGMANAGER_TOKEN) { - metadata += p.curToken.Literal + metadata += p.currentToken.Literal found = true p.nextToken() break } - metadata += p.curToken.Literal + metadata += p.currentToken.Literal p.nextToken() } configManagerToken.WithMetadata(metadata) From ea155f2fca1d7c9235f88703787ffd41f637dd88 Mon Sep 17 00:00:00 2001 From: dnitsch Date: Thu, 6 Nov 2025 16:48:11 +0000 Subject: [PATCH 19/19] fix: add some docs --- docs/v3-updates-migrations.md | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/v3-updates-migrations.md diff --git a/docs/v3-updates-migrations.md b/docs/v3-updates-migrations.md new file mode 100644 index 0000000..3f05690 --- /dev/null +++ b/docs/v3-updates-migrations.md @@ -0,0 +1,49 @@ +# V3 Changes and updates + +As part of the V3 update we are aiming to streamline and improve the following high level areas. + +- Network call optimisation +- Backing store plugin architecture + +## Network Call optimisation + +There are many cases when an input string/file or array of `--token` can point to the same underlying token. + +e.g. +```yaml +db_user: AWSSECRETS:///app1/db|user +db_password: AWSSECRETS:///app1/db|password +db_port: AWSSECRETS:///app1/db|port +db_host: AWSSECRETS:///app1/db|host +``` + +Given the above input passed into the CLI i.e. `configmanager fromstr -i above-config.yaml` + +This would result in 4 network calls to the underlying service, in this case the AWS Secrets Manager. + +The V3 update would fan in these 4 tokens into a single network call and then fan out back to a full map with the individual values for each of the look up keys. + +> NB: any token using a metadata annotation on any token would guarantee a unique call to the underlying service + +e.g.: + +```yaml +db_user: AWSSECRETS:///app1/db|user +db_password: AWSSECRETS:///app1/db|password +db_port: AWSSECRETS:///app1/db|port +db_host: AWSSECRETS:///app1/db|host +db_host_2: AWSSECRETS:///app1/db|host[version=2] +``` + +Even though `AWSSECRETS:///app1/db|host[version=2]` and `AWSSECRETS:///app1/db|host` are technically the same AWS Secrets Manager item, specifying the version requires two separate network calls. + +## Backing Store plugin architecture + +The current implementation of the backing stores is defined entirely within the configmanager source code which becomes part of the final staticly linked binary. In order to avoid the bigger size binary and **more importantly** avoid security alerts for libraries that are nothing to do with a backing store provider which is not used! + +Most probably, and most commonly, one would only use single or a combination of providers within the same Cloud for example. + +### Plugin Architecture + +There are a few options to choose from - terraform style provider plugins using gRPC. +