Skip to content

Commit ea1e9e8

Browse files
tobiotobio
authored
Support secret ref arrays in integration policies (#1316)
* Support secret ref arrays in integration policies * Bump the highest stack version to 9.1.3
1 parent d89e2c7 commit ea1e9e8

File tree

5 files changed

+186
-63
lines changed

5 files changed

+186
-63
lines changed

‎.github/workflows/test.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,11 @@ jobs:
125125
- '8.14.3'
126126
- '8.15.5'
127127
- '8.16.2'
128-
- '8.17.0'
129-
- '8.18.3'
130-
- '9.0.3'
128+
- '8.17.10'
129+
- '8.18.7'
130+
- '8.19.3'
131+
- '9.0.7'
132+
- '9.1.3'
131133
steps:
132134
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
133135
- uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ SWAGGER_VERSION ?= 8.7
1515

1616
GOVERSION ?= $(shell grep -e '^go' go.mod | cut -f 2 -d ' ')
1717

18-
STACK_VERSION ?= 9.0.3
18+
STACK_VERSION ?= 9.1.3
1919

2020
ELASTICSEARCH_NAME ?= terraform-elasticstack-es
2121
ELASTICSEARCH_ENDPOINTS ?= http://$(ELASTICSEARCH_NAME):9200

‎internal/fleet/integration_policy/resource_test.go‎

Lines changed: 124 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ import (
2020
"github.com/stretchr/testify/require"
2121
)
2222

23-
var minVersionIntegrationPolicy = version.Must(version.NewVersion("8.10.0"))
23+
var (
24+
minVersionIntegrationPolicy = version.Must(version.NewVersion("8.10.0"))
25+
minVersionSqlIntegration = version.Must(version.NewVersion("9.1.0"))
26+
)
2427

2528
func TestJsonTypes(t *testing.T) {
2629
mapBytes, err := json.Marshal(map[string]string{})
@@ -127,53 +130,92 @@ func TestAccResourceIntegrationPolicySecretsFromSDK(t *testing.T) {
127130
func TestAccResourceIntegrationPolicySecrets(t *testing.T) {
128131
policyName := sdkacctest.RandStringFromCharSet(22, sdkacctest.CharSetAlphaNum)
129132

130-
resource.Test(t, resource.TestCase{
131-
PreCheck: func() { acctest.PreCheck(t) },
132-
CheckDestroy: checkResourceIntegrationPolicyDestroy,
133-
ProtoV6ProviderFactories: acctest.Providers,
134-
Steps: []resource.TestStep{
135-
{
136-
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
137-
Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"),
138-
Check: resource.ComposeTestCheckFunc(
139-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
140-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "IntegrationPolicyTest Policy"),
141-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
142-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
143-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-1","endpoint":"endpoint","secret_access_key":"created %s","session_token":"placeholder"}`, policyName)),
144-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
145-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"),
146-
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
147-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":true,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"1m","start_position":"beginning","tags":["forwarded"]}}}`),
148-
),
149-
},
150-
{
151-
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
152-
Config: testAccResourceIntegrationPolicySecretsUpdate(policyName, "updated"),
153-
Check: resource.ComposeTestCheckFunc(
154-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
155-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "Updated Integration Policy"),
156-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
157-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
158-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-2","endpoint":"endpoint","secret_access_key":"updated %s","session_token":"placeholder"}`, policyName)),
159-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
160-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "false"),
161-
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
162-
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":false,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"2m","start_position":"beginning","tags":["forwarded"]}}}`),
163-
),
133+
t.Run("single valued secrets", func(t *testing.T) {
134+
resource.Test(t, resource.TestCase{
135+
PreCheck: func() { acctest.PreCheck(t) },
136+
CheckDestroy: checkResourceIntegrationPolicyDestroy,
137+
ProtoV6ProviderFactories: acctest.Providers,
138+
Steps: []resource.TestStep{
139+
{
140+
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
141+
Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"),
142+
Check: resource.ComposeTestCheckFunc(
143+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
144+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "IntegrationPolicyTest Policy"),
145+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
146+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
147+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-1","endpoint":"endpoint","secret_access_key":"created %s","session_token":"placeholder"}`, policyName)),
148+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
149+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"),
150+
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
151+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":true,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"1m","start_position":"beginning","tags":["forwarded"]}}}`),
152+
),
153+
},
154+
{
155+
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
156+
Config: testAccResourceIntegrationPolicySecretsUpdate(policyName, "updated"),
157+
Check: resource.ComposeTestCheckFunc(
158+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
159+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "Updated Integration Policy"),
160+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
161+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
162+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-2","endpoint":"endpoint","secret_access_key":"updated %s","session_token":"placeholder"}`, policyName)),
163+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
164+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "false"),
165+
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
166+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":false,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"2m","start_position":"beginning","tags":["forwarded"]}}}`),
167+
),
168+
},
169+
{
170+
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
171+
ResourceName: "elasticstack_fleet_integration_policy.test_policy",
172+
Config: testAccResourceIntegrationPolicyUpdate(policyName),
173+
ImportState: true,
174+
ImportStateVerify: true,
175+
ImportStateVerifyIgnore: []string{"vars_json"},
176+
Check: resource.ComposeTestCheckFunc(
177+
resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", regexp.MustCompile(`{"access_key_id":{"id":"\S+","isSecretRef":true},"default_region":"us-east-2","endpoint":"endpoint","secret_access_key":{"id":"\S+","isSecretRef":true},"session_token":{"id":"\S+","isSecretRef":true}}`)),
178+
),
179+
},
164180
},
165-
{
166-
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
167-
ResourceName: "elasticstack_fleet_integration_policy.test_policy",
168-
Config: testAccResourceIntegrationPolicyUpdate(policyName),
169-
ImportState: true,
170-
ImportStateVerify: true,
171-
ImportStateVerifyIgnore: []string{"vars_json"},
172-
Check: resource.ComposeTestCheckFunc(
173-
resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", regexp.MustCompile(`{"access_key_id":{"id":"\S+","isSecretRef":true},"default_region":"us-east-2","endpoint":"endpoint","secret_access_key":{"id":"\S+","isSecretRef":true},"session_token":{"id":"\S+","isSecretRef":true}}`)),
174-
),
181+
})
182+
})
183+
184+
t.Run("multi-valued secrets", func(t *testing.T) {
185+
resource.Test(t, resource.TestCase{
186+
PreCheck: func() { acctest.PreCheck(t) },
187+
CheckDestroy: checkResourceIntegrationPolicyDestroy,
188+
ProtoV6ProviderFactories: acctest.Providers,
189+
Steps: []resource.TestStep{
190+
{
191+
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionSqlIntegration),
192+
Config: testAccResourceIntegrationPolicySecretsIds(policyName, "created"),
193+
Check: resource.ComposeTestCheckFunc(
194+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
195+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "SQL Integration Policy"),
196+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "sql"),
197+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.1.0"),
198+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "sql-sql/metrics"),
199+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"),
200+
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
201+
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"sql.sql":{"enabled":true,"vars":{"data_stream.dataset":"sql","driver":"mysql","hosts":["root:test@tcp(127.0.0.1:3306)/"],"merge_results":false,"period":"1m","processors":"","sql_queries":"- query: SHOW GLOBAL STATUS LIKE 'Innodb_system%'\n response_format: variables\n \n","ssl":""}}}`),
202+
),
203+
},
204+
{
205+
SkipFunc: func() (bool, error) {
206+
return versionutils.CheckIfVersionIsUnsupported(minVersionSqlIntegration)()
207+
},
208+
ResourceName: "elasticstack_fleet_integration_policy.test_policy",
209+
Config: testAccResourceIntegrationPolicySecretsIds(policyName, "created"),
210+
ImportState: true,
211+
ImportStateVerify: true,
212+
ImportStateVerifyIgnore: []string{"input.0.streams_json"},
213+
Check: resource.ComposeTestCheckFunc(
214+
resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", regexp.MustCompile(`"hosts":{"ids":["\S+"],"isSecretRef":true}`)),
215+
),
216+
},
175217
},
176-
},
218+
})
177219
})
178220
}
179221

@@ -451,3 +493,39 @@ resource "elasticstack_fleet_integration_policy" "test_policy" {
451493
}
452494
`, common, id, key, id)
453495
}
496+
497+
func testAccResourceIntegrationPolicySecretsIds(id string, key string) string {
498+
common := testAccResourceIntegrationPolicyCommon(id, "sql", "1.1.0")
499+
return fmt.Sprintf(`
500+
%s
501+
502+
resource "elasticstack_fleet_integration_policy" "test_policy" {
503+
name = "%s"
504+
namespace = "default"
505+
description = "SQL Integration Policy"
506+
agent_policy_id = elasticstack_fleet_agent_policy.test_policy.policy_id
507+
integration_name = elasticstack_fleet_integration.test_policy.name
508+
integration_version = elasticstack_fleet_integration.test_policy.version
509+
510+
input {
511+
input_id = "sql-sql/metrics"
512+
enabled = true
513+
streams_json = jsonencode({
514+
"sql.sql" : {
515+
"enabled" : true,
516+
"vars" : {
517+
"hosts" : ["root:test@tcp(127.0.0.1:3306)/"],
518+
"period" : "1m",
519+
"driver" : "mysql",
520+
"sql_queries" : "- query: SHOW GLOBAL STATUS LIKE 'Innodb_system%%'\n response_format: variables\n \n",
521+
"merge_results" : false,
522+
"ssl" : "",
523+
"data_stream.dataset" : "sql",
524+
"processors" : ""
525+
}
526+
}
527+
})
528+
}
529+
}
530+
`, common, id)
531+
}

‎internal/fleet/integration_policy/secrets.go‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,21 @@ func HandleRespSecrets(ctx context.Context, resp *kbapi.PackagePolicy, private p
7171
}
7272

7373
handleVar := func(key string, mval map[string]any, vars map[string]any) {
74-
refID := mval["id"].(string)
75-
if original, ok := secrets[refID]; ok {
76-
vars[key] = original
74+
if refID, ok := mval["id"]; ok {
75+
if original, ok := secrets[refID.(string)]; ok {
76+
vars[key] = original
77+
}
78+
} else if ids, ok := mval["ids"]; ok {
79+
values := []any{}
80+
for _, id := range ids.([]any) {
81+
if original, ok := secrets[id.(string)]; ok {
82+
values = append(values, original)
83+
}
84+
}
85+
86+
if len(values) > 0 {
87+
vars[key] = values
88+
}
7789
}
7890
}
7991

@@ -136,8 +148,22 @@ func HandleReqRespSecrets(ctx context.Context, req kbapi.PackagePolicyRequest, r
136148
}
137149
}
138150

139-
refID := mval["id"].(string)
140-
secrets[refID] = original
151+
if refID, ok := mval["id"]; ok {
152+
secrets[refID.(string)] = original
153+
} else if ids, ok := mval["ids"]; ok {
154+
originals, ok := original.([]any)
155+
if !ok || len(originals) != len(ids.([]any)) {
156+
diags.AddError("mismatched secret ref ids and original values", "the number of secret ref ids does not match the number of original values")
157+
return
158+
}
159+
160+
// Map each id to the corresponding original value by position.
161+
// The API does not return the original value with the id,
162+
// so we have to assume the order is preserved.
163+
for i, id := range ids.([]any) {
164+
secrets[id.(string)] = originals[i]
165+
}
166+
}
141167
}
142168
}
143169

‎internal/fleet/integration_policy/secrets_test.go‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@ func TestHandleRespSecrets(t *testing.T) {
3333
t.Parallel()
3434

3535
ctx := context.Background()
36-
private := privateData{"secrets": `{"known-secret":"secret"}`}
36+
private := privateData{"secrets": `{"known-secret":"secret", "known-secret-1":"secret1", "known-secret-2":"secret2"}`}
3737

3838
secretRefs := &[]kbapi.PackagePolicySecretRef{
3939
{Id: "known-secret"},
40+
{Id: "known-secret-1"},
41+
{Id: "known-secret-2"},
4042
}
4143

4244
tests := []struct {
@@ -64,6 +66,11 @@ func TestHandleRespSecrets(t *testing.T) {
6466
input: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}},
6567
want: Map{"k": "secret"},
6668
},
69+
{
70+
name: "converts secret with multiple values",
71+
input: Map{"k": Map{"isSecretRef": true, "ids": []any{"known-secret-1", "known-secret-2"}}},
72+
want: Map{"k": []any{"secret1", "secret2"}},
73+
},
6774
{
6875
name: "converts wrapped secret",
6976
input: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "known-secret"}}},
@@ -121,7 +128,7 @@ func TestHandleRespSecrets(t *testing.T) {
121128
require.Equal(t, want, got)
122129

123130
// privateData
124-
privateWants := privateData{"secrets": `{"known-secret":"secret"}`}
131+
privateWants := privateData{"secrets": `{"known-secret":"secret","known-secret-1":"secret1","known-secret-2":"secret2"}`}
125132
require.Equal(t, privateWants, private)
126133
})
127134
}
@@ -134,6 +141,8 @@ func TestHandleReqRespSecrets(t *testing.T) {
134141

135142
secretRefs := &[]kbapi.PackagePolicySecretRef{
136143
{Id: "known-secret"},
144+
{Id: "known-secret-1"},
145+
{Id: "known-secret-2"},
137146
}
138147

139148
tests := []struct {
@@ -166,6 +175,12 @@ func TestHandleReqRespSecrets(t *testing.T) {
166175
respInput: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}},
167176
want: Map{"k": "secret"},
168177
},
178+
{
179+
name: "converts secret with multiple values",
180+
reqInput: Map{"k": []any{"secret1", "secret2"}},
181+
respInput: Map{"k": Map{"isSecretRef": true, "ids": []any{"known-secret-1", "known-secret-2"}}},
182+
want: Map{"k": []any{"secret1", "secret2"}},
183+
},
169184
{
170185
name: "converts wrapped secret",
171186
reqInput: Map{"k": "secret"},
@@ -236,13 +251,15 @@ func TestHandleReqRespSecrets(t *testing.T) {
236251
want = *(*wants.Inputs["input1"].Streams)["stream1"].Vars
237252
require.Equal(t, want, got)
238253

239-
if v, ok := (*req.Vars)["k"]; ok && v == "secret" {
240-
privateWants := privateData{"secrets": `{"known-secret":"secret"}`}
241-
require.Equal(t, privateWants, private)
242-
} else {
243-
privateWants := privateData{"secrets": `{}`}
244-
require.Equal(t, privateWants, private)
254+
privateWants := privateData{"secrets": `{}`}
255+
if v, ok := (*req.Vars)["k"]; ok {
256+
if s, ok := v.(string); ok && s == "secret" {
257+
privateWants = privateData{"secrets": `{"known-secret":"secret"}`}
258+
} else if _, ok := v.([]any); ok {
259+
privateWants = privateData{"secrets": `{"known-secret-1":"secret1","known-secret-2":"secret2"}`}
260+
}
245261
}
262+
require.Equal(t, privateWants, private)
246263
})
247264
}
248265
}

0 commit comments

Comments
 (0)