Skip to content

Commit 5ed4523

Browse files
tobiotobio
committed
Support both mapped and typed input schemas in Fleet integration policies
1 parent 28212a7 commit 5ed4523

File tree

7 files changed

+665
-124
lines changed

7 files changed

+665
-124
lines changed

‎generated/kbapi/kibana.gen.go‎

Lines changed: 316 additions & 34 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎generated/kbapi/transform_schema.go‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"strconv"
1818
"strings"
1919

20+
"github.com/google/go-cmp/cmp"
2021
"gopkg.in/yaml.v3"
2122
)
2223

@@ -348,7 +349,11 @@ func (m Map) CreateRef(schema *Schema, name string, key string) Map {
348349
if reflect.DeepEqual(refTarget, existing) {
349350
writeComponent = false
350351
} else {
351-
log.Panicf("Component schema key already in use and not an exact duplicate: %q", refPath)
352+
log.Panicf(
353+
"Component schema key already in use and not an exact duplicate: %q\n\n%s",
354+
refPath,
355+
cmp.Diff(existing, refTarget),
356+
)
352357
return nil
353358
}
354359
}
@@ -1108,31 +1113,46 @@ func transformFleetPaths(schema *Schema) {
11081113
epmPoliciesPath.Get.CreateRef(schema, "package_policy", "responses.200.content.application/json.schema.properties.items.items")
11091114
epmPoliciesPath.Post.CreateRef(schema, "package_policy", "responses.200.content.application/json.schema.properties.item")
11101115

1111-
epmPoliciesPath.Post.Move("requestBody.content.application/json.schema.anyOf.1", "requestBody.content.application/json.schema") // anyOf.0 is the deprecated array format
1112-
epmPolicyPath.Put.Move("requestBody.content.application/json.schema.anyOf.1", "requestBody.content.application/json.schema") // anyOf.0 is the deprecated array format
1113-
epmPoliciesPath.Post.CreateRef(schema, "package_policy_request", "requestBody.content.application/json.schema")
1116+
epmPolicyPath.Put.CreateRef(schema, "package_policy_request_typed_inputs", "requestBody.content.application/json.schema.anyOf.0")
1117+
epmPolicyPath.Put.CreateRef(schema, "package_policy_request_mapped_inputs", "requestBody.content.application/json.schema.anyOf.1")
11141118
epmPolicyPath.Put.CreateRef(schema, "package_policy_request", "requestBody.content.application/json.schema")
11151119

1120+
epmPoliciesPath.Post.Set("requestBody.content.application/json.schema", epmPolicyPath.Put.MustGetMap("requestBody.content.application/json.schema"))
1121+
11161122
epmPolicyPath.Get.CreateRef(schema, "package_policy", "responses.200.content.application/json.schema.properties.item")
11171123
epmPolicyPath.Put.CreateRef(schema, "package_policy", "responses.200.content.application/json.schema.properties.item")
11181124

11191125
schema.Components.CreateRef(schema, "package_policy_secret_ref", "schemas.package_policy.properties.secret_references.items")
1120-
schema.Components.Move("schemas.package_policy.properties.inputs.anyOf.1", "schemas.package_policy.properties.inputs") // anyOf.0 is the deprecated array format
1126+
schema.Components.CreateRef(schema, "package_policy_typed_inputs", "schemas.package_policy.properties.inputs.anyOf.0")
1127+
schema.Components.CreateRef(schema, "package_policy_mapped_inputs", "schemas.package_policy.properties.inputs.anyOf.1")
1128+
schema.Components.CreateRef(schema, "package_policy_typed_input", "schemas.package_policy_typed_inputs.items")
1129+
schema.Components.CreateRef(schema, "package_policy_mapped_input", "schemas.package_policy_mapped_inputs.additionalProperties")
1130+
schema.Components.CreateRef(schema, "package_policy_typed_input_stream", "schemas.package_policy_typed_input.properties.streams.items")
1131+
schema.Components.CreateRef(schema, "package_policy_mapped_input_stream", "schemas.package_policy_mapped_input.properties.streams.additionalProperties")
1132+
1133+
schema.Components.CreateRef(schema, "package_policy_request_package", "schemas.package_policy_request_mapped_inputs.properties.package")
1134+
schema.Components.CreateRef(schema, "package_policy_request_package", "schemas.package_policy_request_typed_inputs.properties.package")
11211135

1122-
schema.Components.CreateRef(schema, "package_policy_input", "schemas.package_policy.properties.inputs.additionalProperties")
1123-
schema.Components.CreateRef(schema, "package_policy_input_stream", "schemas.package_policy_input.properties.streams.additionalProperties")
1136+
schema.Components.CreateRef(schema, "package_policy_request_mapped_input", "schemas.package_policy_request_mapped_inputs.properties.inputs.additionalProperties")
1137+
schema.Components.CreateRef(schema, "package_policy_request_mapped_input_stream", "schemas.package_policy_request_mapped_input.properties.streams.additionalProperties")
11241138

1125-
schema.Components.CreateRef(schema, "package_policy_request_package", "schemas.package_policy_request.properties.package")
1126-
schema.Components.CreateRef(schema, "package_policy_request_input", "schemas.package_policy_request.properties.inputs.additionalProperties")
1127-
schema.Components.CreateRef(schema, "package_policy_request_input_stream", "schemas.package_policy_request_input.properties.streams.additionalProperties")
1139+
schema.Components.CreateRef(schema, "package_policy_request_typed_input", "schemas.package_policy_request_typed_inputs.properties.inputs.items")
1140+
schema.Components.CreateRef(schema, "package_policy_request_typed_input_stream", "schemas.package_policy_request_typed_input.properties.streams.items")
11281141

11291142
// Simplify all of the vars
11301143
schema.Components.Set("schemas.package_policy.properties.vars", Map{"type": "object"})
1131-
schema.Components.Set("schemas.package_policy_input.properties.vars", Map{"type": "object"})
1132-
schema.Components.Set("schemas.package_policy_input_stream.properties.vars", Map{"type": "object"})
1133-
schema.Components.Set("schemas.package_policy_request.properties.vars", Map{"type": "object"})
1134-
schema.Components.Set("schemas.package_policy_request_input.properties.vars", Map{"type": "object"})
1135-
schema.Components.Set("schemas.package_policy_request_input_stream.properties.vars", Map{"type": "object"})
1144+
schema.Components.Set("schemas.package_policy_typed_input.properties.vars", Map{"type": "object"})
1145+
schema.Components.Set("schemas.package_policy_mapped_input.properties.vars", Map{"type": "object"})
1146+
schema.Components.Set("schemas.package_policy_typed_input_stream.properties.vars", Map{"type": "object"})
1147+
schema.Components.Set("schemas.package_policy_mapped_input_stream.properties.vars", Map{"type": "object"})
1148+
schema.Components.Set("schemas.package_policy_request_mapped_inputs.properties.vars", Map{"type": "object"})
1149+
schema.Components.Set("schemas.package_policy_request_mapped_input.properties.vars", Map{"type": "object"})
1150+
schema.Components.Set("schemas.package_policy_request_mapped_input_stream.properties.vars", Map{"type": "object"})
1151+
schema.Components.Set("schemas.package_policy_request_typed_inputs.properties.vars", Map{"type": "object"})
1152+
schema.Components.Set("schemas.package_policy_request_typed_input.properties.vars", Map{"type": "object"})
1153+
schema.Components.Set("schemas.package_policy_request_typed_input.properties.config", Map{"type": "object"})
1154+
schema.Components.Set("schemas.package_policy_request_typed_input_stream.properties.vars", Map{"type": "object"})
1155+
schema.Components.Set("schemas.package_policy_request_typed_input_stream.properties.config", Map{"type": "object"})
11361156
}
11371157

11381158
func setAllXOmitEmpty(key string, node Map) {

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/disaster37/go-kibana-rest/v8 v8.5.0
77
github.com/elastic/elastic-transport-go/v8 v8.8.0
88
github.com/elastic/go-elasticsearch/v8 v8.19.0
9+
github.com/google/go-cmp v0.7.0
910
github.com/google/gofuzz v1.2.0
1011
github.com/google/uuid v1.6.0
1112
github.com/hashicorp/go-cty v1.5.0
@@ -25,6 +26,7 @@ require (
2526
github.com/oapi-codegen/runtime v1.1.2
2627
github.com/stretchr/testify v1.11.1
2728
go.uber.org/mock v0.6.0
29+
gopkg.in/yaml.v3 v3.0.1
2830
)
2931

3032
require (
@@ -205,7 +207,6 @@ require (
205207
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
206208
github.com/golang/protobuf v1.5.4 // indirect
207209
github.com/google/certificate-transparency-go v1.3.1 // indirect
208-
github.com/google/go-cmp v0.7.0 // indirect
209210
github.com/google/go-containerregistry v0.20.6 // indirect
210211
github.com/google/go-github/v74 v74.0.0 // indirect
211212
github.com/google/go-querystring v1.1.0 // indirect
@@ -409,7 +410,6 @@ require (
409410
gopkg.in/mail.v2 v2.3.1 // indirect
410411
gopkg.in/warnings.v0 v0.1.2 // indirect
411412
gopkg.in/yaml.v2 v2.4.0 // indirect
412-
gopkg.in/yaml.v3 v3.0.1 // indirect
413413
gotest.tools/gotestsum v1.13.0 // indirect
414414
k8s.io/klog/v2 v2.130.1 // indirect
415415
lukechampine.com/blake3 v1.2.1 // indirect

‎internal/fleet/integration_policy/models.go‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -107,13 +107,20 @@ func (model *integrationPolicyModel) populateFromAPI(ctx context.Context, data *
107107
model.SpaceIds = types.SetNull(types.StringType)
108108
}
109109
// If originally set but API didn't return it, keep the original value
110-
111-
model.populateInputFromAPI(ctx, data.Inputs, &diags)
110+
mappedInputs, err := data.Inputs.AsPackagePolicyMappedInputs()
111+
if err != nil {
112+
diags.AddError(
113+
"Error reading integration policy inputs",
114+
"Could not parse integration policy inputs from API response: "+err.Error(),
115+
)
116+
return diags
117+
}
118+
model.populateInputFromAPI(ctx, mappedInputs, &diags)
112119

113120
return diags
114121
}
115122

116-
func (model *integrationPolicyModel) populateInputFromAPI(ctx context.Context, inputs map[string]kbapi.PackagePolicyInput, diags *diag.Diagnostics) {
123+
func (model *integrationPolicyModel) populateInputFromAPI(ctx context.Context, inputs map[string]kbapi.PackagePolicyMappedInput, diags *diag.Diagnostics) {
117124
// Handle input population based on context:
118125
// 1. If model.Input is unknown: we're importing or reading fresh state → populate from API
119126
// 2. If model.Input is known and null/empty: user explicitly didn't configure inputs → don't populate (avoid inconsistent state)
@@ -135,7 +142,7 @@ func (model *integrationPolicyModel) populateInputFromAPI(ctx context.Context, i
135142
// Case 3: Known and not null/empty - user configured inputs, populate from API (continue below)
136143

137144
newInputs := utils.TransformMapToSlice(ctx, inputs, path.Root("input"), diags,
138-
func(inputData kbapi.PackagePolicyInput, meta utils.MapMeta) integrationPolicyInputModel {
145+
func(inputData kbapi.PackagePolicyMappedInput, meta utils.MapMeta) integrationPolicyInputModel {
139146
return integrationPolicyInputModel{
140147
InputID: types.StringValue(meta.Key),
141148
Enabled: types.BoolPointerValue(inputData.Enabled),
@@ -186,7 +193,7 @@ func (model integrationPolicyModel) toAPIModel(ctx context.Context, isUpdate boo
186193
}
187194
}
188195

189-
body := kbapi.PackagePolicyRequest{
196+
body := kbapi.PackagePolicyRequestMappedInputs{
190197
Description: model.Description.ValueStringPointer(),
191198
Force: model.Force.ValueBoolPointer(),
192199
Name: model.Name.ValueString(),
@@ -219,17 +226,27 @@ func (model integrationPolicyModel) toAPIModel(ctx context.Context, isUpdate boo
219226
}
220227

221228
body.Inputs = utils.MapRef(utils.ListTypeToMap(ctx, model.Input, path.Root("input"), &diags,
222-
func(inputModel integrationPolicyInputModel, meta utils.ListMeta) (string, kbapi.PackagePolicyRequestInput) {
223-
return inputModel.InputID.ValueString(), kbapi.PackagePolicyRequestInput{
229+
func(inputModel integrationPolicyInputModel, meta utils.ListMeta) (string, kbapi.PackagePolicyRequestMappedInput) {
230+
return inputModel.InputID.ValueString(), kbapi.PackagePolicyRequestMappedInput{
224231
Enabled: inputModel.Enabled.ValueBoolPointer(),
225-
Streams: utils.MapRef(utils.NormalizedTypeToMap[kbapi.PackagePolicyRequestInputStream](inputModel.StreamsJson, meta.Path.AtName("streams_json"), &diags)),
232+
Streams: utils.MapRef(utils.NormalizedTypeToMap[kbapi.PackagePolicyRequestMappedInputStream](inputModel.StreamsJson, meta.Path.AtName("streams_json"), &diags)),
226233
Vars: utils.MapRef(utils.NormalizedTypeToMap[any](inputModel.VarsJson, meta.Path.AtName("vars_json"), &diags)),
227234
}
228235
}))
229236

230237
// Note: space_ids is read-only for integration policies and inherited from the agent policy
231238

232-
return body, diags
239+
var req kbapi.PackagePolicyRequest
240+
err := req.FromPackagePolicyRequestMappedInputs(body)
241+
if err != nil {
242+
diags.AddError(
243+
"Error constructing integration policy request",
244+
"Could not convert integration policy to API request: "+err.Error(),
245+
)
246+
return kbapi.PackagePolicyRequest{}, diags
247+
}
248+
249+
return req, diags
233250
}
234251

235252
// sortInputs will sort the 'incoming' list of input definitions based on

‎internal/fleet/integration_policy/models_test.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ func TestOutputIdHandling(t *testing.T) {
9595
OutputId: &outputId,
9696
}
9797

98+
// Initialize inputs union to an empty mapped set to avoid JSON parse errors
99+
_ = data.Inputs.FromPackagePolicyMappedInputs(map[string]kbapi.PackagePolicyMappedInput{})
100+
98101
diags := model.populateFromAPI(context.Background(), data)
99102
require.Empty(t, diags)
100103
require.Equal(t, "test-output-id", model.OutputID.ValueString())
@@ -115,8 +118,11 @@ func TestOutputIdHandling(t *testing.T) {
115118

116119
result, diags := model.toAPIModel(context.Background(), false, feat)
117120
require.Empty(t, diags)
118-
require.NotNil(t, result.OutputId)
119-
require.Equal(t, "test-output-id", *result.OutputId)
121+
122+
mappedResult, err := result.AsPackagePolicyRequestMappedInputs()
123+
require.NoError(t, err)
124+
require.NotNil(t, mappedResult.OutputId)
125+
require.Equal(t, "test-output-id", *mappedResult.OutputId)
120126
})
121127

122128
t.Run("toAPIModel_unsupported_version", func(t *testing.T) {

‎internal/fleet/integration_policy/secrets.go‎

Lines changed: 75 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package integration_policy
33
import (
44
"context"
55
"encoding/json"
6+
"maps"
67

78
"github.com/elastic/terraform-provider-elasticstack/generated/kbapi"
89
"github.com/elastic/terraform-provider-elasticstack/internal/utils"
@@ -113,15 +114,46 @@ func HandleRespSecrets(ctx context.Context, resp *kbapi.PackagePolicy, private p
113114
}
114115

115116
handleVars(utils.Deref(resp.Vars))
116-
for _, input := range resp.Inputs {
117-
handleVars(utils.Deref(input.Vars))
118-
for _, stream := range utils.Deref(input.Streams) {
119-
handleVars(utils.Deref(stream.Vars))
117+
118+
// Mapped inputs: extract, mutate, and write back
119+
mappedInputs, _ := resp.Inputs.AsPackagePolicyMappedInputs()
120+
typedInputs, _ := resp.Inputs.AsPackagePolicyTypedInputs()
121+
if mappedInputs != nil {
122+
for _, input := range mappedInputs {
123+
if input.Vars != nil {
124+
handleVars(*input.Vars)
125+
}
126+
127+
if input.Streams != nil {
128+
for _, stream := range *input.Streams {
129+
if stream.Vars != nil {
130+
handleVars(*stream.Vars)
131+
}
132+
}
133+
}
134+
}
135+
// Write back the mutated data
136+
if err := resp.Inputs.FromPackagePolicyMappedInputs(mappedInputs); err != nil {
137+
diags.AddError("could not write back mapped inputs", err.Error())
138+
}
139+
} else if typedInputs != nil {
140+
// Typed inputs: mutate in place through pointers
141+
for i := range typedInputs {
142+
if typedInputs[i].Vars != nil {
143+
handleVars(*typedInputs[i].Vars)
144+
}
145+
for j := range typedInputs[i].Streams {
146+
if typedInputs[i].Streams[j].Vars != nil {
147+
handleVars(*typedInputs[i].Streams[j].Vars)
148+
}
149+
}
150+
}
151+
if err := resp.Inputs.FromPackagePolicyTypedInputs(typedInputs); err != nil {
152+
diags.AddError("could not write back typed inputs", err.Error())
120153
}
121154
}
122155

123-
nd = secrets.Save(ctx, private)
124-
diags.Append(nd...)
156+
diags.Append(secrets.Save(ctx, private)...)
125157

126158
return
127159
}
@@ -188,19 +220,46 @@ func HandleReqRespSecrets(ctx context.Context, req kbapi.PackagePolicyRequest, r
188220
}
189221
}
190222

191-
handleVars(utils.Deref(req.Vars), utils.Deref(resp.Vars))
192-
for inputID, inputReq := range utils.Deref(req.Inputs) {
193-
inputResp := resp.Inputs[inputID]
194-
handleVars(utils.Deref(inputReq.Vars), utils.Deref(inputResp.Vars))
195-
streamsResp := utils.Deref(inputResp.Streams)
196-
for streamID, streamReq := range utils.Deref(inputReq.Streams) {
197-
streamResp := streamsResp[streamID]
198-
handleVars(utils.Deref(streamReq.Vars), utils.Deref(streamResp.Vars))
223+
mappedReq, _ := req.AsPackagePolicyRequestMappedInputs()
224+
typedReq, _ := req.AsPackagePolicyRequestTypedInputs()
225+
226+
reqVars := map[string]any{}
227+
maps.Copy(reqVars, utils.Deref(mappedReq.Vars))
228+
maps.Copy(reqVars, utils.Deref(typedReq.Vars))
229+
// Policy-level vars
230+
if resp.Vars != nil {
231+
handleVars(reqVars, utils.Deref(resp.Vars))
232+
}
233+
234+
respMappedInputs, _ := resp.Inputs.AsPackagePolicyMappedInputs()
235+
respTypedInputs, _ := resp.Inputs.AsPackagePolicyTypedInputs()
236+
if respMappedInputs != nil {
237+
for inputID, inputReq := range utils.Deref(mappedReq.Inputs) {
238+
inputResp := respMappedInputs[inputID]
239+
handleVars(utils.Deref(inputReq.Vars), utils.Deref(inputResp.Vars))
240+
241+
streamsResp := utils.Deref(inputResp.Streams)
242+
for streamID, streamReq := range utils.Deref(inputReq.Streams) {
243+
handleVars(utils.Deref(streamReq.Vars), utils.Deref(streamsResp[streamID].Vars))
244+
}
245+
}
246+
if err := resp.Inputs.FromPackagePolicyMappedInputs(respMappedInputs); err != nil {
247+
diags.AddError("failed to update mapped inputs", err.Error())
248+
}
249+
} else if respTypedInputs != nil {
250+
for inputIdx, inputReq := range utils.Deref(typedReq.Inputs) {
251+
inputResp := respTypedInputs[inputIdx]
252+
handleVars(utils.Deref(inputReq.Vars), utils.Deref(inputResp.Vars))
253+
for streamIdx, streamReq := range utils.Deref(inputReq.Streams) {
254+
handleVars(utils.Deref(streamReq.Vars), utils.Deref(inputResp.Streams[streamIdx].Vars))
255+
}
256+
}
257+
if err := resp.Inputs.FromPackagePolicyTypedInputs(respTypedInputs); err != nil {
258+
diags.AddError("failed to update typed inputs", err.Error())
199259
}
200260
}
201261

202-
nd = secrets.Save(ctx, private)
203-
diags.Append(nd...)
262+
diags.Append(secrets.Save(ctx, private)...)
204263

205264
return
206265
}

0 commit comments

Comments
 (0)