Create custom list and map fields (#222)

Issue #, if available: aws-controllers-k8s/community#881

Description of changes:
Supports creating custom list and map fields where the member shapes of those fields already exist in the SDK.
Rather than supplying a `from` in the field config, you can now supply a `custom_field` property which requires
either `list_of` or `map_of` with the name of an API shape as the value.

An example of generating a list (for an S3 Bucket):
```yaml
resources:
  Bucket:
    fields:
      AnalyticsConfigurations:
        custom_field:
          list_of: AnalyticsConfiguration
```

Produces the following:
```golang
AnalyticsConfigurations []*AnalyticsConfiguration `json:"analyticsConfigurations,omitempty"`
```

An example of generating a map (for an S3 Bucket):
```yaml
resources:
  Bucket:
    fields:
      AnalyticsConfigurations:
        custom_field:
          map_of: AnalyticsConfiguration
```

Produces the following:
```golang
AnalyticsConfigurations map[string]*AnalyticsConfiguration `json:"analyticsConfigurations,omitempty"` 
```

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: RedbackThomson

cmd/ack-generate/command/common.go

@@ -237,7 +237,7 @@ func loadModel(svcAlias string, apiVersion string) (*ackmodel.Model, error) {
 		modelName = svcAlias
 	}
 
-	sdkHelper := acksdk.NewHelper(sdkDir)
+	sdkHelper := acksdk.NewHelper(sdkDir, cfg)
 	sdkAPI, err := sdkHelper.API(modelName)
 	if err != nil {
 		retryModelName, err := FallBackFindServiceID(sdkDir, svcAlias)

cmd/ack-generate/command/crossplane.go

@@ -70,7 +70,7 @@ func generateCrossplane(_ *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	sdkHelper := acksdk.NewHelper(sdkDir)
+	sdkHelper := acksdk.NewHelper(sdkDir, cfg)
 	sdkHelper.APIGroupSuffix = "aws.crossplane.io"
 	sdkAPI, err := sdkHelper.API(svcAlias)
 	if err != nil {

pkg/generate/config/config.go

@@ -74,6 +74,38 @@ type PrefixConfig struct {
 	StatusField string `json:"status_field,omitempty"`
 }
 
+// GetCustomListFieldMembers finds all of the custom list fields that need to
+// be generated as defined in the generator config.
+func (c *Config) GetCustomListFieldMembers() []string {
+	members := []string{}
+
+	for _, resource := range c.Resources {
+		for _, field := range resource.Fields {
+			if field.CustomField != nil && field.CustomField.ListOf != "" {
+				members = append(members, field.CustomField.ListOf)
+			}
+		}
+	}
+
+	return members
+}
+
+// GetCustomMapFieldMembers finds all of the custom map fields that need to be
+// generated as defined in the generator config.
+func (c *Config) GetCustomMapFieldMembers() []string {
+	members := []string{}
+
+	for _, resource := range c.Resources {
+		for _, field := range resource.Fields {
+			if field.CustomField != nil && field.CustomField.MapOf != "" {
+				members = append(members, field.CustomField.MapOf)
+			}
+		}
+	}
+
+	return members
+}
+
 // ResourceContainsSecret returns true if any of the fields in any resource are
 // defined as secrets.
 func (c *Config) ResourceContainsSecret() bool {

pkg/generate/config/field.go

@@ -136,6 +136,18 @@ type PrintFieldConfig struct {
 	Index int `json:"index"`
 }
 
+// CustomField instructs the code generator to create a new list or map field
+// type using a shape that exists in the SDK.
+type CustomFieldConfig struct {
+	// ListOf provides the name of the SDK shape which will become the
+	// member of a custom slice field.
+	ListOf string `json:"list_of,omitempty"`
+	// MapOf provides the name of the SDK shape which will become the value
+	// shape for a custom map field. All maps will have `string` as their key
+	// type.
+	MapOf string `json:"map_of,omitempty"`
+}
+
 // LateInitializeConfig contains instructions for how to handle the
 // retrieval and setting of server-side defaulted fields.
 // NOTE: Currently the members of this have no effect on late initialization of fields.
@@ -193,6 +205,9 @@ type FieldConfig struct {
 	// From instructs the code generator that the value of the field should
 	// be retrieved from the specified operation and member path
 	From *SourceFieldConfig `json:"from,omitempty"`
+	// CustomField instructs the code generator to create a new field that does
+	// not exist in the SDK.
+	CustomField *CustomFieldConfig `json:"custom_field,omitempty"`
 	// Compare instructs the code generator how to produce code that compares
 	// the value of the field in two resources
 	Compare *CompareFieldConfig `json:"compare,omitempty"`

pkg/model/model.go

@@ -137,25 +137,45 @@ func (m *Model) GetCRDs() ([]*CRD, error) {
 				// It's a Status field...
 				continue
 			}
-			if fieldConfig.From == nil {
-				// Isn't an additional Spec field...
-				continue
-			}
-			from := fieldConfig.From
-			memberShapeRef, found := m.SDKAPI.GetInputShapeRef(
-				from.Operation, from.Path,
-			)
-			if found {
-				memberNames := names.New(targetFieldName)
-				crd.AddSpecField(memberNames, memberShapeRef)
-			} else {
-				// This is a compile-time failure, just bomb out...
-				msg := fmt.Sprintf(
-					"unknown additional Spec field with Op: %s and Path: %s",
+
+			var found bool
+			var memberShapeRef *awssdkmodel.ShapeRef
+
+			if fieldConfig.From != nil {
+				from := fieldConfig.From
+				memberShapeRef, found = m.SDKAPI.GetInputShapeRef(
 					from.Operation, from.Path,
 				)
-				panic(msg)
+				if !found {
+					// This is a compile-time failure, just bomb out...
+					msg := fmt.Sprintf(
+						"unknown additional Spec field with Op: %s and Path: %s",
+						from.Operation, from.Path,
+					)
+					panic(msg)
+				}
+			} else if fieldConfig.CustomField != nil {
+				customField := fieldConfig.CustomField
+				if customField.ListOf != "" {
+					memberShapeRef = m.SDKAPI.GetCustomShapeRef(customField.ListOf)
+				} else {
+					memberShapeRef = m.SDKAPI.GetCustomShapeRef(customField.MapOf)
+				}
+				if memberShapeRef == nil {
+					// This is a compile-time failure, just bomb out...
+					msg := fmt.Sprintf(
+						"unknown additional Spec field with custom field %+v",
+						customField,
+					)
+					panic(msg)
+				}
+			} else {
+				// Spec field is not well defined
+				continue
 			}
+
+			memberNames := names.New(targetFieldName)
+			crd.AddSpecField(memberNames, memberShapeRef)
 		}
 
 		// Now process the fields that will go into the Status struct. We want
@@ -209,25 +229,45 @@ func (m *Model) GetCRDs() ([]*CRD, error) {
 				// It's a Spec field...
 				continue
 			}
-			if fieldConfig.From == nil {
-				// Isn't an additional Status field...
-				continue
-			}
-			from := fieldConfig.From
-			memberShapeRef, found := m.SDKAPI.GetOutputShapeRef(
-				from.Operation, from.Path,
-			)
-			if found {
-				memberNames := names.New(targetFieldName)
-				crd.AddStatusField(memberNames, memberShapeRef)
-			} else {
-				// This is a compile-time failure, just bomb out...
-				msg := fmt.Sprintf(
-					"unknown additional Status field with Op: %s and Path: %s",
+
+			var found bool
+			var memberShapeRef *awssdkmodel.ShapeRef
+
+			if fieldConfig.From != nil {
+				from := fieldConfig.From
+				memberShapeRef, found = m.SDKAPI.GetOutputShapeRef(
 					from.Operation, from.Path,
 				)
-				panic(msg)
+				if !found {
+					// This is a compile-time failure, just bomb out...
+					msg := fmt.Sprintf(
+						"unknown additional Status field with Op: %s and Path: %s",
+						from.Operation, from.Path,
+					)
+					panic(msg)
+				}
+			} else if fieldConfig.CustomField != nil {
+				customField := fieldConfig.CustomField
+				if customField.ListOf != "" {
+					memberShapeRef = m.SDKAPI.GetCustomShapeRef(customField.ListOf)
+				} else {
+					memberShapeRef = m.SDKAPI.GetCustomShapeRef(customField.MapOf)
+				}
+				if memberShapeRef == nil {
+					// This is a compile-time failure, just bomb out...
+					msg := fmt.Sprintf(
+						"unknown additional Status field with custom field %+v",
+						customField,
+					)
+					panic(msg)
+				}
+			} else {
+				// Status field is not well defined
+				continue
 			}
+
+			memberNames := names.New(targetFieldName)
+			crd.AddStatusField(memberNames, memberShapeRef)
 		}
 
 		crds = append(crds, crd)

pkg/model/multiversion/manager.go

@@ -72,8 +72,6 @@ func NewAPIVersionManager(
 		return nil, fmt.Errorf("cannot read sdk git repository: %v", err)
 	}
 
-	SDKAPIHelper := acksdk.NewHelper(sdkCacheDir)
-
 	// create model for each non-deprecated api version
 	models := map[string]*ackmodel.Model{}
 	for _, version := range metadata.APIVersions {
@@ -90,23 +88,24 @@ func NewAPIVersionManager(
 			return nil, fmt.Errorf("could not find API info for API version %s", version.APIVersion)
 		}
 
-		err = SDKAPIHelper.WithSDKVersion(apiInfo.AWSSDKVersion)
+		cfg, err := ackgenconfig.New(apiInfo.GeneratorConfigPath, defaultConfig)
 		if err != nil {
 			return nil, err
 		}
 
-		cfg, err := ackgenconfig.New(apiInfo.GeneratorConfigPath, defaultConfig)
+		sdkAPIHelper := acksdk.NewHelper(sdkCacheDir, cfg)
+		err = sdkAPIHelper.WithSDKVersion(apiInfo.AWSSDKVersion)
 		if err != nil {
 			return nil, err
 		}
 
-		SDKAPI, err := SDKAPIHelper.API(servicePackageName)
+		sdkAPI, err := sdkAPIHelper.API(servicePackageName)
 		if err != nil {
 			return nil, err
 		}
 
 		i, err := ackmodel.New(
-			SDKAPI,
+			sdkAPI,
 			servicePackageName,
 			version.APIVersion,
 			cfg,

pkg/model/sdk_api.go

@@ -34,6 +34,7 @@ const (
 type SDKAPI struct {
 	API            *awssdkmodel.API
 	APIGroupSuffix string
+	CustomShapes   []*CustomShape
 	// A map of operation type and resource name to
 	// aws-sdk-go/private/model/api.Operation structs
 	opMap *OperationMap
@@ -75,6 +76,36 @@ func (a *SDKAPI) GetOperationMap(cfg *ackgenconfig.Config) *OperationMap {
 	return &opMap
 }
 
+// GetCustomShapeRef finds a ShapeRef for a custom shape using either its member
+// or its value shape name.
+func (a *SDKAPI) GetCustomShapeRef(shapeName string) *awssdkmodel.ShapeRef {
+	customList := a.getCustomListRef(shapeName)
+	if customList != nil {
+		return customList
+	}
+	return a.getCustomMapRef(shapeName)
+}
+
+// getCustomListRef finds a ShapeRef for a supplied custom list field
+func (a *SDKAPI) getCustomListRef(memberShapeName string) *awssdkmodel.ShapeRef {
+	for _, shape := range a.CustomShapes {
+		if shape.MemberShapeName != nil && *shape.MemberShapeName == memberShapeName {
+			return shape.ShapeRef
+		}
+	}
+	return nil
+}
+
+// getCustomMapRef finds a ShapeRef for a supplied custom map field
+func (a *SDKAPI) getCustomMapRef(valueShapeName string) *awssdkmodel.ShapeRef {
+	for _, shape := range a.CustomShapes {
+		if shape.ValueShapeName != nil && *shape.ValueShapeName == valueShapeName {
+			return shape.ShapeRef
+		}
+	}
+	return nil
+}
+
 // GetInputShapeRef finds a ShapeRef for a supplied member path (dot-notation)
 // for given API operation
 func (a *SDKAPI) GetInputShapeRef(
@@ -267,3 +298,32 @@ func getMemberByPath(
 	}
 	return nil, false
 }
+
+// CustomShape represents a shape created by the generator that does not exist
+// in the standard AWS SDK models.
+type CustomShape struct {
+	Shape           *awssdkmodel.Shape
+	ShapeRef        *awssdkmodel.ShapeRef
+	MemberShapeName *string
+	ValueShapeName  *string
+}
+
+// NewCustomListShape creates a custom shape object for a new list.
+func NewCustomListShape(shape *awssdkmodel.Shape, ref *awssdkmodel.ShapeRef, memberShapeName string) *CustomShape {
+	return &CustomShape{
+		Shape:           shape,
+		ShapeRef:        ref,
+		MemberShapeName: &memberShapeName,
+		ValueShapeName:  nil,
+	}
+}
+
+// NewCustomMapShape creates a custom shape object for a new map.
+func NewCustomMapShape(shape *awssdkmodel.Shape, ref *awssdkmodel.ShapeRef, valueShapeName string) *CustomShape {
+	return &CustomShape{
+		Shape:           shape,
+		ShapeRef:        ref,
+		MemberShapeName: nil,
+		ValueShapeName:  &valueShapeName,
+	}
+}

pkg/model/types_test.go

@@ -3,9 +3,8 @@ package model_test
 import (
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/aws-controllers-k8s/code-generator/pkg/model"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestReplacePkgName(t *testing.T) {