Unify properties to map conversion code

In order to do validation against the JSON schema the native properties.Map format of the Arduino project configuration
files must be converted to the map[string]interface{} type. This process is made more complex by the need to selectively
recurse the data into multiple key levels.

Previously, this was done in three different places in the code, each
tailored to the specific needs of the configuration files it worked with, and using slightly different approaches.

Unifying all that code into a single general purpose function will make the code easier to maintain.

Author: per1234

internal/project/general/general.go

@@ -21,23 +21,33 @@ import (
 )
 
 /*
-PropertiesToFirstLevelExpandedMap converts properties.Map data structures to map[string]interface that maps between .
-Even though boards/properties.txt have a multi-level nested data structure, the format has the odd characteristic of
-allowing a key to be both an object and a string simultaneously, which is not compatible with Golang maps or JSON. So
-the data structure used is a map of the first level keys (necessary to accommodate the board/prograrmmer IDs) to the
-full remainder of the keys (rather than recursing through each key level individually), to string values.
+PropertiesToMap converts properties.Map data structures to map[string]interface with the specified number of key levels.
+The Arduino project configuration fields have an odd usage of the properties.Map format. Dots may sometimes indicate
+nested keys, but in other cases they are merely a character in the key string. There are cases where a completely
+programmatic recursion of the properties into a fully nested structure would result in the impossibility of some keys
+having bot a string and a map type, which is not supported. For this reason, it's necessary to manually configure the
+recursion of key levels on a case-by-case basis.
+In the event a full recursion of key levels is desired, set the levels argument to a value <1.
 */
-func PropertiesToFirstLevelExpandedMap(flatProperties *properties.Map) map[string]interface{} {
+func PropertiesToMap(flatProperties *properties.Map, levels int) map[string]interface{} {
 	propertiesInterface := make(map[string]interface{})
-	keys := flatProperties.FirstLevelKeys()
+
+	var keys []string
+	if levels != 1 {
+		keys = flatProperties.FirstLevelKeys()
+	} else {
+		keys = flatProperties.Keys()
+	}
+
 	for _, key := range keys {
-		subtreeMap := flatProperties.SubTree(key).AsMap()
-		// This level also must be converted to map[string]interface{}.
-		subtreeInterface := make(map[string]interface{})
-		for subtreeKey, subtreeValue := range subtreeMap {
-			subtreeInterface[subtreeKey] = subtreeValue
+		subTree := flatProperties.SubTree(key)
+		if subTree.Size() > 0 {
+			// This key contains a map.
+			propertiesInterface[key] = PropertiesToMap(subTree, levels-1)
+		} else {
+			// This key contains a string, no more recursion is possible.
+			propertiesInterface[key] = flatProperties.Get(key)
 		}
-		propertiesInterface[key] = subtreeInterface
 	}
 
 	return propertiesInterface

internal/project/general/general_test.go

@@ -24,8 +24,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestPropertiesToFirstLevelExpandedMap(t *testing.T) {
+func TestPropertiesToMap(t *testing.T) {
 	rawProperties := []byte(`
+		hello=world
+		goodbye=
 		foo.bar=asdf
 		foo.baz=zxcv
 		bar.bat.bam=123
@@ -34,6 +36,18 @@ func TestPropertiesToFirstLevelExpandedMap(t *testing.T) {
 	require.Nil(t, err)
 
 	expectedMapOutput := map[string]interface{}{
+		"hello":       "world",
+		"goodbye":     "",
+		"foo.bar":     "asdf",
+		"foo.baz":     "zxcv",
+		"bar.bat.bam": "123",
+	}
+
+	assert.True(t, reflect.DeepEqual(expectedMapOutput, PropertiesToMap(propertiesInput, 1)))
+
+	expectedMapOutput = map[string]interface{}{
+		"hello":   "world",
+		"goodbye": "",
 		"foo": map[string]interface{}{
 			"bar": "asdf",
 			"baz": "zxcv",
@@ -43,5 +57,22 @@ func TestPropertiesToFirstLevelExpandedMap(t *testing.T) {
 		},
 	}
 
-	assert.True(t, reflect.DeepEqual(expectedMapOutput, PropertiesToFirstLevelExpandedMap(propertiesInput)))
+	assert.True(t, reflect.DeepEqual(expectedMapOutput, PropertiesToMap(propertiesInput, 2)))
+
+	expectedMapOutput = map[string]interface{}{
+		"hello":   "world",
+		"goodbye": "",
+		"foo": map[string]interface{}{
+			"bar": "asdf",
+			"baz": "zxcv",
+		},
+		"bar": map[string]interface{}{
+			"bat": map[string]interface{}{
+				"bam": "123",
+			},
+		},
+	}
+
+	assert.True(t, reflect.DeepEqual(expectedMapOutput, PropertiesToMap(propertiesInput, 3)))
+	assert.True(t, reflect.DeepEqual(expectedMapOutput, PropertiesToMap(propertiesInput, 0)))
 }

internal/project/library/libraryproperties/libraryproperties.go

@@ -17,6 +17,7 @@
 package libraryproperties
 
 import (
+	"github.com/arduino/arduino-lint/internal/project/general"
 	"github.com/arduino/arduino-lint/internal/rule/schema"
 	"github.com/arduino/arduino-lint/internal/rule/schema/compliancelevel"
 	"github.com/arduino/arduino-lint/internal/rule/schema/schemadata"
@@ -48,11 +49,7 @@ func Validate(libraryProperties *properties.Map) map[compliancelevel.Type]schema
 
 	// Convert the library.properties data from the native properties.Map type to the interface type required by the schema
 	// validation package.
-	libraryPropertiesMap := libraryProperties.AsMap()
-	libraryPropertiesInterface := make(map[string]interface{}, len(libraryPropertiesMap))
-	for k, v := range libraryPropertiesMap {
-		libraryPropertiesInterface[k] = v
-	}
+	libraryPropertiesInterface := general.PropertiesToMap(libraryProperties, 1)
 
 	validationResults[compliancelevel.Permissive] = schema.Validate(libraryPropertiesInterface, schemaObject[compliancelevel.Permissive])
 	validationResults[compliancelevel.Specification] = schema.Validate(libraryPropertiesInterface, schemaObject[compliancelevel.Specification])

internal/project/platform/boardstxt/boardstxt.go

@@ -51,7 +51,7 @@ func Validate(boardsTxt *properties.Map) map[compliancelevel.Type]schema.Validat
 	}
 
 	//Convert the boards.txt data from the native properties.Map type to the interface type required by the schema validation package.
-	boardsTxtInterface := general.PropertiesToFirstLevelExpandedMap(boardsTxt)
+	boardsTxtInterface := general.PropertiesToMap(boardsTxt, 2)
 
 	validationResults[compliancelevel.Permissive] = schema.Validate(boardsTxtInterface, schemaObject[compliancelevel.Permissive])
 	validationResults[compliancelevel.Specification] = schema.Validate(boardsTxtInterface, schemaObject[compliancelevel.Specification])

internal/project/platform/programmerstxt/programmerstxt.go

@@ -50,7 +50,7 @@ func Validate(programmersTxt *properties.Map) map[compliancelevel.Type]schema.Va
 	}
 
 	//Convert the programmers.txt data from the native properties.Map type to the interface type required by the schema validation package.
-	programmersTxtInterface := general.PropertiesToFirstLevelExpandedMap(programmersTxt)
+	programmersTxtInterface := general.PropertiesToMap(programmersTxt, 2)
 
 	validationResults[compliancelevel.Permissive] = schema.Validate(programmersTxtInterface, schemaObject[compliancelevel.Permissive])
 	validationResults[compliancelevel.Specification] = schema.Validate(programmersTxtInterface, schemaObject[compliancelevel.Specification])

internal/rule/schema/schema_test.go

@@ -19,6 +19,7 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/arduino/arduino-lint/internal/project/general"
 	"github.com/arduino/arduino-lint/internal/rule/schema/testdata"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/ory/jsonschema/v3"
@@ -46,23 +47,6 @@ func init() {
 	)
 }
 
-// propertiesToMap converts properties.Map data structures to map[string]interface.
-func propertiesToMap(propertiesInput *properties.Map) map[string]interface{} {
-	mapOutput := make(map[string]interface{})
-	keys := propertiesInput.FirstLevelKeys()
-	for _, key := range keys {
-		subTree := propertiesInput.SubTree(key)
-		if subTree.Size() > 0 {
-			// This key contains a map, recurse it.
-			mapOutput[key] = propertiesToMap(subTree)
-		} else {
-			// This key contains a string, no more recursion is possible.
-			mapOutput[key] = propertiesInput.Get(key)
-		}
-	}
-	return mapOutput
-}
-
 func TestCompile(t *testing.T) {
 	require.Panics(t, func() {
 		Compile("valid-schema-with-references.json", []string{"nonexistent.json"}, testdata.Asset)
@@ -99,87 +83,87 @@ func TestCompile(t *testing.T) {
 func TestValidate(t *testing.T) {
 	schemaObject := Compile("valid-schema.json", []string{}, testdata.Asset)
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), schemaObject)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), schemaObject)
 	require.Nil(t, validationResult.Result)
 
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.Nil(t, validationResult.Result)
 
 	propertiesMap.Set("property1", "a")
-	validationResult = Validate(propertiesToMap(propertiesMap), schemaObject)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), schemaObject)
 	require.Equal(t, "#/property1", validationResult.Result.InstancePtr)
 	require.Equal(t, "#/properties/property1/minLength", validationResult.Result.SchemaPtr)
 }
 
 func TestRequiredPropertyMissing(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, RequiredPropertyMissing("property1", validationResult))
 
 	propertiesMap.Remove("property1")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, RequiredPropertyMissing("property1", validationResult))
 }
 
 func TestPropertyPatternMismatch(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, PropertyPatternMismatch("property2", validationResult))
 
 	propertiesMap.Set("property2", "fOo")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, PropertyPatternMismatch("property2", validationResult))
 
 	require.False(t, PropertyPatternMismatch("property1", validationResult))
 }
 
 func TestPropertyLessThanMinLength(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, PropertyLessThanMinLength("property1", validationResult))
 
 	propertiesMap.Set("property1", "a")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, PropertyLessThanMinLength("property1", validationResult))
 }
 
 func TestPropertyGreaterThanMaxLength(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, PropertyGreaterThanMaxLength("property1", validationResult))
 
 	propertiesMap.Set("property1", "12345")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, PropertyGreaterThanMaxLength("property1", validationResult))
 }
 
 func TestPropertyEnumMismatch(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, PropertyEnumMismatch("property3", validationResult))
 
 	propertiesMap.Set("property3", "invalid")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, PropertyEnumMismatch("property3", validationResult))
 }
 
 func TestMisspelledOptionalPropertyFound(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, MisspelledOptionalPropertyFound(validationResult))
 
 	propertiesMap.Set("porperties", "foo")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, MisspelledOptionalPropertyFound(validationResult))
 }
 
 func TestValidationErrorMatch(t *testing.T) {
 	propertiesMap := properties.NewFromHashmap(validMap)
-	validationResult := Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult := Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, ValidationErrorMatch("", "", "", "", validationResult))
 
 	propertiesMap.Set("property2", "fOo")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, ValidationErrorMatch("nomatch", "nomatch", "nomatch", "nomatch", validationResult))
 	require.False(t, ValidationErrorMatch("^#/property2$", "nomatch", "nomatch", "nomatch", validationResult))
 	require.False(t, ValidationErrorMatch("^#/property2$", "/pattern$", "nomatch", "nomatch", validationResult))
@@ -188,12 +172,12 @@ func TestValidationErrorMatch(t *testing.T) {
 	require.True(t, ValidationErrorMatch("", "", "", "", validationResult))
 
 	propertiesMap.Set("property3", "bAz")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.True(t, ValidationErrorMatch("^#/property3$", "/pattern$", "", "", validationResult), "Match pointer below logic inversion keyword")
 
 	propertiesMap = properties.NewFromHashmap(validMap)
 	propertiesMap.Remove("property1")
-	validationResult = Validate(propertiesToMap(propertiesMap), validSchemaWithReferences)
+	validationResult = Validate(general.PropertiesToMap(propertiesMap, 0), validSchemaWithReferences)
 	require.False(t, ValidationErrorMatch("nomatch", "nomatch", "nomatch", "nomatch", validationResult))
 	require.True(t, ValidationErrorMatch("", "", "", "^#/property1$", validationResult))
 }