Adding column index and line matching attributes (line_index, and line_pattern) to csv2 column decl and json validation schema (#175)

* Adding column index and line matching attributes (line_index, and line_pattern) to csv2 column decl and json validation schema

Turns out if we want to support multi-line record (whether it's fixed number of rows based or header/footer based) in csv
we will have to let each column to specify which line the column value should come from, and of that particular line, which
indexed value to use, thus the adding of `index`, `line_index` and `line_pattern`.

Author: jf-tech

extensions/omniv21/fileformat/flatfile/csv/.snapshots/TestValidateFileDecl_Success

@@ -0,0 +1,23 @@
+([]*csv.ColumnDecl) (len=3) {
+  (*csv.ColumnDecl)({
+    Name: (string) (len=2) "c1",
+    Index: (*int)(1),
+    LineIndex: (*int)(1),
+    LinePattern: (*string)(<nil>),
+    linePatternRegexp: (*regexp.Regexp)(<nil>)
+  }),
+  (*csv.ColumnDecl)({
+    Name: (string) (len=2) "c2",
+    Index: (*int)(3),
+    LineIndex: (*int)(<nil>),
+    LinePattern: (*string)(<nil>),
+    linePatternRegexp: (*regexp.Regexp)(<nil>)
+  }),
+  (*csv.ColumnDecl)({
+    Name: (string) (len=2) "c3",
+    Index: (*int)(4),
+    LineIndex: (*int)(<nil>),
+    LinePattern: (*string)((len=3) "^C$"),
+    linePatternRegexp: (*regexp.Regexp)(^C$)
+  })
+}

extensions/omniv21/fileformat/flatfile/csv/decl.go

@@ -11,12 +11,30 @@ import (
 
 // ColumnDecl describes a column of an csv record column.
 type ColumnDecl struct {
-	Name string `json:"name,omitempty"`
+	Name        string  `json:"name,omitempty"`
+	Index       *int    `json:"index,omitempty"`        // 1-based. optional.
+	LineIndex   *int    `json:"line_index,omitempty"`   // 1-based. optional
+	LinePattern *string `json:"line_pattern,omitempty"` // optional
+
+	linePatternRegexp *regexp.Regexp
 }
 
-// Design note: given currently ColumnDecl contains only Name field, we could've simply
-// change RecordDecl.Columns into a []string. But in the future, if we ever need to add
-// anything to a column decl, we'd have to introduce a breaking schema change.
+func (c *ColumnDecl) lineMatch(lineIndex int, line line) bool {
+	if c.LineIndex != nil {
+		return *c.LineIndex == lineIndex+1 // c.LineIndex is 1 based.
+	}
+	if c.linePatternRegexp != nil {
+		return c.linePatternRegexp.MatchString(line.raw)
+	}
+	return true
+}
+
+func (c *ColumnDecl) lineToColumnValue(line line) string {
+	if *c.Index < 1 || *c.Index > len(line.record) {
+		return ""
+	}
+	return line.record[*c.Index-1]
+}
 
 const (
 	typeRecord = "record"

extensions/omniv21/fileformat/flatfile/csv/decl_test.go

@@ -11,6 +11,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestColumnDecl_LineMatch(t *testing.T) {
+	assert.True(t, (&ColumnDecl{}).lineMatch(0, line{}))
+	assert.False(t, (&ColumnDecl{LineIndex: testlib.IntPtr(2)}).lineMatch(0, line{}))
+	assert.True(t, (&ColumnDecl{LineIndex: testlib.IntPtr(2)}).lineMatch(1, line{}))
+	assert.False(t, (&ColumnDecl{linePatternRegexp: regexp.MustCompile("^ABC.*$")}).
+		lineMatch(0, line{raw: "1234567"}))
+	assert.True(t, (&ColumnDecl{linePatternRegexp: regexp.MustCompile("^ABC.*$")}).
+		lineMatch(0, line{raw: "ABCDEFG"}))
+}
+
+func TestColumnDecl_LineToColumnValue(t *testing.T) {
+	assert.Equal(t, "", (&ColumnDecl{Index: testlib.IntPtr(2)}).lineToColumnValue(
+		line{record: []string{"1"}})) // index out of range
+	assert.Equal(t, "", (&ColumnDecl{Index: testlib.IntPtr(0)}).lineToColumnValue(
+		line{record: []string{"1"}})) // index out of range
+	assert.Equal(t, "9", (&ColumnDecl{Index: testlib.IntPtr(5)}).lineToColumnValue(
+		line{record: []string{"1", "3", "5", "7", "9", "11"}})) // in range
+}
+
 func TestRecordDecl(t *testing.T) {
 	// DeclName()
 	r := &RecordDecl{Name: "r1"}

extensions/omniv21/fileformat/flatfile/csv/line.go

@@ -0,0 +1,7 @@
+package csv
+
+type line struct {
+	lineNum int // 1-based
+	record  []string
+	raw     string
+}

extensions/omniv21/fileformat/flatfile/csv/validate.go

@@ -62,6 +62,15 @@ func (ctx *validateCtx) validateRecordDecl(fqdn string, decl *RecordDecl) (err e
 		return fmt.Errorf("record/record_group '%s' has 'min' value %d > 'max' value %d",
 			fqdn, decl.MinOccurs(), decl.MaxOccurs())
 	}
+	for i, col := range decl.Columns {
+		prevCol := (*ColumnDecl)(nil)
+		if i > 0 {
+			prevCol = decl.Columns[i-1]
+		}
+		if err = ctx.validateColumnDecl(fqdn, prevCol, col); err != nil {
+			return err
+		}
+	}
 	for _, c := range decl.Children {
 		if err = ctx.validateRecordDecl(strs.BuildFQDN2("/", fqdn, c.Name), c); err != nil {
 			return err
@@ -70,3 +79,33 @@ func (ctx *validateCtx) validateRecordDecl(fqdn string, decl *RecordDecl) (err e
 	decl.childRecDecls = toFlatFileRecDecls(decl.Children)
 	return nil
 }
+
+func intPtr(v int) *int {
+	return &v
+}
+
+func (ctx *validateCtx) validateColumnDecl(fqdn string, prevDecl, decl *ColumnDecl) (err error) {
+	// If column.index not specified, then we'll use the previous column's index value + 1 unless
+	// the column is the first column, then 1 will be used.
+	// if column.index is explicitly specified, it will be honored.
+	if decl.Index == nil {
+		if prevDecl == nil {
+			decl.Index = intPtr(1)
+		} else {
+			decl.Index = intPtr(*prevDecl.Index + 1)
+		}
+	}
+	if decl.LineIndex != nil && decl.LinePattern != nil {
+		return fmt.Errorf(
+			"record '%s' column '%s' cannot have both `line_index` and `line_pattern` specified at the same time",
+			fqdn, decl.Name)
+	}
+	if decl.LinePattern != nil {
+		if decl.linePatternRegexp, err = caches.GetRegex(*decl.LinePattern); err != nil {
+			return fmt.Errorf(
+				"record '%s' column '%s' has an invalid 'line_pattern' regexp '%s': %s",
+				fqdn, decl.Name, *decl.LinePattern, err.Error())
+		}
+	}
+	return nil
+}

extensions/omniv21/fileformat/flatfile/csv/validate_test.go

@@ -3,6 +3,7 @@ package csv
 import (
 	"testing"
 
+	"github.com/bradleyjkemp/cupaloy"
 	"github.com/jf-tech/go-corelib/strs"
 	"github.com/jf-tech/go-corelib/testlib"
 	"github.com/stretchr/testify/assert"
@@ -99,10 +100,36 @@ func TestValidateFileDecl_MinGreaterThanMax(t *testing.T) {
 	assert.Equal(t, `record/record_group 'A/B' has 'min' value 2 > 'max' value 1`, err.Error())
 }
 
+func TestValidateFileDecl_ColumnLineIndexAndLinePatternSameTime(t *testing.T) {
+	err := (&validateCtx{}).validateFileDecl(&FileDecl{
+		Records: []*RecordDecl{
+			{Name: "A", Columns: []*ColumnDecl{
+				{Name: "c", LineIndex: testlib.IntPtr(2), LinePattern: strs.StrPtr(".")}}},
+		},
+	})
+	assert.Error(t, err)
+	assert.Equal(t,
+		"record 'A' column 'c' cannot have both `line_index` and `line_pattern` specified at the same time",
+		err.Error())
+}
+
+func TestValidateFileDecl_InvalidColumnLinePattern(t *testing.T) {
+	err := (&validateCtx{}).validateFileDecl(&FileDecl{
+		Records: []*RecordDecl{
+			{Name: "A", Columns: []*ColumnDecl{
+				{Name: "c", LinePattern: strs.StrPtr("[invalid")}}},
+		},
+	})
+	assert.Error(t, err)
+	assert.Equal(t,
+		"record 'A' column 'c' has an invalid 'line_pattern' regexp '[invalid': error parsing regexp: missing closing ]: `[invalid`",
+		err.Error())
+}
+
 func TestValidateFileDecl_Success(t *testing.T) {
-	col1 := &ColumnDecl{Name: "c1"}
-	col2 := &ColumnDecl{Name: "c2"}
-	col3 := &ColumnDecl{Name: "c3"}
+	col1 := &ColumnDecl{Name: "c1", LineIndex: testlib.IntPtr(1)}
+	col2 := &ColumnDecl{Name: "c2", Index: testlib.IntPtr(3)}
+	col3 := &ColumnDecl{Name: "c3", LinePattern: strs.StrPtr("^C$")}
 	fd := &FileDecl{
 		Records: []*RecordDecl{
 			{
@@ -126,5 +153,5 @@ func TestValidateFileDecl_Success(t *testing.T) {
 	assert.Equal(t, 1, len(fd.Records[0].childRecDecls))
 	assert.Same(t, fd.Records[0].Children[0], fd.Records[0].childRecDecls[0].(*RecordDecl))
 	assert.Equal(t, "A/B", fd.Records[0].Children[0].fqdn)
-	assert.Equal(t, []*ColumnDecl{col1, col2, col3}, fd.Records[0].Children[0].Columns)
+	cupaloy.SnapshotT(t, fd.Records[0].Children[0].Columns)
 }

extensions/omniv21/validation/csv2FileDeclaration.go

@@ -77,7 +77,10 @@
             "items": {
                 "type": "object",
                 "properties": {
-                    "name": { "type": "string", "minLength": 1 }
+                    "name": { "type": "string", "minLength": 1 },
+                    "index": { "type": "integer", "minimum": 1 },
+                    "line_index": { "type": "integer", "minimum": 1 },
+                    "line_pattern": { "type": "string", "minLength": 1 }
                 },
                 "required": [ "name" ],
                 "additionalProperties": false