Add glob file pattern matching support for the excludes attribute. (#354)

SBGoods · austinvalle · web-flow · commit cb367027d756 · 2024-07-30T15:05:58.000-04:00
* Add glob pattern matching support for the `excludes` attribute.

* Regenerate provider documentation

* Switch from `Match()` to `PathMatch()` to be OS agnostic.

* Add changelog entries

* Add missing punctuation.

* Regenerate provider documentation.

* Update internal/provider/zip_archiver.go

Co-authored-by: Austin Valle &lt;austinvalle@gmail.com&gt;

* Add acceptance tests

---------

Co-authored-by: Austin Valle &lt;austinvalle@gmail.com&gt;
diff --git a/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml b/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml
@@ -0,0 +1,5 @@
+kind: ENHANCEMENTS
+body: 'data-source/archive_file: Add glob pattern matching support to the `excludes` attribute.'
+time: 2024-07-24T16:07:24.058378-04:00
+custom:
+    Issue: "354"
diff --git a/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml b/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml
@@ -0,0 +1,5 @@
+kind: ENHANCEMENTS
+body: 'resource/archive_file: Add glob pattern matching support to the `excludes` attribute.'
+time: 2024-07-24T16:08:30.211413-04:00
+custom:
+    Issue: "354"
diff --git a/docs/data-sources/file.md b/docs/data-sources/file.md
@@ -63,7 +63,7 @@ data "archive_file" "lambda_my_function" {
 ### Optional
 
 - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`.
-- `excludes` (Set of String) Specify files to ignore when reading the `source_dir`.
+- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns.
 - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior.
 - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source))
 - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified.
diff --git a/docs/resources/file.md b/docs/resources/file.md
@@ -23,7 +23,7 @@ description: |-
 ### Optional
 
 - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`.
-- `excludes` (Set of String) Specify files to ignore when reading the `source_dir`.
+- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns.
 - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior.
 - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source))
 - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified.
diff --git a/go.mod b/go.mod
@@ -5,6 +5,7 @@ go 1.21
 toolchain go1.21.3
 
 require (
+	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/hashicorp/terraform-plugin-framework v1.10.0
 	github.com/hashicorp/terraform-plugin-framework-validators v0.13.0
 	github.com/hashicorp/terraform-plugin-go v0.23.0
diff --git a/go.sum b/go.sum
@@ -9,6 +9,8 @@ github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki
 github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
+github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
+github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
diff --git a/internal/provider/data_source_archive_file.go b/internal/provider/data_source_archive_file.go
@@ -134,7 +134,8 @@ func (d *archiveFileDataSource) Schema(ctx context.Context, req datasource.Schem
 				},
 			},
 			"excludes": schema.SetAttribute{
-				Description: "Specify files to ignore when reading the `source_dir`.",
+				Description: "Specify files/directories to ignore when reading the `source_dir`. " +
+					"Supports glob file matching patterns including doublestar/globstar (`**`) patterns.",
 				ElementType: types.StringType,
 				Optional:    true,
 				Validators: []validator.Set{
diff --git a/internal/provider/data_source_archive_file_test.go b/internal/provider/data_source_archive_file_test.go
@@ -105,6 +105,13 @@ func TestAccArchiveFile_Basic(t *testing.T) {
 					r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize),
 				),
 			},
+			{
+				Config: testAccArchiveFileDirExcludesGlobConfig(f),
+				Check: r.ComposeTestCheckFunc(
+					testAccArchiveFileSize(f, &fileSize),
+					r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize),
+				),
+			},
 			{
 				Config: testAccArchiveFileMultiSourceConfig(f),
 				Check: r.ComposeTestCheckFunc(
@@ -1389,6 +1396,17 @@ data "archive_file" "foo" {
 `, filepath.ToSlash(outputPath))
 }
 
+func testAccArchiveFileDirExcludesGlobConfig(outputPath string) string {
+	return fmt.Sprintf(`
+data "archive_file" "foo" {
+  type        = "zip"
+  source_dir  = "test-fixtures/test-dir/test-dir1"
+  excludes    = ["test-fixtures/test-dir/test-dir1/file2.txt", "**/file[2-3].txt"]
+  output_path = "%s"
+}
+`, filepath.ToSlash(outputPath))
+}
+
 func testAccArchiveFileMultiSourceConfig(outputPath string) string {
 	return fmt.Sprintf(`
 data "archive_file" "foo" {
diff --git a/internal/provider/resource_archive_file.go b/internal/provider/resource_archive_file.go
@@ -154,7 +154,8 @@ func (d *archiveFileResource) Schema(ctx context.Context, req resource.SchemaReq
 				},
 			},
 			"excludes": schema.SetAttribute{
-				Description: "Specify files to ignore when reading the `source_dir`.",
+				Description: "Specify files/directories to ignore when reading the `source_dir`. " +
+					"Supports glob file matching patterns including doublestar/globstar (`**`) patterns.",
 				ElementType: types.StringType,
 				Optional:    true,
 				Validators: []validator.Set{
diff --git a/internal/provider/resource_archive_file_test.go b/internal/provider/resource_archive_file_test.go
@@ -104,6 +104,13 @@ func TestAccArchiveFile_Resource_Basic(t *testing.T) {
 					r.TestCheckResourceAttrPtr("archive_file.foo", "output_size", &fileSize),
 				),
 			},
+			{
+				Config: testAccArchiveFileResourceDirExcludesGlobConfig(f),
+				Check: r.ComposeTestCheckFunc(
+					testAccArchiveFileSize(f, &fileSize),
+					r.TestCheckResourceAttrPtr("archive_file.foo", "output_size", &fileSize),
+				),
+			},
 			{
 				Config: testAccArchiveFileResourceMultiSourceConfig(f),
 				Check: r.ComposeTestCheckFunc(
@@ -1485,6 +1492,17 @@ resource "archive_file" "foo" {
 `, filepath.ToSlash(outputPath))
 }
 
+func testAccArchiveFileResourceDirExcludesGlobConfig(outputPath string) string {
+	return fmt.Sprintf(`
+resource "archive_file" "foo" {
+  type        = "zip"
+  source_dir  = "test-fixtures/test-dir"
+  excludes    = ["test-fixtures/test-dir/file2.txt", "**/file[2-3].txt"]
+  output_path = "%s"
+}
+`, filepath.ToSlash(outputPath))
+}
+
 func testAccArchiveFileResourceMultiSourceConfig(outputPath string) string {
 	return fmt.Sprintf(`
 resource "archive_file" "foo" {
diff --git a/internal/provider/zip_archiver.go b/internal/provider/zip_archiver.go
@@ -11,6 +11,8 @@ import (
 	"sort"
 	"strconv"
 	"time"
+
+	"github.com/bmatcuk/doublestar/v4"
 )
 
 type ZipArchiver struct {
@@ -83,17 +85,22 @@ func (a *ZipArchiver) ArchiveFile(infilename string) error {
 	return err
 }
 
-func checkMatch(fileName string, excludes []string) (value bool) {
+func checkMatch(fileName string, excludes []string) (value bool, err error) {
 	for _, exclude := range excludes {
 		if exclude == "" {
 			continue
 		}
 
-		if exclude == fileName {
-			return true
+		match, err := doublestar.PathMatch(exclude, fileName)
+		if err != nil {
+			return false, err
+		}
+
+		if match {
+			return true, nil
 		}
 	}
-	return false
+	return false, nil
 }
 
 func (a *ZipArchiver) ArchiveDir(indirname string, opts ArchiveDirOpts) error {
@@ -141,7 +148,10 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir
 
 		archivePath := filepath.Join(basePath, relname)
 
-		isMatch := checkMatch(archivePath, opts.Excludes)
+		isMatch, err := checkMatch(archivePath, opts.Excludes)
+		if err != nil {
+			return fmt.Errorf("error checking excludes matches: %w", err)
+		}
 
 		if info.IsDir() {
 			if isMatch {
@@ -154,10 +164,6 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir
 			return nil
 		}
 
-		if err != nil {
-			return err
-		}
-
 		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
 			realPath, err := filepath.EvalSymlinks(path)
 			if err != nil {
diff --git a/internal/provider/zip_archiver_test.go b/internal/provider/zip_archiver_test.go
@@ -254,6 +254,32 @@ func TestZipArchiver_Dir_Exclude_DoNotExcludeSymlinkDirectories(t *testing.T) {
 	})
 }
 
+func TestZipArchiver_Dir_Exclude_Glob_DoNotExcludeSymlinkDirectories(t *testing.T) {
+	zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip")
+
+	archiver := NewZipArchiver(zipFilePath)
+	if err := archiver.ArchiveDir("./test-fixtures", ArchiveDirOpts{
+		Excludes: []string{
+			"**/file1.txt",
+			"**/file2.*",
+			"test-dir-with-symlink-dir/test-symlink-dir",
+			"test-symlink-dir-with-symlink-file/test-symlink.txt",
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	ensureContents(t, zipFilePath, map[string][]byte{
+		"test-dir/test-dir1/file3.txt":                     []byte("This is file 3"),
+		"test-dir/test-dir2/file3.txt":                     []byte("This is file 3"),
+		"test-dir/test-file.txt":                           []byte("This is test content"),
+		"test-dir-with-symlink-file/test-file.txt":         []byte("This is test content"),
+		"test-dir-with-symlink-file/test-symlink.txt":      []byte("This is test content"),
+		"test-symlink-dir/file3.txt":                       []byte("This is file 3"),
+		"test-symlink-dir-with-symlink-file/test-file.txt": []byte("This is test content"),
+	})
+}
+
 func TestZipArchiver_Dir_Exclude_ExcludeSymlinkDirectories(t *testing.T) {
 	zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip")
 
@@ -269,6 +295,40 @@ func TestZipArchiver_Dir_Exclude_ExcludeSymlinkDirectories(t *testing.T) {
 	if err != nil {
 		t.Errorf("expected no error: %s", err)
 	}
+
+	ensureContents(t, zipFilePath, map[string][]byte{
+		"test-dir/test-dir1/file2.txt":                []byte("This is file 2"),
+		"test-dir/test-dir1/file3.txt":                []byte("This is file 3"),
+		"test-dir/test-dir2/file1.txt":                []byte("This is file 1"),
+		"test-dir/test-dir2/file2.txt":                []byte("This is file 2"),
+		"test-dir/test-dir2/file3.txt":                []byte("This is file 3"),
+		"test-dir/test-file.txt":                      []byte("This is test content"),
+		"test-dir-with-symlink-file/test-file.txt":    []byte("This is test content"),
+		"test-dir-with-symlink-file/test-symlink.txt": []byte("This is test content"),
+	})
+}
+
+func TestZipArchiver_Dir_Exclude_Glob_ExcludeSymlinkDirectories(t *testing.T) {
+	zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip")
+
+	archiver := NewZipArchiver(zipFilePath)
+	err := archiver.ArchiveDir("./test-fixtures", ArchiveDirOpts{
+		Excludes: []string{
+			"test-dir/test-dir1/file1.txt",
+			"**/file[2-3].txt",
+			"test-dir-with-symlink-file",
+		},
+		ExcludeSymlinkDirectories: true,
+	})
+
+	if err != nil {
+		t.Errorf("expected no error: %s", err)
+	}
+
+	ensureContents(t, zipFilePath, map[string][]byte{
+		"test-dir/test-dir2/file1.txt": []byte("This is file 1"),
+		"test-dir/test-file.txt":       []byte("This is test content"),
+	})
 }
 
 func ensureContents(t *testing.T, zipfilepath string, wants map[string][]byte) {
