Merge pull request #17628 from smowton/smowton/admin/go-vendor-dir-extraction-option

Go: add extractor option for vendor-directory extraction

Author: smowton

go/codeql-extractor.yml

@@ -27,3 +27,10 @@ options:
       The default is 'false'.
     type: string
     pattern: "^(false|true)$"
+  extract_vendor_dirs:
+    title: Whether to include Go vendor directories in the CodeQL database.
+    description: >
+      A value indicating whether Go vendor directories should be included in the CodeQL database.
+      The default is 'false'.
+    type: string
+    pattern: "^(false|true)$"

go/extractor/configurebaseline/configurebaseline.go

@@ -28,7 +28,8 @@ type BaselineConfig struct {
 func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) {
 	vendorDirs := make([]string, 0)
 
-	if util.IsVendorDirExtractionEnabled() {
+	extractVendorDirs, _ := util.IsVendorDirExtractionEnabled()
+	if extractVendorDirs {
 		// The user wants vendor directories scanned; emit an empty report.
 	} else {
 		filepath.WalkDir(rootDir, func(dirPath string, d fs.DirEntry, err error) error {

go/extractor/extractor.go

@@ -81,11 +81,27 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool)
 		}
 	}
 
-	testMessage := ""
+	// If CODEQL_EXTRACTOR_GO_[OPTION_]EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories;
+	// otherwise (the default) is to exclude them from extraction
+	includeVendor, oldOptionUsed := util.IsVendorDirExtractionEnabled()
+
+	if oldOptionUsed {
+		log.Println("Warning: obsolete option \"CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS\" was set. Use \"CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_VENDOR_DIRS\" or pass `--extractor-option extract_vendor_dirs=true` instead.")
+	}
+
+	modeNotifications := make([]string, 0, 2)
 	if extractTests {
-		testMessage = " (test extraction enabled)"
+		modeNotifications = append(modeNotifications, "test extraction enabled")
 	}
-	log.Printf("Running packages.Load%s.", testMessage)
+	if includeVendor {
+		modeNotifications = append(modeNotifications, "extracting vendor directories")
+	}
+
+	modeMessage := strings.Join(modeNotifications, ", ")
+	if modeMessage != "" {
+		modeMessage = " (" + modeMessage + ")"
+	}
+	log.Printf("Running packages.Load%s.", modeMessage)
 
 	// This includes test packages if either we're tracing a `go test` command,
 	// or if CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS is set to "true".
@@ -233,9 +249,6 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool)
 	// Construct a list of directory segments to exclude from extraction, starting with ".."
 	excludedDirs := []string{`\.\.`}
 
-	// If CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories;
-	// otherwise (the default) is to exclude them from extraction
-	includeVendor := util.IsVendorDirExtractionEnabled()
 	if !includeVendor {
 		excludedDirs = append(excludedDirs, "vendor")
 	}

go/extractor/util/extractvendordirs.go

@@ -4,6 +4,8 @@ import (
 	"os"
 )
 
-func IsVendorDirExtractionEnabled() bool {
-	return os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true"
+func IsVendorDirExtractionEnabled() (bool, bool) {
+	oldOptionVal := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS")
+	return (oldOptionVal == "true" ||
+		os.Getenv("CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_VENDOR_DIRS") == "true"), oldOptionVal != ""
 }

go/ql/integration-tests/extract-vendor/test.py

@@ -4,3 +4,6 @@
 def test(codeql, go):
     os.environ["CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS"] = "true"
     codeql.database.create(source_root="src")
+
+def test_extractor_option(codeql, go):
+    codeql.database.create(source_root="src", extractor_option = "extract_vendor_dirs=true")