@@ -59,11 +59,11 @@ func init() {
5959
6060// Extract extracts the packages specified by the given patterns
6161func Extract (patterns []string ) error {
62- return ExtractWithFlags (nil , patterns )
62+ return ExtractWithFlags (nil , patterns , false )
6363}
6464
6565// ExtractWithFlags extracts the packages specified by the given patterns and build flags
66- func ExtractWithFlags (buildFlags []string , patterns []string ) error {
66+ func ExtractWithFlags (buildFlags []string , patterns []string , extractTests bool ) error {
6767 startTime := time .Now ()
6868
6969 extraction := NewExtraction (buildFlags , patterns )
@@ -81,14 +81,22 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
8181 }
8282 }
8383
84- log .Println ("Running packages.Load." )
84+ testMessage := ""
85+ if extractTests {
86+ testMessage = " (test extraction enabled)"
87+ }
88+ log .Printf ("Running packages.Load%s." , testMessage )
89+
90+ // This includes test packages if either we're tracing a `go test` command,
91+ // or if CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS is set to "true".
8592 cfg := & packages.Config {
8693 Mode : packages .NeedName | packages .NeedFiles |
8794 packages .NeedCompiledGoFiles |
8895 packages .NeedImports | packages .NeedDeps |
8996 packages .NeedTypes | packages .NeedTypesSizes |
9097 packages .NeedTypesInfo | packages .NeedSyntax ,
9198 BuildFlags : buildFlags ,
99+ Tests : extractTests ,
92100 }
93101 pkgs , err := packages .Load (cfg , patterns ... )
94102 if err != nil {
@@ -123,7 +131,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
123131 if os .Getenv ("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO" ) != "false" {
124132 log .Printf ("Running go list to resolve package and module directories." )
125133 // get all packages information
126- pkgInfos , err = toolchain .GetPkgsInfo (patterns , true , modFlags ... )
134+ pkgInfos , err = toolchain .GetPkgsInfo (patterns , true , extractTests , modFlags ... )
127135 if err != nil {
128136 log .Fatalf ("Error getting dependency package or module directories: %v." , err )
129137 }
@@ -132,8 +140,36 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
132140
133141 pkgsNotFound := make ([]string , 0 , len (pkgs ))
134142
143+ // Build a map from package paths to their longest IDs--
144+ // in the context of a `go test -c` compilation, we will see the same package more than
145+ // once, with IDs like "abc.com/pkgname [abc.com/pkgname.test]" to distinguish the version
146+ // that contains and is used by test code.
147+ // For our purposes it is simplest to just ignore the non-test version, since the test
148+ // version seems to be a superset of it.
149+ longestPackageIds := make (map [string ]string )
150+ packages .Visit (pkgs , nil , func (pkg * packages.Package ) {
151+ if longestIDSoFar , present := longestPackageIds [pkg .PkgPath ]; present {
152+ if len (pkg .ID ) > len (longestIDSoFar ) {
153+ longestPackageIds [pkg .PkgPath ] = pkg .ID
154+ }
155+ } else {
156+ longestPackageIds [pkg .PkgPath ] = pkg .ID
157+ }
158+ })
159+
135160 // Do a post-order traversal and extract the package scope of each package
136161 packages .Visit (pkgs , nil , func (pkg * packages.Package ) {
162+ // Note that if test extraction is enabled, we will encounter a package twice here:
163+ // once as the main package, and once as the test package (with a package ID like
164+ // "abc.com/pkgname [abc.com/pkgname.test]").
165+ //
166+ // We will extract it both times however, because we need to visit the packages
167+ // in the right order in order to visit used types before their users, and the
168+ // ordering determined by packages.Visit for the main and the test package may differ.
169+ //
170+ // This should only cause some wasted time and not inconsistency because the names for
171+ // objects seen in this process should be the same each time.
172+
137173 log .Printf ("Processing package %s." , pkg .PkgPath )
138174
139175 if _ , ok := pkgInfos [pkg .PkgPath ]; ! ok {
@@ -210,6 +246,19 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
210246
211247 // extract AST information for all packages
212248 packages .Visit (pkgs , nil , func (pkg * packages.Package ) {
249+
250+ // If this is a variant of a package that also occurs with a longer ID, skip it;
251+ // otherwise we would extract the same file more than once including extracting the
252+ // body of methods twice, causing database inconsistencies.
253+ //
254+ // We prefer the version with the longest ID because that is (so far as I know) always
255+ // the version that defines more entities -- the only case I'm aware of being a test
256+ // variant of a package, which includes test-only functions in addition to the complete
257+ // contents of the main variant.
258+ if pkg .ID != longestPackageIds [pkg .PkgPath ] {
259+ return
260+ }
261+
213262 for root := range wantedRoots {
214263 pkgInfo := pkgInfos [pkg .PkgPath ]
215264 relDir , err := filepath .Rel (root , pkgInfo .PkgDir )
0 commit comments