Extractor: move overlay-changes check from traverser to worker

This way, we filter both root modules and (transitive) imports against the overlay-changes json.

Author: d10c

python/extractor/semmle/traverser.py

@@ -31,19 +31,6 @@ def __init__(self, options, modulenames, logger):
                 if not os.path.exists(p) and not options.ignore_missing_modules:
                     raise FileNotFoundError("'%s' does not exist." % p)
                 self.paths.add(p)
-        # During overlay extraction, only traverse the files that were changed.
-        self.overlay_changes = None
-        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES' in os.environ:
-            overlay_changes_file = os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES']
-            logger.info("Overlay extraction mode: only extracting files changed according to '%s'", overlay_changes_file)
-            try:
-                with open(overlay_changes_file, 'r', encoding='utf-8') as f:
-                    data = json.load(f)
-                    changed_paths = data.get('changes', [])
-                    self.overlay_changes = { os.path.abspath(p) for p in changed_paths }
-            except (IOError, ValueError) as e:
-                logger.warn("Failed to read overlay changes from '%s' (falling back to full extraction): %s", overlay_changes_file, e)
-                self.overlay_changes = None
         self.exclude_paths = set([ os.path.abspath(f) for f in options.exclude_file ])
         self.exclude = exclude_filter_from_options(options)
         self.filter = filter_from_options_and_environment(options)
@@ -62,20 +49,11 @@ def __iter__(self):
                 if mod is None:
                     self.logger.error("No module named '%s'.", name)
                     raise ExtractorFailure()
-                if self.overlay_changes is not None and mod.path not in self.overlay_changes:
-                    self.logger.debug("Skipping module '%s' as it was not changed in overlay extraction.", name)
-                    continue
                 yield mod.get_extractable()
         for path in self.paths:
-            if self.overlay_changes is not None and path not in self.overlay_changes:
-                self.logger.debug("Skipping path '%s' as it was not changed in overlay extraction.", path)
-                continue
             yield Extractable.from_path(path)
         for path in self.recurse_files:
             for modpath in self._treewalk(path):
-                if self.overlay_changes is not None and modpath not in self.overlay_changes:
-                    self.logger.debug("Skipping file '%s' as it was not changed in overlay extraction.", modpath)
-                    continue
                 yield Extractable.from_path(modpath)
         for name in self.recurse_packages:
             mod = self.finder.find(name)
@@ -89,9 +67,6 @@ def __iter__(self):
                 self.logger.error("Package '%s' does not have a path.", name)
                 raise ExtractorFailure()
             for modpath in self._treewalk(path):
-                if self.overlay_changes is not None and modpath not in self.overlay_changes:
-                    self.logger.debug("Skipping package '%s' as it was not changed in overlay extraction.", modpath)
-                    continue
                 yield Extractable.from_path(modpath)
 
     def _treewalk(self, path):

python/extractor/semmle/worker.py

@@ -11,6 +11,7 @@
 from semmle.profiling import get_profiler
 from semmle.path_rename import renamer_from_options_and_env
 from semmle.logging import WARN, recursion_error_message, internal_error_message, Logger
+from semmle.util import FileExtractable, FolderExtractable
 
 class ExtractorFailure(Exception):
     'Generic exception representing the failure of an extractor.'
@@ -19,17 +20,32 @@ class ExtractorFailure(Exception):
 
 class ModuleImportGraph(object):
 
-    def __init__(self, max_depth):
+    def __init__(self, max_depth, logger: Logger):
         self.modules = {}
         self.succ = defaultdict(set)
         self.todo = set()
         self.done = set()
         self.max_depth = max_depth
+        self.logger = logger
+
+        # During overlay extraction, only traverse the files that were changed.
+        self.overlay_changes = None
+        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES' in os.environ:
+            overlay_changes_file = os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES']
+            logger.info("Overlay extraction mode: only extracting files changed according to '%s'", overlay_changes_file)
+            try:
+                with open(overlay_changes_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    changed_paths = data.get('changes', [])
+                    self.overlay_changes = { os.path.abspath(p) for p in changed_paths }
+            except (IOError, ValueError) as e:
+                logger.warn("Failed to read overlay changes from '%s' (falling back to full extraction): %s", overlay_changes_file, e)
+                self.overlay_changes = None
 
     def add_root(self, mod):
         self.modules[mod] = 0
         if mod not in self.done:
-            self.todo.add(mod)
+            self.add_todo(mod)
 
     def add_import(self, mod, imported):
         assert mod in self.modules
@@ -39,7 +55,7 @@ def add_import(self, mod, imported):
                 self._reduce_depth(imported, self.modules[mod] + 1)
         else:
             if self.modules[mod] < self.max_depth and imported not in self.done:
-                self.todo.add(imported)
+                self.add_todo(imported)
             self.modules[imported] = self.modules[mod] + 1
 
     def _reduce_depth(self, mod, depth):
@@ -48,7 +64,7 @@ def _reduce_depth(self, mod, depth):
         if depth > self.max_depth:
             return
         if mod not in self.done:
-            self.todo.add(mod)
+            self.add_todo(mod)
         self.modules[mod] = depth
         for imp in self.succ[mod]:
             self._reduce_depth(imp, depth+1)
@@ -61,11 +77,25 @@ def get(self):
 
     def push_back(self, mod):
         self.done.remove(mod)
-        self.todo.add(mod)
+        self.add_todo(mod)
 
     def empty(self):
         return not self.todo
 
+    def add_todo(self, mod):
+        if not self._module_in_overlay_changes(mod):
+            self.logger.debug("Skipping module '%s' as it was not changed in overlay extraction.", mod)
+            return
+        self.todo.add(mod)
+
+    def _module_in_overlay_changes(self, mod):
+        if self.overlay_changes is not None:
+            if isinstance(mod, FileExtractable):
+                return mod.path in self.overlay_changes
+            if isinstance(mod, FolderExtractable):
+                return mod.path + '/__init__.py' in self.overlay_changes
+        return True
+
 class ExtractorPool(object):
     '''Pool of worker processes running extractors'''
 
@@ -90,7 +120,7 @@ def __init__(self, outdir, archive, proc_count, options, logger: Logger):
         self.enqueued = set()
         self.done = set()
         self.requirements = {}
-        self.import_graph = ModuleImportGraph(options.max_import_depth)
+        self.import_graph = ModuleImportGraph(options.max_import_depth, logger)
         logger.debug("Source archive: %s", archive)
         self.logger = logger
         DiagnosticsWriter.create_output_dir()