Merge pull request #19583 from github/aibaars/lib-as-source

Rust: add option to extract dependencies as source files

Author: aibaars

rust/codeql-extractor.yml

@@ -82,3 +82,11 @@ options:
     title: Skip path resolution
     description: >
       Skip path resolution. This is experimental, while we move path resolution from the extractor to the QL library.
+    type: string
+    pattern: "^(false|true)$"
+  extract_dependencies_as_source:
+    title: Extract dependencies as source code
+    description: >
+      Extract the full source code of dependencies instead of only extracting signatures.
+    type: string
+    pattern: "^(false|true)$"

rust/extractor/src/config.rs

@@ -67,6 +67,7 @@ pub struct Config {
     pub extra_includes: Vec<PathBuf>,
     pub proc_macro_server: Option<PathBuf>,
     pub skip_path_resolution: bool,
+    pub extract_dependencies_as_source: bool,
 }
 
 impl Config {

rust/extractor/src/main.rs

@@ -277,6 +277,11 @@ fn main() -> anyhow::Result<()> {
     } else {
         ResolvePaths::Yes
     };
+    let (library_mode, library_resolve_paths) = if cfg.extract_dependencies_as_source {
+        (SourceKind::Source, resolve_paths)
+    } else {
+        (SourceKind::Library, ResolvePaths::No)
+    };
     let mut processed_files: HashSet<PathBuf, RandomState> =
         HashSet::from_iter(files.iter().cloned());
     for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) {
@@ -312,12 +317,13 @@ fn main() -> anyhow::Result<()> {
                             .source_root(db)
                             .is_library
                     {
+                        tracing::info!("file: {}", file.display());
                         extractor.extract_with_semantics(
                             file,
                             &semantics,
                             vfs,
-                            ResolvePaths::No,
-                            SourceKind::Library,
+                            library_resolve_paths,
+                            library_mode,
                         );
                         extractor.archiver.archive(file);
                     }