C#: Introduce extractor mode to identify DBs created with codeql test run

Author: hvitved

csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs

@@ -6,6 +6,7 @@
 using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Globalization;
+using Semmle.Util;
 using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CIL.Driver
@@ -169,18 +170,14 @@ public void ResolveReferences()
     /// <summary>
     /// Parses the command line and collates a list of DLLs/EXEs to extract.
     /// </summary>
-    internal class ExtractorOptions
+    internal class ExtractorOptions : CommonOptions
     {
         private readonly AssemblyList assemblyList = new AssemblyList();
 
         public ExtractorOptions(string[] args)
         {
-            Verbosity = Verbosity.Info;
-            Threads = System.Environment.ProcessorCount;
             PDB = true;
-            TrapCompression = TrapWriter.CompressionMode.Gzip;
-
-            ParseArgs(args);
+            this.ParseArguments(args);
 
             AddFrameworkDirectories(false);
 
@@ -203,12 +200,6 @@ private void AddFrameworkDirectories(bool extractAll)
             AddDirectory(RuntimeEnvironment.GetRuntimeDirectory(), extractAll);
         }
 
-        public Verbosity Verbosity { get; private set; }
-        public bool NoCache { get; private set; }
-        public int Threads { get; private set; }
-        public bool PDB { get; private set; }
-        public TrapWriter.CompressionMode TrapCompression { get; private set; }
-
         private void AddFileOrDirectory(string path)
         {
             path = Path.GetFullPath(path);
@@ -237,43 +228,25 @@ private void AddFileOrDirectory(string path)
         /// </summary>
         public IEnumerable<AssemblyName> MissingReferences => assemblyList.MissingReferences;
 
-        private void ParseArgs(string[] args)
+        public override bool HandleFlag(string flag, bool value)
         {
-            foreach (var arg in args)
+            switch (flag)
             {
-                if (arg == "--verbose")
-                {
-                    Verbosity = Verbosity.All;
-                }
-                else if (arg == "--silent")
-                {
-                    Verbosity = Verbosity.Off;
-                }
-                else if (arg.StartsWith("--verbosity:"))
-                {
-                    Verbosity = (Verbosity)int.Parse(arg.Substring(12));
-                }
-                else if (arg == "--dotnet")
-                {
-                    AddFrameworkDirectories(true);
-                }
-                else if (arg == "--nocache")
-                {
-                    NoCache = true;
-                }
-                else if (arg.StartsWith("--threads:"))
-                {
-                    Threads = int.Parse(arg.Substring(10));
-                }
-                else if (arg == "--no-pdb")
-                {
-                    PDB = false;
-                }
-                else
-                {
-                    AddFileOrDirectory(arg);
-                }
+                case "dotnet":
+                    if (value)
+                        AddFrameworkDirectories(true);
+                    return true;
+                default:
+                    return base.HandleFlag(flag, value);
             }
         }
+
+        public override bool HandleArgument(string argument)
+        {
+            AddFileOrDirectory(argument);
+            return true;
+        }
+
+        public override void InvalidArgument(string argument) { }
     }
 }

csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs

@@ -20,11 +20,11 @@ private static void DisplayHelp()
             Console.WriteLine("    path       A directory/dll/exe to analyze");
         }
 
-        private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression)
+        private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, CommonOptions options)
         {
             var sw = new Stopwatch();
             sw.Start();
-            Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _);
+            Analyser.ExtractCIL(layout, assemblyPath, logger, options, out _, out _);
             sw.Stop();
             logger.Log(Severity.Info, "  {0} ({1})", assemblyPath, sw.Elapsed);
         }
@@ -43,8 +43,7 @@ public static void Main(string[] args)
 
             var actions = options.AssembliesToExtract
                 .Select(asm => asm.Filename)
-                .Select<string, Action>(filename =>
-                    () => ExtractAssembly(layout, filename, logger, options.NoCache, options.PDB, options.TrapCompression))
+                .Select<string, Action>(filename => () => ExtractAssembly(layout, filename, logger, options))
                 .ToArray();
 
             foreach (var missingRef in options.MissingReferences)

csharp/extractor/Semmle.Extraction.CIL/Analyser.cs

@@ -25,22 +25,22 @@ private static void ExtractCIL(TracingExtractor extractor, TrapWriter trapWriter
         /// <param name="extractPdbs">Whether to extract PDBs.</param>
         /// <param name="trapFile">The path of the trap file.</param>
         /// <param name="extracted">Whether the file was extracted (false=cached).</param>
-        public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted)
+        public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
         {
             trapFile = "";
             extracted = false;
             try
             {
                 var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
                 var pathTransformer = new PathTransformer(canonicalPathCache);
-                var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer);
+                var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
                 var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
                 var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
-                using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true);
+                using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), options.TrapCompression, discardDuplicates: true);
                 trapFile = trapWriter.TrapFile;
-                if (nocache || !System.IO.File.Exists(trapFile))
+                if (!options.Cache || !System.IO.File.Exists(trapFile))
                 {
-                    ExtractCIL(extractor, trapWriter, extractPdbs);
+                    ExtractCIL(extractor, trapWriter, options.PDB);
                     extracted = true;
                 }
             }

csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs

@@ -25,7 +25,7 @@ public override IEnumerable<IExtractionProduct> Contents
                 else
                     Context.TrapWriter.Archive(TransformedPath, text);
 
-                yield return Tuples.file_extraction_mode(this, 2);
+                yield return Tuples.file_extraction_mode(this, Context.Extractor.Mode | ExtractorMode.Pdb);
             }
         }
     }

csharp/extractor/Semmle.Extraction.CIL/Tuples.cs

@@ -206,7 +206,7 @@ internal static Tuple containerparent(Folder parent, IFileOrFolder child) =>
         internal static Tuple files(File file, string fullName) =>
             new Tuple("files", file, fullName);
 
-        internal static Tuple file_extraction_mode(File file, int mode) =>
+        internal static Tuple file_extraction_mode(File file, ExtractorMode mode) =>
             new Tuple("file_extraction_mode", file, mode);
 
         internal static Tuple folders(Folder folder, string path) =>

csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs

@@ -64,7 +64,7 @@ private class AssemblyConstructorFactory : CachedEntityFactory<Microsoft.CodeAna
 
         public static Assembly CreateOutputAssembly(Context cx)
         {
-            if (cx.Extractor.Standalone)
+            if (cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
                 throw new InternalError("Attempting to create the output assembly in standalone extraction mode");
             return AssemblyConstructorFactory.Instance.CreateEntity(cx, outputAssemblyCacheKey, null);
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs

@@ -53,7 +53,7 @@ public override void Populate(TextWriter trapFile)
 
             if (attributeSyntax is not null)
             {
-                if (!Context.Extractor.Standalone)
+                if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
                 {
                     trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
                 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs

@@ -85,7 +85,7 @@ public virtual IEnumerable<Extraction.Entities.Location> Locations
                 {
                     // Some built in operators lack locations, so loc is null.
                     yield return Context.CreateLocation(ReportingLocation);
-                    if (!Context.Extractor.Standalone && loc.Kind == LocationKind.SourceFile)
+                    if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && loc.Kind == LocationKind.SourceFile)
                         yield return Assembly.CreateOutputAssembly(Context);
                 }
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs

@@ -129,7 +129,7 @@ public IMethodSymbol? TargetSymbol
                         .Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count)
                         .Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count);
 
-                    return Context.Extractor.Standalone ?
+                    return Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) ?
                         candidates.FirstOrDefault() :
                         candidates.SingleOrDefault();
                 }

csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

@@ -61,7 +61,7 @@ public override void Populate(TextWriter trapFile)
                 }
             }
 
-            trapFile.file_extraction_mode(this, Context.Extractor.Standalone ? 1 : 0);
+            trapFile.file_extraction_mode(this, Context.Extractor.Mode);
         }
 
         private bool IsPossiblyTextFile()