Merge pull request github#6646 from tamasvajk/fix/csv-timeseries

Fix CSV timeseries script to create DB with scheme from correct git SHA

Author: tamasvajk

misc/scripts/library-coverage/generate-timeseries.py

@@ -41,14 +41,15 @@ def get_previous_sha(sha, date):
         return (parent_sha, parent_date)
 
 
-def get_packages(lang, query, search_path):
+def get_packages(config, search_path):
     try:
-        db = "empty_" + lang
-        ql_output = "output-" + lang + ".csv"
+        db = "empty_" + config.lang
+        ql_output = "output-" + config.lang + ".csv"
         if os.path.isdir(db):
             shutil.rmtree(db)
-        utils.create_empty_database(lang, ".java", db)
-        utils.run_codeql_query(query, db, ql_output, search_path)
+        utils.create_empty_database(
+            config.lang, config.ext, db, config.dbscheme)
+        utils.run_codeql_query(config.ql_path, db, ql_output, search_path)
 
         return pack.PackageCollection(ql_output)
     except:
@@ -71,9 +72,9 @@ def get_packages(lang, query, search_path):
 
 configs = [
     utils.LanguageConfig(
-        "java", "Java", ".java", "java/ql/src/meta/frameworks/Coverage.ql"),
+        "java", "Java", ".java", "java/ql/src/meta/frameworks/Coverage.ql", ["java/ql/lib/config/semmlecode.dbscheme", "java/ql/src/config/semmlecode.dbscheme"]),
     utils.LanguageConfig(
-        "csharp", "C#", ".cs", "csharp/ql/src/meta/frameworks/Coverage.ql")
+        "csharp", "C#", ".cs", "csharp/ql/src/meta/frameworks/Coverage.ql", ["csharp/ql/lib/semmlecode.csharp.dbscheme", "csharp/ql/src/semmlecode.csharp.dbscheme"])
 ]
 
 output_prefix = "framework-coverage-timeseries-"
@@ -102,7 +103,8 @@ def get_packages(lang, query, search_path):
             "file_total": file_total,
             "file_packages": file_packages,
             "csvwriter_total": csvwriter_total,
-            "csvwriter_packages": csvwriter_packages
+            "csvwriter_packages": csvwriter_packages,
+            "last_row": (None, None, None)
         }
 
 try:
@@ -141,15 +143,20 @@ def get_packages(lang, query, search_path):
                 frameworks: fr.FrameworkCollection = language_utils[lang]["frameworks"]
                 csvwriter_total = language_utils[lang]["csvwriter_total"]
                 csvwriter_packages = language_utils[lang]["csvwriter_packages"]
+                last_row = language_utils[lang]["last_row"]
 
-                packages = get_packages(lang, config.ql_path, ".")
+                packages = get_packages(config, ".")
 
-                csvwriter_total.writerow([
-                    current_sha,
-                    current_date,
-                    packages.get_part_count("source"),
-                    packages.get_part_count("sink"),
-                    packages.get_part_count("summary")])
+                new_row = (packages.get_part_count("source"),
+                           packages.get_part_count("sink"),
+                           packages.get_part_count("summary"))
+
+                if last_row != new_row:
+                    csvwriter_total.writerow([
+                        current_sha,
+                        current_date,
+                        new_row[0], new_row[1], new_row[2]])
+                    language_utils[lang]["last_row"] = new_row
 
                 matched_packages = set()
 
@@ -158,7 +165,7 @@ def get_packages(lang, query, search_path):
                     framework: fr.Framework = framework
 
                     row = [current_sha, current_date,
-                           framework.name, framework.package_pattern]
+                           framework.name, ", ".join(sorted(framework.package_pattern.split(" ")))]
 
                     sources = 0
                     sinks = 0

misc/scripts/library-coverage/utils.py

@@ -17,14 +17,23 @@ def subprocess_check_output(cmd):
     return subprocess.check_output(cmd, text=True, env=os.environ.copy())
 
 
-def create_empty_database(lang, extension, database):
+def create_empty_database(lang, extension, database, dbscheme=None):
     """Creates an empty database for the given language."""
     subprocess_run(["codeql", "database", "init", "--language=" + lang,
                    "--source-root=/tmp/empty", "--allow-missing-source-root", database])
     subprocess_run(["mkdir", "-p", database + "/src/tmp/empty"])
     subprocess_run(["touch", database + "/src/tmp/empty/empty" + extension])
-    subprocess_run(["codeql", "database", "finalize",
-                   database, "--no-pre-finalize"])
+
+    finalize_cmd = ["codeql", "database", "finalize",
+                    database, "--no-pre-finalize"]
+    if dbscheme is not None:
+        for scheme in dbscheme:
+            if os.path.exists(scheme):
+                finalize_cmd.append("--dbscheme")
+                finalize_cmd.append(scheme)
+                break
+
+    subprocess_run(finalize_cmd)
 
 
 def run_codeql_query(query, database, output, search_path):
@@ -38,11 +47,12 @@ def run_codeql_query(query, database, output, search_path):
 
 
 class LanguageConfig:
-    def __init__(self, lang, capitalized_lang, ext, ql_path):
+    def __init__(self, lang, capitalized_lang, ext, ql_path, dbscheme=None):
         self.lang = lang
         self.capitalized_lang = capitalized_lang
         self.ext = ext
         self.ql_path = ql_path
+        self.dbscheme = dbscheme
 
 
 def read_cwes(path):