Bulk generator: Specify 'with-summaries', 'with-sources', and 'with-sinks' in the config file.

Author: MathiasVP

cpp/misc/bulk_generation_targets.json

@@ -1,8 +1,8 @@
 {
   "strategy": "dca",
   "targets": [
-      { "name": "openssl" },
-      { "name": "sqlite" }
+      { "name": "openssl", "with_summaries": true },
+      { "name": "sqlite", "with_summaries": true }
   ],
   "destination": "cpp/ql/lib/ext/generated"
 }

misc/scripts/models-as-data/bulk_generate_mad.py

@@ -41,7 +41,18 @@ class Project(TypedDict):
     name: str
     git_repo: NotRequired[str]
     git_tag: NotRequired[str]
+    with_sinks: NotRequired[bool]
+    with_sinks: NotRequired[bool]
+    with_summaries: NotRequired[bool]
 
+def shouldGenerateSinks(project: Project) -> bool:
+    return project.get("with_sinks", False)
+
+def shouldGenerateSources(project: Project) -> bool:
+    return project.get("with_sources", False)
+
+def shouldGenerateSummaries(project: Project) -> bool:
+    return project.get("with_summaries", False)
 
 def clone_project(project: Project) -> str:
     """
@@ -185,7 +196,7 @@ def build_database(
     return database_dir
 
 
-def generate_models(args, project: Project, database_dir: str) -> None:
+def generate_models(language: str, config, project: Project, database_dir: str) -> None:
     """
     Generate models for a project.
 
@@ -196,10 +207,11 @@ def generate_models(args, project: Project, database_dir: str) -> None:
     """
     name = project["name"]
 
-    generator = mad.Generator(args.lang)
-    generator.generateSinks = args.with_sinks
-    generator.generateSources = args.with_sources
-    generator.generateSummaries = args.with_summaries
+    generator = mad.Generator(language)
+    # Note: The argument parser converts with-sinks to with_sinks, etc.
+    generator.generateSinks = shouldGenerateSinks(project)
+    generator.generateSources = shouldGenerateSources(project)
+    generator.generateSummaries = shouldGenerateSummaries(project)
     generator.setenvironment(database=database_dir, folder=name)
     generator.run()
 
@@ -309,13 +321,14 @@ def download_dca_databases(
         pat,
     )
     targets = response["targets"]
+    project_map = {project["name"]: project for project in projects}
     for data in targets.values():
         downloads = data["downloads"]
         analyzed_database = downloads["analyzed_database"]
         artifact_name = analyzed_database["artifact_name"]
         pretty_name = pretty_name_from_artifact_name(artifact_name)
 
-        if not pretty_name in [project["name"] for project in projects]:
+        if not pretty_name in project_map:
             print(f"Skipping {pretty_name} as it is not in the list of projects")
             continue
 
@@ -350,7 +363,7 @@ def download_dca_databases(
                     tar_ref.extractall(artifact_unzipped_location)
                     database_results.append(
                         (
-                            {"name": pretty_name},
+                            project_map[pretty_name],
                             os.path.join(
                                 artifact_unzipped_location, remove_extension(entry)
                             ),
@@ -451,7 +464,7 @@ def main(config, args) -> None:
 
     for project, database_dir in database_results:
         if database_dir is not None:
-            generate_models(args, project, database_dir)
+            generate_models(language, config, project, database_dir)
 
 
 if __name__ == "__main__":
@@ -474,15 +487,6 @@ def main(config, args) -> None:
     parser.add_argument(
         "--lang", type=str, help="The language to generate models for", required=True
     )
-    parser.add_argument(
-        "--with-sources", action="store_true", help="Generate sources", required=False
-    )
-    parser.add_argument(
-        "--with-sinks", action="store_true", help="Generate sinks", required=False
-    )
-    parser.add_argument(
-        "--with-summaries", action="store_true", help="Generate sinks", required=False
-    )
     args = parser.parse_args()
 
     # Load config file

rust/misc/bulk_generation_targets.json

@@ -4,67 +4,106 @@
         {
             "name": "libc",
             "git_repo": "https://github.com/rust-lang/libc",
-            "git_tag": "0.2.172"
+            "git_tag": "0.2.172",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "log",
             "git_repo": "https://github.com/rust-lang/log",
-            "git_tag": "0.4.27"
+            "git_tag": "0.4.27",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "memchr",
             "git_repo": "https://github.com/BurntSushi/memchr",
-            "git_tag": "2.7.4"
+            "git_tag": "2.7.4",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "once_cell",
             "git_repo": "https://github.com/matklad/once_cell",
-            "git_tag": "v1.21.3"
+            "git_tag": "v1.21.3",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "rand",
             "git_repo": "https://github.com/rust-random/rand",
-            "git_tag": "0.9.1"
+            "git_tag": "0.9.1",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "smallvec",
             "git_repo": "https://github.com/servo/rust-smallvec",
-            "git_tag": "v1.15.0"
+            "git_tag": "v1.15.0",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "serde",
             "git_repo": "https://github.com/serde-rs/serde",
-            "git_tag": "v1.0.219"
+            "git_tag": "v1.0.219",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "tokio",
             "git_repo": "https://github.com/tokio-rs/tokio",
-            "git_tag": "tokio-1.45.0"
+            "git_tag": "tokio-1.45.0",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "reqwest",
             "git_repo": "https://github.com/seanmonstar/reqwest",
-            "git_tag": "v0.12.15"
+            "git_tag": "v0.12.15",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "rocket",
             "git_repo": "https://github.com/SergioBenitez/Rocket",
-            "git_tag": "v0.5.1"
+            "git_tag": "v0.5.1",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "actix-web",
             "git_repo": "https://github.com/actix/actix-web",
-            "git_tag": "web-v4.11.0"
+            "git_tag": "web-v4.11.0",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "hyper",
             "git_repo": "https://github.com/hyperium/hyper",
-            "git_tag": "v1.6.0"
+            "git_tag": "v1.6.0",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         },
         {
             "name": "clap",
             "git_repo": "https://github.com/clap-rs/clap",
-            "git_tag": "v4.5.38"
+            "git_tag": "v4.5.38",
+            "with-sources": true,
+            "with-sinks": true,
+            "with-summaries": true
         }
     ],
     "destination": "rust/ql/lib/ext/generated",