Merge pull request #19667 from github/redsun82/rust-mad

Rust: Use QL computed canonical paths in MaD `Field` tokens

Author: redsun82

misc/scripts/models-as-data/bulk_generate_mad.py

@@ -225,7 +225,7 @@ def build_database(
     return database_dir
 
 
-def generate_models(config, project: Project, database_dir: str) -> None:
+def generate_models(config, args, project: Project, database_dir: str) -> None:
     """
     Generate models for a project.
 
@@ -243,6 +243,8 @@ def generate_models(config, project: Project, database_dir: str) -> None:
     generator.generateSources = should_generate_sources(project)
     generator.generateSummaries = should_generate_summaries(project)
     generator.setenvironment(database=database_dir, folder=name)
+    generator.threads = args.codeql_threads
+    generator.ram = args.codeql_ram
     generator.run()
 
 
@@ -333,43 +335,44 @@ def pretty_name_from_artifact_name(artifact_name: str) -> str:
 
 def download_dca_databases(
     language: str,
-    experiment_name: str,
+    experiment_names: list[str],
     pat: str,
     projects: List[Project],
 ) -> List[tuple[Project, str | None]]:
     """
     Download databases from a DCA experiment.
     Args:
-        experiment_name: The name of the DCA experiment to download databases from.
+        experiment_names: The names of the DCA experiments to download databases from.
         pat: Personal Access Token for GitHub API authentication.
         projects: List of projects to download databases for.
     Returns:
         List of (project_name, database_dir) pairs, where database_dir is None if the download failed.
     """
     print("\n=== Finding projects ===")
-    response = get_json_from_github(
-        f"https://raw.githubusercontent.com/github/codeql-dca-main/data/{experiment_name}/reports/downloads.json",
-        pat,
-    )
-    targets = response["targets"]
     project_map = {project["name"]: project for project in projects}
     analyzed_databases = {}
-    for data in targets.values():
-        downloads = data["downloads"]
-        analyzed_database = downloads["analyzed_database"]
-        artifact_name = analyzed_database["artifact_name"]
-        pretty_name = pretty_name_from_artifact_name(artifact_name)
-
-        if not pretty_name in project_map:
-            print(f"Skipping {pretty_name} as it is not in the list of projects")
-            continue
-
-        if pretty_name in analyzed_databases:
-            print(
-                f"Skipping previous database {analyzed_databases[pretty_name]['artifact_name']} for {pretty_name}"
-            )
+    for experiment_name in experiment_names:
+        response = get_json_from_github(
+            f"https://raw.githubusercontent.com/github/codeql-dca-main/data/{experiment_name}/reports/downloads.json",
+            pat,
+        )
+        targets = response["targets"]
+        for data in targets.values():
+            downloads = data["downloads"]
+            analyzed_database = downloads["analyzed_database"]
+            artifact_name = analyzed_database["artifact_name"]
+            pretty_name = pretty_name_from_artifact_name(artifact_name)
+
+            if not pretty_name in project_map:
+                print(f"Skipping {pretty_name} as it is not in the list of projects")
+                continue
+
+            if pretty_name in analyzed_databases:
+                print(
+                    f"Skipping previous database {analyzed_databases[pretty_name]['artifact_name']} for {pretty_name}"
+                )
 
-        analyzed_databases[pretty_name] = analyzed_database
+            analyzed_databases[pretty_name] = analyzed_database
 
     def download_and_decompress(analyzed_database: dict) -> str:
         artifact_name = analyzed_database["artifact_name"]
@@ -450,23 +453,6 @@ def main(config, args) -> None:
     if not os.path.exists(build_dir):
         os.makedirs(build_dir)
 
-    # Check if any of the MaD directories contain working directory changes in git
-    for project in projects:
-        mad_dir = get_mad_destination_for_project(config, project["name"])
-        if os.path.exists(mad_dir):
-            git_status_output = subprocess.check_output(
-                ["git", "status", "-s", mad_dir], text=True
-            ).strip()
-            if git_status_output:
-                print(
-                    f"""ERROR: Working directory changes detected in {mad_dir}.
-
-Before generating new models, the existing models are deleted.
-
-To avoid loss of data, please commit your changes."""
-                )
-                sys.exit(1)
-
     database_results = []
     match get_strategy(config):
         case "repo":
@@ -477,8 +463,8 @@ def main(config, args) -> None:
                 projects,
             )
         case "dca":
-            experiment_name = args.dca
-            if experiment_name is None:
+            experiment_names = args.dca
+            if experiment_names is None:
                 print("ERROR: --dca argument is required for DCA strategy")
                 sys.exit(1)
 
@@ -492,7 +478,7 @@ def main(config, args) -> None:
                 pat = f.read().strip()
                 database_results = download_dca_databases(
                     language,
-                    experiment_name,
+                    experiment_names,
                     pat,
                     projects,
                 )
@@ -518,7 +504,7 @@ def main(config, args) -> None:
 
     for project, database_dir in database_results:
         if database_dir is not None:
-            generate_models(config, project, database_dir)
+            generate_models(config, args, project, database_dir)
 
 
 if __name__ == "__main__":
@@ -529,14 +515,26 @@ def main(config, args) -> None:
     parser.add_argument(
         "--dca",
         type=str,
-        help="Name of a DCA run that built all the projects",
-        required=False,
+        help="Name of a DCA run that built all the projects. Can be repeated, with sources taken from all provided runs, "
+             "the last provided ones having priority",
+        action="append",
     )
     parser.add_argument(
         "--pat",
         type=str,
         help="Path to a file containing the PAT token required to grab DCA databases (the same as the one you use for DCA)",
-        required=False,
+    )
+    parser.add_argument(
+        "--codeql-ram",
+        type=int,
+        help="What `--ram` value to pass to `codeql` while generating models (by default the flag is not passed)",
+        default=None,
+    )
+    parser.add_argument(
+        "--codeql-threads",
+        type=int,
+        help="What `--threads` value to pass to `codeql` (default %(default)s)",
+        default=0,
     )
     args = parser.parse_args()
 

misc/scripts/models-as-data/generate_mad.py

@@ -62,6 +62,8 @@ def __init__(self, language):
         self.generateTypeBasedSummaries = False
         self.dryRun = False
         self.dirname = "modelgenerator"
+        self.ram = 2**15
+        self.threads = 8
 
 
     def setenvironment(self, database, folder):
@@ -138,8 +140,12 @@ def runQuery(self, query):
         queryFile = os.path.join(self.codeQlRoot, f"{self.language}/ql/src/utils/{self.dirname}", query)
         resultBqrs = os.path.join(self.workDir, "out.bqrs")
 
-        helpers.run_cmd(['codeql', 'query', 'run', queryFile, '--database',
-               self.database, '--output', resultBqrs, '--threads', '8', '--ram', '32768'], "Failed to generate " + query)
+        cmd = ['codeql', 'query', 'run', queryFile, '--database', self.database, '--output', resultBqrs]
+        if self.threads is not None:
+            cmd += ["--threads", str(self.threads)]
+        if self.ram is not None:
+            cmd += ["--ram", str(self.ram)]
+        helpers.run_cmd(cmd, "Failed to generate " + query)
 
         return helpers.readData(self.workDir, resultBqrs)
 
@@ -214,4 +220,4 @@ def run(self):
             self.save(typeBasedContent, ".typebased.model.yml")
 
 if __name__ == '__main__':
-    Generator.make().run()
+    Generator.make().run()

rust/ql/lib/codeql/rust/dataflow/internal/FlowSummaryImpl.qll

@@ -68,29 +68,14 @@ module Input implements InputSig<Location, RustDataFlow> {
       result = "Field" and
       (
         exists(Addressable a, int pos, string prefix |
-          // TODO: calculate in QL
-          arg = prefix + "(" + pos + ")" and
-          (
-            prefix = a.getExtendedCanonicalPath()
-            or
-            a = any(OptionEnum o).getSome() and
-            prefix = "crate::option::Option::Some"
-            or
-            exists(string name |
-              a = any(ResultEnum r).getVariant(name) and
-              prefix = "crate::result::Result::" + name
-            )
-          )
+          arg = prefix + "(" + pos + ")" and prefix = a.getCanonicalPath()
         |
           c.(TupleFieldContent).isStructField(a, pos)
           or
           c.(TupleFieldContent).isVariantField(a, pos)
         )
         or
-        exists(Addressable a, string field |
-          // TODO: calculate in QL
-          arg = a.getExtendedCanonicalPath() + "::" + field
-        |
+        exists(Addressable a, string field | arg = a.getCanonicalPath() + "::" + field |
           c.(StructFieldContent).isStructField(a, field)
           or
           c.(StructFieldContent).isVariantField(a, field)

rust/ql/lib/codeql/rust/dataflow/internal/ModelsAsData.qll

@@ -30,9 +30,9 @@
  *     - `ReturnValue`: the value returned by a function call.
  *     - `Element`: an element in a collection.
  *     - `Field[t::f]`: field `f` of the variant/struct with canonical path `t`, for example
- *                      `Field[crate::ihex::Record::Data::value]`.
+ *                      `Field[ihex::Record::Data::value]`.
  *     - `Field[t(i)]`: position `i` inside the variant/struct with canonical path `v`, for example
- *                      `Field[crate::option::Option::Some(0)]`.
+ *                      `Field[core::option::Option::Some(0)]`.
  *     - `Field[i]`: the `i`th element of a tuple.
  * 4. The `kind` column is a tag that can be referenced from QL to determine to
  *    which classes the interpreted elements should be added. For example, for

rust/ql/lib/codeql/rust/frameworks/async-rs.model.yml

@@ -3,4 +3,4 @@ extensions:
       pack: codeql/rust-all
       extensible: sourceModel
     data:
-      - ["repo:https://github.com/async-rs/async-std:async-std", "<crate::net::tcp::stream::TcpStream>::connect", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
+      - ["repo:https://github.com/async-rs/async-std:async-std", "<crate::net::tcp::stream::TcpStream>::connect", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]

rust/ql/lib/codeql/rust/frameworks/futures.model.yml

@@ -13,7 +13,7 @@ extensions:
       - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::read_line", "Argument[self].Reference", "Argument[0].Reference", "taint", "manual"]
       - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::read_until", "Argument[self]", "Argument[1].Reference", "taint", "manual"]
       - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::read_until", "Argument[self].Reference", "Argument[1].Reference", "taint", "manual"]
-      - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::fill_buf", "Argument[self]", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "taint", "manual"]
+      - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::fill_buf", "Argument[self]", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "taint", "manual"]
       - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::io::AsyncBufReadExt::lines", "Argument[self]", "ReturnValue", "taint", "manual"]
-      - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::stream::stream::StreamExt::next", "Argument[self]", "ReturnValue.Future.Field[crate::option::Option::Some(0)]", "taint", "manual"]
-      - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "<crate::io::buf_reader::BufReader as crate::if_std::AsyncBufRead>::poll_fill_buf", "Argument[self].Reference", "ReturnValue.Field[crate::task::poll::Poll::Ready(0)].Field[crate::result::Result::Ok(0)]", "taint", "manual"]
+      - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "crate::stream::stream::StreamExt::next", "Argument[self]", "ReturnValue.Future.Field[core::option::Option::Some(0)]", "taint", "manual"]
+      - ["repo:https://github.com/rust-lang/futures-rs:futures-util", "<crate::io::buf_reader::BufReader as crate::if_std::AsyncBufRead>::poll_fill_buf", "Argument[self].Reference", "ReturnValue.Field[core::task::poll::Poll::Ready(0)].Field[core::result::Result::Ok(0)]", "taint", "manual"]

rust/ql/lib/codeql/rust/frameworks/http.model.yml

@@ -3,10 +3,10 @@ extensions:
       pack: codeql/rust-all
       extensible: sourceModel
     data:
-      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
-      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
-      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::try_send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
-      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::try_send_request", "ReturnValue.Future.Field[crate::result::Result::Ok(0)]", "remote", "manual"]
+      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
+      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
+      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http1::SendRequest>::try_send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
+      - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::conn::http2::SendRequest>::try_send_request", "ReturnValue.Future.Field[core::result::Result::Ok(0)]", "remote", "manual"]
       - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::Client>::get", "ReturnValue.Future", "remote", "manual"]
       - ["repo:https://github.com/hyperium/hyper:hyper", "<crate::client::Client>::request", "ReturnValue.Future", "remote", "manual"]
       - ["repo:https://github.com/hyperium/hyper-util:hyper-util", "<crate::client::legacy::Client>::get", "ReturnValue.Future", "remote", "manual"]

rust/ql/lib/codeql/rust/frameworks/log.model.yml

@@ -15,9 +15,9 @@ extensions:
       - ["lang:std", "<crate::io::stdio::StderrLock as crate::io::Write>::write", "Argument[0]", "log-injection", "manual"]
       - ["lang:std", "<crate::io::stdio::StderrLock as crate::io::Write>::write_all", "Argument[0]", "log-injection", "manual"]
       - ["lang:core", "crate::panicking::panic_fmt", "Argument[0]", "log-injection", "manual"]
-      - ["lang:core", "crate::panicking::assert_failed", "Argument[3].Field[crate::option::Option::Some(0)]", "log-injection", "manual"]
+      - ["lang:core", "crate::panicking::assert_failed", "Argument[3].Field[core::option::Option::Some(0)]", "log-injection", "manual"]
       - ["lang:core", "<crate::option::Option>::expect", "Argument[0]", "log-injection", "manual"]
       - ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::option::Option as crate::LogErrOption>::log_expect", "Argument[0]", "log-injection", "manual"]
-      - ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_unwrap", "Argument[self].Field[crate::result::Result::Err(0)]", "log-injection", "manual"]
+      - ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_unwrap", "Argument[self].Field[core::result::Result::Err(0)]", "log-injection", "manual"]
       - ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_expect", "Argument[0]", "log-injection", "manual"]
-      - ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_expect", "Argument[self].Field[crate::result::Result::Err(0)]", "log-injection", "manual"]
+      - ["repo:https://github.com/DesmondWillowbrook/rs-log_err:log_err", "<crate::result::Result as crate::LogErrResult>::log_expect", "Argument[self].Field[core::result::Result::Err(0)]", "log-injection", "manual"]