C#: Update model conversion queries

Author: hvitved

csharp/ql/lib/semmle/code/csharp/dataflow/internal/ExternalFlow.qll

@@ -371,46 +371,49 @@ private string paramsString(InterpretedCallable c) {
     )
 }
 
+/** Gets the source/sink/summary/neutral base declaration corresponding to the supplied parameters. */
 pragma[nomagic]
-private Element interpretElement0(
-  string namespace, string type, boolean subtypes, string name, string signature
-) {
-  exists(UnboundValueOrRefType t | elementSpec(namespace, type, subtypes, name, signature, _, t) |
-    exists(Declaration m |
-      (
-        result = m
-        or
-        subtypes = true and result.(UnboundCallable).overridesOrImplementsUnbound(m)
-      ) and
-      m.getDeclaringType() = t and
-      hasName(m, name)
-    |
-      signature = ""
-      or
-      paramsString(m) = signature
-    )
+Declaration interpretBaseDeclaration(string namespace, string type, string name, string signature) {
+  exists(UnboundValueOrRefType t | elementSpec(namespace, type, _, name, signature, _, t) |
+    result =
+      any(Declaration d |
+        d.getDeclaringType() = t and
+        hasName(d, name) and
+        (
+          signature = ""
+          or
+          paramsString(d) = signature
+        )
+      )
     or
-    (
-      result = t
-      or
-      subtypes = true and
-      result = t.getASubTypeUnbound+()
-    ) and
     result = t and
     name = "" and
     signature = ""
   )
 }
 
 /** Gets the source/sink/summary/neutral element corresponding to the supplied parameters. */
+pragma[nomagic]
 Element interpretElement(
   string namespace, string type, boolean subtypes, string name, string signature, string ext
 ) {
   elementSpec(namespace, type, subtypes, name, signature, ext) and
-  exists(Element e | e = interpretElement0(namespace, type, subtypes, name, signature) |
-    ext = "" and result = e
+  exists(Declaration base, Declaration d |
+    base = interpretBaseDeclaration(namespace, type, name, signature) and
+    (
+      d = base
+      or
+      subtypes = true and
+      (
+        d.(UnboundCallable).overridesOrImplementsUnbound(base)
+        or
+        d = base.(UnboundValueOrRefType).getASubTypeUnbound+()
+      )
+    )
+  |
+    ext = "" and result = d
     or
-    ext = "Attribute" and result.(Attributable).getAnAttribute().getType() = e
+    ext = "Attribute" and result.(Attributable).getAnAttribute().getType() = d
   )
 }
 
@@ -491,7 +494,7 @@ string parameterQualifiedTypeNamesToString(Callable c) {
     concat(int i, string s | s = parameterQualifiedType(c.getParameter(i)) | s, "," order by i)
 }
 
-private predicate partialModel(
+predicate partialModel(
   UnboundCallable c, string namespace, string type, string name, string parameters
 ) {
   QN::hasQualifiedName(c, namespace, type, name) and

csharp/ql/src/utils/modelconverter/ConvertExtensions.py

@@ -27,7 +27,15 @@
 dbDir = os.path.join(workDir, "db")
 
 # Make dummy project
-helpers.run_cmd(['dotnet', 'new', 'console', '-o', projectDir], "Failed to create dummy project.")
+helpers.run_cmd(['dotnet', 'new', 'webapp', '-o', projectDir], "Failed to create dummy project.")
+# Add nuget packages for all packages where we have models
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Newtonsoft.Json'])
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Microsoft.EntityFrameworkCore'])
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Microsoft.EntityFrameworkCore.Relational'])
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'Dapper'])
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'ServiceStack'])
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'ServiceStack.OrmLite'])
+helpers.run_cmd(['dotnet', 'add', projectDir, 'package', 'System.Collections.Immutable'])
 helpers.run_cmd(['codeql', 'database', 'create', f'--language={language}', '-c', f'dotnet build {projectDir}', dbDir], "Failed to create dummy database.")
 
 print('Converting data extensions for C#.')

csharp/ql/src/utils/modelconverter/ExtractNeutrals.ql

@@ -5,11 +5,13 @@
  */
 
 import csharp
-import semmle.code.csharp.dataflow.internal.ExternalFlow
+import InterpretModel
 
-from string package, string type, string name, string signature, string kind, string provenance
+from
+  string namespace0, string namespace, string type0, string type, string name0, string name,
+  string signature0, string signature, string kind, string provenance
 where
-  neutralModel(package, type, name, signature, kind, provenance) and
-  not provenance.matches("%generated")
-select package, type, name, signature, kind, provenance order by
-    package, type, name, signature, kind
+  neutralModel(namespace0, type0, name0, signature0, kind, provenance) and
+  interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature)
+select namespace, type, name, signature, kind, provenance order by
+    namespace, type, name, signature, kind

csharp/ql/src/utils/modelconverter/ExtractSinks.ql

@@ -5,13 +5,14 @@
  */
 
 import csharp
-import semmle.code.csharp.dataflow.internal.ExternalFlow
+import InterpretModel
 
 from
-  string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string input, string kind, string provenance
+  string namespace0, string namespace, string type0, string type, boolean subtypes, string name0,
+  string name, string signature0, string signature, string ext, string input, string kind,
+  string provenance
 where
-  sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, provenance) and
-  not provenance.matches("%generated")
+  sinkModel(namespace0, type0, subtypes, name0, signature0, ext, input, kind, provenance) and
+  interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature)
 select namespace, type, subtypes, name, signature, ext, input, kind, provenance order by
     namespace, type, name, signature, input, kind

csharp/ql/src/utils/modelconverter/ExtractSources.ql

@@ -5,13 +5,14 @@
  */
 
 import csharp
-import semmle.code.csharp.dataflow.internal.ExternalFlow
+import InterpretModel
 
 from
-  string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string output, string kind, string provenance
+  string namespace0, string namespace, string type0, string type, boolean subtypes, string name0,
+  string name, string signature0, string signature, string ext, string output, string kind,
+  string provenance
 where
-  sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, provenance) and
-  not provenance.matches("%generated")
+  sourceModel(namespace0, type0, subtypes, name0, signature0, ext, output, kind, provenance) and
+  interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature)
 select namespace, type, subtypes, name, signature, ext, output, kind, provenance order by
     namespace, type, name, signature, output, kind

csharp/ql/src/utils/modelconverter/ExtractSummaries.ql

@@ -5,13 +5,14 @@
  */
 
 import csharp
-import semmle.code.csharp.dataflow.internal.ExternalFlow
+import InterpretModel
 
 from
-  string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string input, string output, string kind, string provenance
+  string namespace0, string namespace, string type0, string type, boolean subtypes, string name0,
+  string name, string signature0, string signature, string ext, string input, string output,
+  string kind, string provenance
 where
-  summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and
-  not provenance.matches("%generated")
+  summaryModel(namespace0, type0, subtypes, name0, signature0, ext, input, output, kind, provenance) and
+  interpretCallable(namespace0, namespace, type0, type, name0, name, signature0, signature)
 select namespace, type, subtypes, name, signature, ext, input, output, kind, provenance order by
     namespace, type, name, signature, input, output, kind

csharp/ql/src/utils/modelconverter/InterpretModel.qll

@@ -0,0 +1,21 @@
+import csharp
+import semmle.code.csharp.dataflow.internal.ExternalFlow
+
+bindingset[namespace0, type0, name0, signature0]
+predicate interpretCallable(
+  string namespace0, string namespace, string type0, string type, string name0, string name,
+  string signature0, string signature
+) {
+  exists(Callable c, string signature1 |
+    c = interpretBaseDeclaration(namespace0, type0, name0, signature0) and
+    partialModel(c, namespace, type, name, signature1) and
+    if signature0 = "" then signature = "" else signature = signature1
+  )
+  or
+  // if the row cannot be parsed (e.g. if the element is not in the DB), return the existing row unchanged
+  not exists(interpretBaseDeclaration(namespace0, type0, name0, signature0)) and
+  namespace = namespace0 and
+  type = type0 and
+  name = name0 and
+  signature = signature0
+}

misc/scripts/models-as-data/convert_extensions.py

@@ -29,10 +29,12 @@ def merge(*dicts):
     return merged
 
 def parseData(data):
-    rows = { }
+    rows = [{ }, { }]
     for row in data:
         d = map(quote_if_needed, row)
-        insert_update(rows, row[0], "      - [" + ', '.join(d) + ']\n')
+        provenance = row[-1]
+        targetRows = rows[1] if provenance.endswith("generated") else rows[0]
+        insert_update(targetRows, row[0], "      - [" + ', '.join(d) + ']\n')
 
     return rows
 
@@ -57,9 +59,10 @@ def runQuery(self, query):
 
 
     def asAddsTo(self, rows, predicate):
-        extensions = { }
-        for key in rows:
-            extensions[key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[key])
+        extensions = [{ }, { }]
+        for i in range(2):
+            for key in rows[i]:
+                extensions[i][key] = helpers.addsToTemplate.format(f"codeql/{self.language}-all", predicate, rows[i][key])
 
         return extensions
 
@@ -75,7 +78,7 @@ def makeContent(self):
         sources = self.getAddsTo("ExtractSources.ql", helpers.sourceModelPredicate)
         sinks = self.getAddsTo("ExtractSinks.ql", helpers.sinkModelPredicate)
         neutrals = self.getAddsTo("ExtractNeutrals.ql", helpers.neutralModelPredicate)
-        return merge(sources, sinks, summaries, neutrals)
+        return [merge(sources[0], sinks[0], summaries[0], neutrals[0]), merge(sources[1], sinks[1], summaries[1], neutrals[1])]
 
 
     def save(self, extensions):
@@ -85,9 +88,13 @@ def save(self, extensions):
         # Create a file for each namespace and save models.
         extensionTemplate = """extensions:
 {0}"""
-        for entry in extensions:
+        for entry in extensions[0]:
             with open(self.extDir + "/" + entry + self.modelFileExtension, "w") as f:
-                f.write(extensionTemplate.format(extensions[entry]))
+                f.write(extensionTemplate.format(extensions[0][entry]))
+        
+        for entry in extensions[1]:
+            with open(self.extDir + "/generated/" + entry + self.modelFileExtension, "w") as f:
+                f.write(extensionTemplate.format(extensions[1][entry]))
 
     def run(self):
         extensions = self.makeContent()