Merge pull request #13737 from asgerf/dynamic/fuzzy-models

Dynamic: add Fuzzy token

Author: asgerf

docs/codeql/codeql-language-guides/customizing-library-models-for-javascript.rst

@@ -220,6 +220,59 @@ For example, the **mysql** model that is included with the CodeQL JS analysis in
 
   - ["mysql.Connection", "mysql", "Member[createConnection].ReturnValue"]
 
+Example: Using fuzzy models to simplify modeling
+------------------------------------------------
+
+In this example, we'll show how to add the following SQL injection sink using a "fuzzy" model:
+
+.. code-block:: ts
+
+  import * as mysql from 'mysql';
+  const pool = mysql.createPool({...});
+  pool.getConnection((err, conn) => {
+    conn.query(q, (err, rows) => {...}); // <-- add 'q' as a SQL injection sink
+  });
+
+We can recognize this using a fuzzy model, as shown in the following extension:
+
+.. code-block:: yaml
+
+  extensions:
+    - addsTo:
+        pack: codeql/javascript-all
+        extensible: sinkModel
+      data:
+        - ["mysql", "Fuzzy.Member[query].Argument[0]", "sql-injection"]
+
+- The first column, **"mysql"**, begins the search at places where the `mysql` package is imported.
+- **Fuzzy** selects all objects that appear to originate from the `mysql` package, such as the `pool`, `conn`, `err`, and `rows` objects.
+- **Member[query]** selects the **query** member from any of those objects. In this case, the only such member is `conn.query`.
+  In principle, this would also find expressions such as `pool.query` and `err.query`, but in practice such expressions
+  are not likely to occur, because the `pool` and `err` objects do not have a member named `query`.
+- **Argument[0]** selects the first argument of a call to the selected member, that is, the `q` argument to `conn.query`.
+- **sql-injection** indicates that this is considered as a sink for the SQL injection query.
+
+For reference, a more detailed model might look like this, as described in the preceding examples:
+
+.. code-block:: yaml
+
+  extensions:
+    - addsTo:
+        pack: codeql/javascript-all
+        extensible: sinkModel
+      data:
+        - ["mysql.Connection", "Member[query].Argument[0]", "sql-injection"]
+
+    - addsTo:
+        pack: codeql/javascript-all
+        extensible: typeModel
+      data:
+        - ["mysql.Pool", "mysql", "Member[createPool].ReturnValue"]
+        - ["mysql.Connection", "mysql.Pool", "Member[getConnection].Argument[0].Parameter[1]"]
+
+The model using the **Fuzzy** component is simpler, at the cost of being approximate.
+This technique is useful when modeling a large or complex library, where it is difficult to write a detailed model.
+
 Example: Adding flow through 'decodeURIComponent'
 -------------------------------------------------
 
@@ -431,6 +484,9 @@ The following components are supported:
 - **MapValue** selects a value of a map object.
 - **Awaited** selects the value of a promise.
 - **Instance** selects instances of a class.
+- **Fuzzy** selects all values that are derived from the current value through a combination of the other operations described in this list.
+  For example, this can be used to find all values that appear to originate from a particular package. This can be useful for finding method calls
+  from a known package, but where the receiver type is not known or is difficult to model.
 
 The following components are called "call site filters". They select a subset of the previously-selected calls, if the call fits certain criteria:
 

javascript/ql/lib/semmle/javascript/frameworks/Vue.qll

@@ -20,9 +20,7 @@ module Vue {
   private class VueExportEntryPoint extends API::EntryPoint {
     VueExportEntryPoint() { this = "VueExportEntryPoint" }
 
-    override DataFlow::Node getASink() {
-      result = any(SingleFileComponent c).getModule().getDefaultOrBulkExport()
-    }
+    override DataFlow::Node getASink() { result = getModuleFromVueFile(_).getDefaultOrBulkExport() }
   }
 
   /**
@@ -455,6 +453,13 @@ module Vue {
     }
   }
 
+  private Module getModuleFromVueFile(VueFile file) {
+    exists(HTML::ScriptElement elem |
+      xmlElements(elem, _, _, _, file) and // Avoid materializing all of Locatable.getFile()
+      result.getTopLevel() = elem.getScript()
+    )
+  }
+
   /**
    * A single file Vue component in a `.vue` file.
    */
@@ -482,12 +487,7 @@ module Vue {
     }
 
     /** Gets the module defined by the `script` tag in this .vue file, if any. */
-    Module getModule() {
-      exists(HTML::ScriptElement elem |
-        xmlElements(elem, _, _, _, file) and // Avoid materializing all of Locatable.getFile()
-        result.getTopLevel() = elem.getScript()
-      )
-    }
+    Module getModule() { result = getModuleFromVueFile(file) }
 
     override API::Node getComponentRef() {
       // There is no explicit `new Vue()` call in .vue files, so instead get all the imports

javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll

@@ -454,6 +454,14 @@ private API::Node getNodeFromPath(string type, AccessPath path, int n) {
   or
   // Apply a type step
   typeStep(getNodeFromPath(type, path, n), result)
+  or
+  // Apply a fuzzy step (without advancing 'n')
+  path.getToken(n).getName() = "Fuzzy" and
+  result = Specific::getAFuzzySuccessor(getNodeFromPath(type, path, n))
+  or
+  // Skip a fuzzy step (advance 'n' without changing the current node)
+  path.getToken(n - 1).getName() = "Fuzzy" and
+  result = getNodeFromPath(type, path, n - 1)
 }
 
 /**
@@ -500,6 +508,14 @@ private API::Node getNodeFromSubPath(API::Node base, AccessPath subPath, int n)
   // will themselves find by following type-steps.
   n > 0 and
   n < subPath.getNumToken()
+  or
+  // Apply a fuzzy step (without advancing 'n')
+  subPath.getToken(n).getName() = "Fuzzy" and
+  result = Specific::getAFuzzySuccessor(getNodeFromSubPath(base, subPath, n))
+  or
+  // Skip a fuzzy step (advance 'n' without changing the current node)
+  subPath.getToken(n - 1).getName() = "Fuzzy" and
+  result = getNodeFromSubPath(base, subPath, n - 1)
 }
 
 /**
@@ -561,7 +577,7 @@ private Specific::InvokeNode getInvocationFromPath(string type, AccessPath path)
  */
 bindingset[name]
 private predicate isValidTokenNameInIdentifyingAccessPath(string name) {
-  name = ["Argument", "Parameter", "ReturnValue", "WithArity", "TypeVar"]
+  name = ["Argument", "Parameter", "ReturnValue", "WithArity", "TypeVar", "Fuzzy"]
   or
   Specific::isExtraValidTokenNameInIdentifyingAccessPath(name)
 }
@@ -572,7 +588,7 @@ private predicate isValidTokenNameInIdentifyingAccessPath(string name) {
  */
 bindingset[name]
 private predicate isValidNoArgumentTokenInIdentifyingAccessPath(string name) {
-  name = "ReturnValue"
+  name = ["ReturnValue", "Fuzzy"]
   or
   Specific::isExtraValidNoArgumentTokenInIdentifyingAccessPath(name)
 }

javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -192,6 +192,43 @@ API::Node getExtraSuccessorFromInvoke(API::InvokeNode node, AccessPathToken toke
   result.asSink() = node.(DataFlow::CallNode).getReceiver()
 }
 
+/**
+ * Holds if `name` is the name of a built-in method on Object, Array, or String.
+ */
+private predicate isCommonBuiltinMethodName(string name) {
+  exists(JS::ExternalInstanceMemberDecl member |
+    member.getBaseName() in ["Object", "Array", "String"] and
+    name = member.getName()
+  )
+}
+
+/**
+ * Holds if fuzzy evaluation should not traverse through `call`.
+ */
+private predicate blockFuzzyCall(DataFlow::CallNode call) {
+  isCommonBuiltinMethodName(call.getCalleeName())
+}
+
+pragma[inline]
+API::Node getAFuzzySuccessor(API::Node node) {
+  result = node.getAMember() and
+  // Block traversal into calls to built-ins like .toString() and .substring()
+  // Since there is no API node representing the call itself, block flow into the callee node.
+  not exists(DataFlow::CallNode call |
+    node.asSource() = call.getCalleeNode() and
+    blockFuzzyCall(call)
+  )
+  or
+  result = node.getAParameter()
+  or
+  result = node.getReturn()
+  or
+  result = node.getPromised()
+  or
+  // include 'this' parameters but not 'this' arguments
+  result = node.getReceiver() and result.asSource() instanceof DataFlow::ThisNode
+}
+
 /**
  * Holds if `invoke` matches the JS-specific call site filter in `token`.
  */

javascript/ql/test/library-tests/frameworks/data/test.expected

@@ -66,6 +66,14 @@ taintFlow
 | test.js:231:59:231:66 | source() | test.js:231:59:231:66 | source() |
 | test.js:232:59:232:66 | source() | test.js:232:59:232:66 | source() |
 | test.js:233:59:233:66 | source() | test.js:233:59:233:66 | source() |
+| test.js:237:21:237:28 | source() | test.js:237:21:237:28 | source() |
+| test.js:238:25:238:32 | source() | test.js:238:25:238:32 | source() |
+| test.js:239:27:239:34 | source() | test.js:239:27:239:34 | source() |
+| test.js:241:17:241:24 | source() | test.js:241:17:241:24 | source() |
+| test.js:244:33:244:40 | source() | test.js:244:33:244:40 | source() |
+| test.js:249:28:249:35 | source() | test.js:249:28:249:35 | source() |
+| test.js:252:15:252:22 | source() | test.js:252:15:252:22 | source() |
+| test.js:254:32:254:39 | source() | test.js:254:32:254:39 | source() |
 isSink
 | test.js:54:18:54:25 | source() | test-sink |
 | test.js:55:22:55:29 | source() | test-sink |
@@ -136,6 +144,14 @@ isSink
 | test.js:231:59:231:66 | source() | test-sink |
 | test.js:232:59:232:66 | source() | test-sink |
 | test.js:233:59:233:66 | source() | test-sink |
+| test.js:237:21:237:28 | source() | test-sink |
+| test.js:238:25:238:32 | source() | test-sink |
+| test.js:239:27:239:34 | source() | test-sink |
+| test.js:241:17:241:24 | source() | test-sink |
+| test.js:244:33:244:40 | source() | test-sink |
+| test.js:249:28:249:35 | source() | test-sink |
+| test.js:252:15:252:22 | source() | test-sink |
+| test.js:254:32:254:39 | source() | test-sink |
 syntaxErrors
 | Member[foo |
 | Member[foo] .Member[bar] |

javascript/ql/test/library-tests/frameworks/data/test.js

@@ -232,3 +232,27 @@ function typeVars() {
   testlib.typevar.left.x.getThis().getThis().right.mySink(source()); // NOT OK
   testlib.typevar.left.x.right.getThis().getThis().mySink(source()); // NOT OK
 }
+
+function fuzzy() {
+  testlib.fuzzyCall(source()); // NOT OK
+  testlib.foo.fuzzyCall(source()); // NOT OK
+  testlib.foo().fuzzyCall(source()); // NOT OK
+  new testlib.Blah().foo.bar(async p => {
+    p.fuzzyCall(source()); // NOT OK
+    p.otherCall(source()); // OK
+    p.fuzzyCall().laterMethod(source()); // OK
+    (await p.promise).fuzzyCall(source()); // NOT OK
+  });
+
+  const wrapped = _.partial(testlib.foo, [123]);
+  wrapped().fuzzyCall(source()); // NOT OK [INCONSISTENCY] - API graphs do not currently propagate return values through partial invocation
+  wrapped(p => p.fuzzyCall(source())); // NOT OK
+
+  const wrappedSink = _.partial(testlib.fuzzyCall);
+  wrappedSink(source()); // NOT OK
+
+  _.partial(testlib.fuzzyCall, source()); // NOT OK
+
+  fuzzyCall(source()); // OK - does not come from 'testlib'
+  require('blah').fuzzyCall(source()); // OK - does not come from 'testlib'
+}

javascript/ql/test/library-tests/frameworks/data/test.ql

@@ -54,6 +54,7 @@ class Sinks extends ModelInput::SinkModelCsv {
         "testlib;Member[typevar].TypeVar[ABC].Member[mySink].Argument[0];test-sink",
         "testlib;Member[typevar].TypeVar[ABC].TypeVar[ABC].Member[mySink].Argument[1];test-sink",
         "testlib;Member[typevar].TypeVar[LeftRight].Member[mySink].Argument[0];test-sink",
+        "testlib;Fuzzy.Member[fuzzyCall].Argument[0];test-sink"
       ]
   }
 }

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll

@@ -454,6 +454,14 @@ private API::Node getNodeFromPath(string type, AccessPath path, int n) {
   or
   // Apply a type step
   typeStep(getNodeFromPath(type, path, n), result)
+  or
+  // Apply a fuzzy step (without advancing 'n')
+  path.getToken(n).getName() = "Fuzzy" and
+  result = Specific::getAFuzzySuccessor(getNodeFromPath(type, path, n))
+  or
+  // Skip a fuzzy step (advance 'n' without changing the current node)
+  path.getToken(n - 1).getName() = "Fuzzy" and
+  result = getNodeFromPath(type, path, n - 1)
 }
 
 /**
@@ -500,6 +508,14 @@ private API::Node getNodeFromSubPath(API::Node base, AccessPath subPath, int n)
   // will themselves find by following type-steps.
   n > 0 and
   n < subPath.getNumToken()
+  or
+  // Apply a fuzzy step (without advancing 'n')
+  subPath.getToken(n).getName() = "Fuzzy" and
+  result = Specific::getAFuzzySuccessor(getNodeFromSubPath(base, subPath, n))
+  or
+  // Skip a fuzzy step (advance 'n' without changing the current node)
+  subPath.getToken(n - 1).getName() = "Fuzzy" and
+  result = getNodeFromSubPath(base, subPath, n - 1)
 }
 
 /**
@@ -561,7 +577,7 @@ private Specific::InvokeNode getInvocationFromPath(string type, AccessPath path)
  */
 bindingset[name]
 private predicate isValidTokenNameInIdentifyingAccessPath(string name) {
-  name = ["Argument", "Parameter", "ReturnValue", "WithArity", "TypeVar"]
+  name = ["Argument", "Parameter", "ReturnValue", "WithArity", "TypeVar", "Fuzzy"]
   or
   Specific::isExtraValidTokenNameInIdentifyingAccessPath(name)
 }
@@ -572,7 +588,7 @@ private predicate isValidTokenNameInIdentifyingAccessPath(string name) {
  */
 bindingset[name]
 private predicate isValidNoArgumentTokenInIdentifyingAccessPath(string name) {
-  name = "ReturnValue"
+  name = ["ReturnValue", "Fuzzy"]
   or
   Specific::isExtraValidNoArgumentTokenInIdentifyingAccessPath(name)
 }

python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModelsSpecific.qll

@@ -108,6 +108,23 @@ API::Node getExtraSuccessorFromInvoke(API::CallNode node, AccessPathToken token)
   )
 }
 
+pragma[inline]
+API::Node getAFuzzySuccessor(API::Node node) {
+  result = node.getAMember()
+  or
+  result = node.getParameter(_)
+  or
+  result = node.getKeywordParameter(_)
+  or
+  result = node.getReturn()
+  or
+  result = node.getASubscript()
+  or
+  result = node.getAwaited()
+  or
+  result = node.getASubclass()
+}
+
 /**
  * Holds if `invoke` matches the PY-specific call site filter in `token`.
  */

python/ql/test/library-tests/frameworks/data/test.expected

@@ -11,6 +11,11 @@ taintFlow
 | test.py:83:50:83:60 | ControlFlowNode for getSource() | test.py:83:8:83:61 | ControlFlowNode for Attribute() |
 | test.py:86:49:86:59 | ControlFlowNode for getSource() | test.py:86:8:86:60 | ControlFlowNode for Attribute() |
 | test.py:87:56:87:66 | ControlFlowNode for getSource() | test.py:87:8:87:67 | ControlFlowNode for Attribute() |
+| test.py:114:19:114:29 | ControlFlowNode for getSource() | test.py:114:19:114:29 | ControlFlowNode for getSource() |
+| test.py:115:20:115:30 | ControlFlowNode for getSource() | test.py:115:20:115:30 | ControlFlowNode for getSource() |
+| test.py:116:31:116:41 | ControlFlowNode for getSource() | test.py:116:31:116:41 | ControlFlowNode for getSource() |
+| test.py:117:31:117:41 | ControlFlowNode for getSource() | test.py:117:31:117:41 | ControlFlowNode for getSource() |
+| test.py:118:35:118:45 | ControlFlowNode for getSource() | test.py:118:35:118:45 | ControlFlowNode for getSource() |
 isSink
 | test.py:4:8:4:8 | ControlFlowNode for x | test-sink |
 | test.py:7:17:7:17 | ControlFlowNode for x | test-sink |
@@ -50,6 +55,11 @@ isSink
 | test.py:91:21:91:23 | ControlFlowNode for one | test-sink |
 | test.py:91:30:91:32 | ControlFlowNode for two | test-sink |
 | test.py:98:6:98:9 | ControlFlowNode for baz2 | test-sink |
+| test.py:114:19:114:29 | ControlFlowNode for getSource() | test-sink |
+| test.py:115:20:115:30 | ControlFlowNode for getSource() | test-sink |
+| test.py:116:31:116:41 | ControlFlowNode for getSource() | test-sink |
+| test.py:117:31:117:41 | ControlFlowNode for getSource() | test-sink |
+| test.py:118:35:118:45 | ControlFlowNode for getSource() | test-sink |
 isSource
 | test.py:3:5:3:15 | ControlFlowNode for getSource() | test-source |
 | test.py:9:8:9:14 | ControlFlowNode for alias() | test-source |
@@ -89,6 +99,12 @@ isSource
 | test.py:104:32:104:37 | ControlFlowNode for param2 | test-source |
 | test.py:107:24:107:28 | ControlFlowNode for name1 | test-source |
 | test.py:107:31:107:35 | ControlFlowNode for name2 | test-source |
+| test.py:114:19:114:29 | ControlFlowNode for getSource() | test-source |
+| test.py:115:20:115:30 | ControlFlowNode for getSource() | test-source |
+| test.py:116:31:116:41 | ControlFlowNode for getSource() | test-source |
+| test.py:117:31:117:41 | ControlFlowNode for getSource() | test-source |
+| test.py:118:35:118:45 | ControlFlowNode for getSource() | test-source |
+| test.py:119:20:119:30 | ControlFlowNode for getSource() | test-source |
 syntaxErrors
 | Member[foo |
 | Member[foo] .Member[bar] |