Merge pull request github#11879 from erik-krogh/rbRegConcept

RB: add a RegexExecution concept, and use it for better regexp tracking

Author: erik-krogh

config/identical-files.json

@@ -36,7 +36,6 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForRegExp.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForPathname.qll",
     "swift/ql/lib/codeql/swift/dataflow/internal/DataFlowImpl.qll"

ruby/ql/lib/codeql/ruby/Concepts.qll

@@ -10,6 +10,7 @@ private import codeql.ruby.DataFlow
 private import codeql.ruby.Frameworks
 private import codeql.ruby.dataflow.RemoteFlowSources
 private import codeql.ruby.ApiGraphs
+private import codeql.ruby.Regexp as RE
 
 /**
  * A data-flow node that constructs a SQL statement.
@@ -77,6 +78,49 @@ module SqlExecution {
   }
 }
 
+/**
+ * A data-flow node that executes a regular expression.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `RegexExecution::Range` instead.
+ */
+class RegexExecution extends DataFlow::Node instanceof RegexExecution::Range {
+  /** Gets the data flow node for the regex being executed by this node. */
+  DataFlow::Node getRegex() { result = super.getRegex() }
+
+  /** Gets a dataflow node for the string to be searched or matched against. */
+  DataFlow::Node getString() { result = super.getString() }
+
+  /**
+   * Gets the name of this regex execution, typically the name of an executing method.
+   * This is used for nice alert messages and should include the module if possible.
+   */
+  string getName() { result = super.getName() }
+}
+
+/** Provides classes for modeling new regular-expression execution APIs. */
+module RegexExecution {
+  /**
+   * A data-flow node that executes a regular expression.
+   *
+   * Extend this class to model new APIs. If you want to refine existing API models,
+   * extend `RegexExecution` instead.
+   */
+  abstract class Range extends DataFlow::Node {
+    /** Gets the data flow node for the regex being executed by this node. */
+    abstract DataFlow::Node getRegex();
+
+    /** Gets a dataflow node for the string to be searched or matched against. */
+    abstract DataFlow::Node getString();
+
+    /**
+     * Gets the name of this regex execution, typically the name of an executing method.
+     * This is used for nice alert messages and should include the module if possible.
+     */
+    abstract string getName();
+  }
+}
+
 /**
  * A data flow node that performs a file system access, including reading and writing data,
  * creating and deleting files and folders, checking and updating permissions, and so on.

ruby/ql/lib/codeql/ruby/Regexp.qll

@@ -7,10 +7,12 @@
 
 import regexp.RegExpTreeView // re-export
 private import regexp.internal.ParseRegExp
-private import regexp.internal.RegExpConfiguration
-private import codeql.ruby.ast.Literal as Ast
+private import regexp.internal.RegExpTracking as RegExpTracking
+private import codeql.ruby.AST as Ast
+private import codeql.ruby.CFG
 private import codeql.ruby.DataFlow
 private import codeql.ruby.ApiGraphs
+private import codeql.ruby.Concepts
 
 /**
  * Provides utility predicates related to regular expressions.
@@ -63,7 +65,11 @@ private class RegExpLiteralPatternSource extends RegExpPatternSource {
 private class StringRegExpPatternSource extends RegExpPatternSource {
   private DataFlow::Node parse;
 
-  StringRegExpPatternSource() { this = regExpSource(parse) }
+  StringRegExpPatternSource() {
+    this = regExpSource(parse) and
+    // `regExpSource()` tracks both strings and regex literals, narrow it down to strings.
+    this.asExpr().getConstantValue().isString(_)
+  }
 
   override DataFlow::Node getAParse() { result = parse }
 
@@ -104,6 +110,7 @@ module RegExpInterpretation {
 
 /**
  * A node interpreted as a regular expression.
+ * Speficically nodes where string values are interpreted as regular expressions.
  */
 class StdLibRegExpInterpretation extends RegExpInterpretation::Range {
   StdLibRegExpInterpretation() {
@@ -115,16 +122,100 @@ class StdLibRegExpInterpretation extends RegExpInterpretation::Range {
       mce.getMethodName() = ["match", "match?"] and
       this = mce.getArgument(0) and
       // exclude https://ruby-doc.org/core-2.4.0/Regexp.html#method-i-match
-      not mce.getReceiver() = trackRegexpType()
+      not mce.getReceiver() = RegExpTracking::trackRegexpType()
     )
   }
 }
 
+/**
+ * Holds if `exec` is a node where `regexp` is interpreted as a regular expression and
+ * tested against the string value of `input`.
+ * `name` describes the regexp execution, typically the name of the method being called.
+ */
+private predicate regexExecution(
+  DataFlow::Node exec, DataFlow::Node input, DataFlow::Node regexp, string name
+) {
+  // `=~` or `!~`
+  exists(CfgNodes::ExprNodes::BinaryOperationCfgNode op |
+    name = op.getOperator() and
+    exec.asExpr() = op and
+    (
+      op.getExpr() instanceof Ast::RegExpMatchExpr or
+      op.getExpr() instanceof Ast::NoRegExpMatchExpr
+    ) and
+    (
+      input.asExpr() = op.getLeftOperand() and regexp.asExpr() = op.getRightOperand()
+      or
+      input.asExpr() = op.getRightOperand() and regexp.asExpr() = op.getLeftOperand()
+    )
+  )
+  or
+  // Any of the methods on `String` that take a regexp.
+  exists(DataFlow::CallNode call | exec = call |
+    name = "String#" + call.getMethodName() and
+    call.getMethodName() =
+      [
+        "[]", "gsub", "gsub!", "index", "match", "match?", "partition", "rindex", "rpartition",
+        "scan", "slice!", "split", "sub", "sub!"
+      ] and
+    input = call.getReceiver() and
+    regexp = call.getArgument(0) and
+    // exclude https://ruby-doc.org/core-2.4.0/Regexp.html#method-i-match, they are handled on the next case of this disjunction
+    // also see `StdLibRegExpInterpretation`
+    not (
+      call.getMethodName() = ["match", "match?"] and
+      call.getReceiver() = RegExpTracking::trackRegexpType()
+    )
+  )
+  or
+  // A call to `match` or `match?` where the regexp is the receiver.
+  exists(DataFlow::CallNode call | exec = call |
+    name = "Regexp#" + call.getMethodName() and
+    call.getMethodName() = ["match", "match?"] and
+    regexp = call.getReceiver() and
+    input = call.getArgument(0)
+  )
+  or
+  // a case-when statement
+  exists(CfgNodes::ExprNodes::CaseExprCfgNode caseExpr |
+    exec.asExpr() = caseExpr and
+    input.asExpr() = caseExpr.getValue()
+  |
+    name = "case-when" and
+    regexp.asExpr() = caseExpr.getBranch(_).(CfgNodes::ExprNodes::WhenClauseCfgNode).getPattern(_)
+    or
+    name = "case-in" and
+    regexp.asExpr() = caseExpr.getBranch(_).(CfgNodes::ExprNodes::InClauseCfgNode).getPattern()
+  )
+}
+
+/**
+ * An execution of a regular expression by the standard library.
+ */
+private class StdRegexpExecution extends RegexExecution::Range {
+  DataFlow::Node regexp;
+  DataFlow::Node input;
+  string name;
+
+  StdRegexpExecution() { regexExecution(this, input, regexp, name) }
+
+  override DataFlow::Node getRegex() { result = regexp }
+
+  override DataFlow::Node getString() { result = input }
+
+  override string getName() { result = name }
+}
+
 /**
  * Gets a node whose value may flow (inter-procedurally) to `re`, where it is interpreted
  * as a part of a regular expression.
  */
 cached
-DataFlow::Node regExpSource(DataFlow::Node re) {
-  exists(RegExpConfiguration c | c.hasFlow(result, re))
+DataFlow::Node regExpSource(DataFlow::Node re) { result = RegExpTracking::regExpSource(re) }
+
+/** Gets a parsed regular expression term that is executed at `exec`. */
+RegExpTerm getTermForExecution(RegexExecution exec) {
+  exists(RegExpPatternSource source | source = regExpSource(exec.getRegex()) |
+    result = source.getRegExpTerm()
+  )
 }