Merge pull request #11699 from erik-krogh/shareHost

Dynamic: Share more regexp code

Author: yoff

config/identical-files.json

@@ -531,11 +531,6 @@
     "ruby/ql/lib/codeql/ruby/internal/ConceptsShared.qll",
     "javascript/ql/lib/semmle/javascript/internal/ConceptsShared.qll"
   ],
-  "Hostname Regexp queries": [
-    "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
-    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
-    "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
-  ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
     "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",

java/ql/lib/semmle/code/java/regex/RegexTreeView.qll

@@ -85,6 +85,9 @@ module Impl implements RegexTreeViewSig {
 
     /** Gets the associated regex. */
     abstract Regex getRegex();
+
+    /** Gets the last child term of this element. */
+    RegExpTerm getLastChild() { result = this.getChild(this.getNumChild() - 1) }
   }
 
   /**
@@ -558,13 +561,31 @@ module Impl implements RegexTreeViewSig {
     }
   }
 
+  /**
+   * A character escape in a regular expression.
+   *
+   * Example:
+   *
+   * ```
+   * \.
+   * ```
+   */
+  class RegExpCharEscape = RegExpEscape;
+
   /**
    * A word boundary, that is, a regular expression term of the form `\b`.
    */
   class RegExpWordBoundary extends RegExpSpecialChar {
     RegExpWordBoundary() { this.getChar() = "\\b" }
   }
 
+  /**
+   * A non-word boundary, that is, a regular expression term of the form `\B`.
+   */
+  class RegExpNonWordBoundary extends RegExpSpecialChar {
+    RegExpNonWordBoundary() { this.getChar() = "\\B" }
+  }
+
   /**
    * Gets the hex number for the `hex` char.
    */
@@ -868,6 +889,9 @@ module Impl implements RegexTreeViewSig {
     predicate isNamedGroupOfLiteral(RegExpLiteral lit, string name) {
       lit = this.getLiteral() and name = this.getName()
     }
+
+    /** Holds if this is a capture group. */
+    predicate isCapture() { exists(this.getNumber()) }
   }
 
   /**
@@ -917,6 +941,21 @@ module Impl implements RegexTreeViewSig {
     override string getPrimaryQLClass() { result = "RegExpDot" }
   }
 
+  /**
+   * A term that matches a specific position between characters in the string.
+   *
+   * Example:
+   *
+   * ```
+   * ^
+   * ```
+   */
+  class RegExpAnchor extends RegExpSpecialChar {
+    RegExpAnchor() { this.getChar() = ["$", "^"] }
+
+    override string getPrimaryQLClass() { result = "RegExpAnchor" }
+  }
+
   /**
    * A dollar assertion `$` matching the end of a line.
    *
@@ -926,7 +965,7 @@ module Impl implements RegexTreeViewSig {
    * $
    * ```
    */
-  class RegExpDollar extends RegExpSpecialChar {
+  class RegExpDollar extends RegExpAnchor {
     RegExpDollar() { this.getChar() = "$" }
 
     override string getPrimaryQLClass() { result = "RegExpDollar" }
@@ -941,7 +980,7 @@ module Impl implements RegexTreeViewSig {
    * ^
    * ```
    */
-  class RegExpCaret extends RegExpSpecialChar {
+  class RegExpCaret extends RegExpAnchor {
     RegExpCaret() { this.getChar() = "^" }
 
     override string getPrimaryQLClass() { result = "RegExpCaret" }

javascript/ql/lib/semmle/javascript/Regexp.qll

@@ -366,6 +366,9 @@ class RegExpAnchor extends RegExpTerm, @regexp_anchor {
   override predicate isNullable() { any() }
 
   override string getAPrimaryQlClass() { result = "RegExpAnchor" }
+
+  /** Gets the char for this term. */
+  abstract string getChar();
 }
 
 /**
@@ -379,6 +382,8 @@ class RegExpAnchor extends RegExpTerm, @regexp_anchor {
  */
 class RegExpCaret extends RegExpAnchor, @regexp_caret {
   override string getAPrimaryQlClass() { result = "RegExpCaret" }
+
+  override string getChar() { result = "^" }
 }
 
 /**
@@ -392,6 +397,8 @@ class RegExpCaret extends RegExpAnchor, @regexp_caret {
  */
 class RegExpDollar extends RegExpAnchor, @regexp_dollar {
   override string getAPrimaryQlClass() { result = "RegExpDollar" }
+
+  override string getChar() { result = "$" }
 }
 
 /**
@@ -999,11 +1006,12 @@ predicate isInterpretedAsRegExp(DataFlow::Node source) {
 /**
  * Provides utility predicates related to regular expressions.
  */
-module RegExpPatterns {
+deprecated module RegExpPatterns {
   /**
    * Gets a pattern that matches common top-level domain names in lower case.
+   * DEPRECATED: use the similarly named predicate from `HostnameRegex` from the `regex` pack instead.
    */
-  string getACommonTld() {
+  deprecated string getACommonTld() {
     // according to ranking by http://google.com/search?q=site:.<<TLD>>
     result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
   }

javascript/ql/lib/semmle/javascript/security/regexp/HostnameRegexp.qll

@@ -0,0 +1,18 @@
+/**
+ * Provides predicates for reasoning about regular expressions
+ * that match URLs and hostname patterns.
+ */
+
+private import javascript as JS
+private import semmle.javascript.security.regexp.RegExpTreeView::RegExpTreeView as TreeImpl
+private import semmle.javascript.Regexp as RegExp
+private import codeql.regex.HostnameRegexp as Shared
+
+/** An implementation of the signature that allows the Hostname analysis to run. */
+module Impl implements Shared::HostnameRegexpSig<TreeImpl> {
+  class DataFlowNode = JS::DataFlow::Node;
+
+  class RegExpPatternSource = RegExp::RegExpPatternSource;
+}
+
+import Shared::Make<TreeImpl, Impl>

javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll

@@ -3,200 +3,5 @@
  * that match URLs and hostname patterns.
  */
 
-private import HostnameRegexpSpecific
-
-/**
- * Holds if the given constant is unlikely to occur in the origin part of a URL.
- */
-predicate isConstantInvalidInsideOrigin(RegExpConstant term) {
-  // Look for any of these cases:
-  // - A character that can't occur in the origin
-  // - Two dashes in a row
-  // - A colon that is not part of port or scheme separator
-  // - A slash that is not part of scheme separator
-  term.getValue().regexpMatch(".*(?:[^a-zA-Z0-9.:/-]|--|:[^0-9/]|(?<![/:]|^)/).*")
-}
-
-/** Holds if `term` is a dot constant of form `\.` or `[.]`. */
-predicate isDotConstant(RegExpTerm term) {
-  term.(RegExpCharEscape).getValue() = "."
-  or
-  exists(RegExpCharacterClass cls |
-    term = cls and
-    not cls.isInverted() and
-    cls.getNumChild() = 1 and
-    cls.getAChild().(RegExpConstant).getValue() = "."
-  )
-}
-
-/** Holds if `term` is a wildcard `.` or an actual `.` character. */
-predicate isDotLike(RegExpTerm term) {
-  term instanceof RegExpDot
-  or
-  isDotConstant(term)
-}
-
-/** Holds if `term` will only ever be matched against the beginning of the input. */
-predicate matchesBeginningOfString(RegExpTerm term) {
-  term.isRootTerm()
-  or
-  exists(RegExpTerm parent | matchesBeginningOfString(parent) |
-    term = parent.(RegExpSequence).getChild(0)
-    or
-    parent.(RegExpSequence).getChild(0) instanceof RegExpCaret and
-    term = parent.(RegExpSequence).getChild(1)
-    or
-    term = parent.(RegExpAlt).getAChild()
-    or
-    term = parent.(RegExpGroup).getAChild()
-  )
-}
-
-/**
- * Holds if the given sequence `seq` contains top-level domain preceded by a dot, such as `.com`,
- * excluding cases where this is at the very beginning of the regexp.
- *
- * `i` is bound to the index of the last child in the top-level domain part.
- */
-predicate hasTopLevelDomainEnding(RegExpSequence seq, int i) {
-  seq.getChild(i)
-      .(RegExpConstant)
-      .getValue()
-      .regexpMatch("(?i)" + RegExpPatterns::getACommonTld() + "(:\\d+)?([/?#].*)?") and
-  isDotLike(seq.getChild(i - 1)) and
-  not (i = 1 and matchesBeginningOfString(seq))
-}
-
-/**
- * Holds if the given regular expression term contains top-level domain preceded by a dot,
- * such as `.com`.
- */
-predicate hasTopLevelDomainEnding(RegExpSequence seq) { hasTopLevelDomainEnding(seq, _) }
-
-/**
- * Holds if `term` will always match a hostname, that is, all disjunctions contain
- * a hostname pattern that isn't inside a quantifier.
- */
-predicate alwaysMatchesHostname(RegExpTerm term) {
-  hasTopLevelDomainEnding(term, _)
-  or
-  // `localhost` is considered a hostname pattern, but has no TLD
-  term.(RegExpConstant).getValue().regexpMatch("\\blocalhost\\b")
-  or
-  not term instanceof RegExpAlt and
-  not term instanceof RegExpQuantifier and
-  alwaysMatchesHostname(term.getAChild())
-  or
-  alwaysMatchesHostnameAlt(term)
-}
-
-/** Holds if every child of `alt` contains a hostname pattern. */
-predicate alwaysMatchesHostnameAlt(RegExpAlt alt) {
-  alwaysMatchesHostnameAlt(alt, alt.getNumChild() - 1)
-}
-
-/**
- * Holds if the first `i` children of `alt` contains a hostname pattern.
- *
- * This is used instead of `forall` to avoid materializing the set of alternatives
- * that don't contains hostnames, which is much larger.
- */
-predicate alwaysMatchesHostnameAlt(RegExpAlt alt, int i) {
-  alwaysMatchesHostname(alt.getChild(0)) and i = 0
-  or
-  alwaysMatchesHostnameAlt(alt, i - 1) and
-  alwaysMatchesHostname(alt.getChild(i))
-}
-
-/**
- * Holds if `term` occurs inside a quantifier or alternative (and thus
- * can not be expected to correspond to a unique match), or as part of
- * a lookaround assertion (which are rarely used for capture groups).
- */
-predicate isInsideChoiceOrSubPattern(RegExpTerm term) {
-  exists(RegExpParent parent | parent = term.getParent() |
-    parent instanceof RegExpAlt
-    or
-    parent instanceof RegExpQuantifier
-    or
-    parent instanceof RegExpSubPattern
-    or
-    isInsideChoiceOrSubPattern(parent)
-  )
-}
-
-/**
- * Holds if `group` is likely to be used as a capture group.
- */
-predicate isLikelyCaptureGroup(RegExpGroup group) {
-  group.isCapture() and
-  not isInsideChoiceOrSubPattern(group)
-}
-
-/**
- * Holds if `seq` contains two consecutive dots `..` or escaped dots.
- *
- * At least one of these dots is not intended to be a subdomain separator,
- * so we avoid flagging the pattern in this case.
- */
-predicate hasConsecutiveDots(RegExpSequence seq) {
-  exists(int i |
-    isDotLike(seq.getChild(i)) and
-    isDotLike(seq.getChild(i + 1))
-  )
-}
-
-predicate isIncompleteHostNameRegExpPattern(RegExpTerm regexp, RegExpSequence seq, string msg) {
-  seq = regexp.getAChild*() and
-  exists(RegExpDot unescapedDot, int i, string hostname |
-    hasTopLevelDomainEnding(seq, i) and
-    not isConstantInvalidInsideOrigin(seq.getChild([0 .. i - 1]).getAChild*()) and
-    not isLikelyCaptureGroup(seq.getChild([i .. seq.getNumChild() - 1]).getAChild*()) and
-    unescapedDot = seq.getChild([0 .. i - 1]).getAChild*() and
-    unescapedDot != seq.getChild(i - 1) and // Should not be the '.' immediately before the TLD
-    not hasConsecutiveDots(unescapedDot.getParent()) and
-    hostname =
-      seq.getChild(i - 2).getRawValue() + seq.getChild(i - 1).getRawValue() +
-        seq.getChild(i).getRawValue()
-  |
-    if unescapedDot.getParent() instanceof RegExpQuantifier
-    then
-      // `.*\.example.com` can match `evil.com/?x=.example.com`
-      //
-      // This problem only occurs when the pattern is applied against a full URL, not just a hostname/origin.
-      // We therefore check if the pattern includes a suffix after the TLD, such as `.*\.example.com/`.
-      // Note that a post-anchored pattern (`.*\.example.com$`) will usually fail to match a full URL,
-      // and patterns with neither a suffix nor an anchor fall under the purview of MissingRegExpAnchor.
-      seq.getChild(0) instanceof RegExpCaret and
-      not seq.getAChild() instanceof RegExpDollar and
-      seq.getChild([i .. i + 1]).(RegExpConstant).getValue().regexpMatch(".*[/?#].*") and
-      msg =
-        "has an unrestricted wildcard '" + unescapedDot.getParent().(RegExpQuantifier).getRawValue()
-          + "' which may cause '" + hostname +
-          "' to be matched anywhere in the URL, outside the hostname."
-    else
-      msg =
-        "has an unescaped '.' before '" + hostname +
-          "', so it might match more hosts than expected."
-  )
-}
-
-predicate incompleteHostnameRegExp(
-  RegExpSequence hostSequence, string message, DataFlow::Node aux, string label
-) {
-  exists(RegExpPatternSource re, RegExpTerm regexp, string msg, string kind |
-    regexp = re.getRegExpTerm() and
-    isIncompleteHostNameRegExpPattern(regexp, hostSequence, msg) and
-    (
-      if re.getAParse() != re
-      then (
-        kind = "string, which is used as a regular expression $@," and
-        aux = re.getAParse()
-      ) else (
-        kind = "regular expression" and aux = re
-      )
-    )
-  |
-    message = "This " + kind + " " + msg and label = "here"
-  )
-}
+deprecated import semmle.javascript.security.regexp.HostnameRegexp as Dep
+import Dep

javascript/ql/src/Security/CWE-020/HostnameRegexpSpecific.qll

@@ -11,6 +11,6 @@
  *       external/cwe/cwe-020
  */
 
-import HostnameRegexpShared
+private import semmle.javascript.security.regexp.HostnameRegexp as HostnameRegexp
 
-query predicate problems = incompleteHostnameRegExp/4;
+query predicate problems = HostnameRegexp::incompleteHostnameRegExp/4;

javascript/ql/src/Security/CWE-020/IncompleteHostnameRegExp.ql

@@ -3,6 +3,7 @@
  */
 
 private import IncompleteUrlSubstringSanitizationSpecific
+private import codeql.regex.HostnameRegexp::Utils
 
 /**
  * A check on a string for whether it contains a given substring, possibly with restrictions on the location of the substring.
@@ -30,9 +31,7 @@ query predicate problems(
   mayHaveStringValue(substring, target) and
   (
     // target contains a domain on a common TLD, and perhaps some other URL components
-    target
-        .regexpMatch("(?i)([a-z]*:?//)?\\.?([a-z0-9-]+\\.)+" + RegExpPatterns::getACommonTld() +
-            "(:[0-9]+)?/?")
+    target.regexpMatch("(?i)([a-z]*:?//)?\\.?([a-z0-9-]+\\.)+" + getACommonTld() + "(:[0-9]+)?/?")
     or
     // target is a HTTP URL to a domain on any TLD
     target.regexpMatch("(?i)https?://([a-z0-9-]+\\.)+([a-z]+)(:[0-9]+)?/?")