Merge branch 'main' into rb/fix-spurious-singleton-calls

Author: asgerf

docs/codeql/support/reusables/frameworks.rst

@@ -221,11 +221,15 @@ and the CodeQL library pack ``codeql/python-all`` (`changelog <https://github.co
    aiopg, Database
    asyncpg, Database
    clickhouse-driver, Database
+   cx_Oracle, Database
    mysql-connector-python, Database
    mysql-connector, Database
    MySQL-python, Database
    mysqlclient, Database
+   oracledb, Database
+   phoenixdb, Database
    psycopg2, Database
+   pyodbc, Database
    pymssql, Database
    PyMySQL, Database
    sqlite3, Database
@@ -241,3 +245,19 @@ and the CodeQL library pack ``codeql/python-all`` (`changelog <https://github.co
    libxml2, XML processing library
    lxml, XML processing library
    xmltodict, XML processing library
+
+
+Ruby built-in support
+====================================
+
+Provided by the current versions of the
+CodeQL query pack ``codeql/ruby-queries`` (`changelog <https://github.com/github/codeql/tree/codeql-cli/latest/ruby/ql/src/CHANGELOG.md>`__, `source <https://github.com/github/codeql/tree/codeql-cli/latest/ruby/ql/src>`__)
+and the CodeQL library pack ``codeql/ruby-all`` (`changelog <https://github.com/github/codeql/tree/codeql-cli/latest/ruby/ql/lib/CHANGELOG.md>`__, `source <https://github.com/github/codeql/tree/codeql-cli/latest/ruby/ql/lib>`__).
+
+.. csv-table::
+   :header-rows: 1
+   :class: fullWidthTable
+   :widths: auto
+
+   Name, Category
+   Ruby on Rails, Web framework

docs/codeql/support/reusables/versions-compilers.rst

@@ -22,7 +22,7 @@
    Eclipse compiler for Java (ECJ) [5]_",``.java``
    JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [6]_"
    Python [7]_,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10",Not applicable,``.py``
-   Ruby [8]_,"up to 3.0.2",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
+   Ruby [8]_,"up to 3.1",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
    TypeScript [9]_,"2.6-4.8",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"
 
 .. container:: footnote-group

java/kotlin-extractor/src/main/kotlin/KotlinExtractorExtension.kt

@@ -74,7 +74,7 @@ class KotlinExtractorExtension(
             // First, if we can find our log directory, then let's try
             // making a log file there:
             val extractorLogDir = System.getenv("CODEQL_EXTRACTOR_JAVA_LOG_DIR")
-            if (extractorLogDir != null || extractorLogDir != "") {
+            if (extractorLogDir != null && extractorLogDir != "") {
                 // We use a slightly different filename pattern compared
                 // to normal logs. Just the existence of a `-top` log is
                 // a sign that something's gone very wrong.
@@ -296,7 +296,9 @@ private fun doFile(
         context.clear()
     }
 
-    val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFilePath")
+    val srcFileRelativePath = srcFilePath.replace(':', '_')
+
+    val dbSrcFilePath = Paths.get("$dbSrcDir/$srcFileRelativePath")
     val dbSrcDirPath = dbSrcFilePath.parent
     Files.createDirectories(dbSrcDirPath)
     val srcTmpFile = File.createTempFile(dbSrcFilePath.fileName.toString() + ".", ".src.tmp", dbSrcDirPath.toFile())
@@ -305,7 +307,7 @@ private fun doFile(
     }
     srcTmpFile.renameTo(dbSrcFilePath.toFile())
 
-    val trapFileName = "$dbTrapDir/$srcFilePath.trap"
+    val trapFileName = "$dbTrapDir/$srcFileRelativePath.trap"
     val trapFileWriter = getTrapFileWriter(compression, logger, trapFileName)
 
     if (checkTrapIdentical || !trapFileWriter.exists()) {

java/ql/lib/change-notes/2022-10-13-stream-collect.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added support for common patterns involving `Stream.collect` and common collectors like `Collectors.toList()`.

java/ql/lib/semmle/code/java/dataflow/ExternalFlow.qll

@@ -75,7 +75,7 @@ import java
 private import semmle.code.java.dataflow.DataFlow::DataFlow
 private import internal.DataFlowPrivate
 private import internal.FlowSummaryImpl::Private::External
-private import internal.FlowSummaryImplSpecific
+private import internal.FlowSummaryImplSpecific as FlowSummaryImplSpecific
 private import internal.AccessPathSyntax
 private import FlowSummary
 
@@ -834,7 +834,7 @@ private module Cached {
    */
   cached
   predicate sourceNode(Node node, string kind) {
-    exists(InterpretNode n | isSourceNode(n, kind) and n.asNode() = node)
+    exists(FlowSummaryImplSpecific::InterpretNode n | isSourceNode(n, kind) and n.asNode() = node)
   }
 
   /**
@@ -843,7 +843,7 @@ private module Cached {
    */
   cached
   predicate sinkNode(Node node, string kind) {
-    exists(InterpretNode n | isSinkNode(n, kind) and n.asNode() = node)
+    exists(FlowSummaryImplSpecific::InterpretNode n | isSinkNode(n, kind) and n.asNode() = node)
   }
 }
 

java/ql/lib/semmle/code/java/dataflow/FlowSummary.qll

@@ -4,7 +4,6 @@
 
 import java
 private import internal.FlowSummaryImpl as Impl
-private import internal.DataFlowDispatch
 private import internal.DataFlowUtil
 
 // import all instances of SummarizedCallable below
@@ -24,6 +23,12 @@ module SummaryComponent {
   /** Gets a summary component for field `f`. */
   SummaryComponent field(Field f) { result = content(any(FieldContent c | c.getField() = f)) }
 
+  /** Gets a summary component for `Element`. */
+  SummaryComponent element() { result = content(any(CollectionContent c)) }
+
+  /** Gets a summary component for `MapValue`. */
+  SummaryComponent mapValue() { result = content(any(MapValueContent c)) }
+
   /** Gets a summary component that represents the return value of a call. */
   SummaryComponent return() { result = return(_) }
 }
@@ -42,10 +47,129 @@ module SummaryComponentStack {
     result = push(SummaryComponent::field(f), object)
   }
 
+  /** Gets a stack representing `Element` of `object`. */
+  SummaryComponentStack elementOf(SummaryComponentStack object) {
+    result = push(SummaryComponent::element(), object)
+  }
+
+  /** Gets a stack representing `MapValue` of `object`. */
+  SummaryComponentStack mapValueOf(SummaryComponentStack object) {
+    result = push(SummaryComponent::mapValue(), object)
+  }
+
   /** Gets a singleton stack representing a (normal) return. */
   SummaryComponentStack return() { result = singleton(SummaryComponent::return()) }
 }
 
+/** A synthetic callable with a set of concrete call sites and a flow summary. */
+abstract class SyntheticCallable extends string {
+  bindingset[this]
+  SyntheticCallable() { any() }
+
+  /** Gets a call that targets this callable. */
+  abstract Call getACall();
+
+  /**
+   * Holds if data may flow from `input` to `output` through this callable.
+   *
+   * See `SummarizedCallable::propagatesFlow` for details.
+   */
+  predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    none()
+  }
+
+  /**
+   * Gets the type of the parameter at the specified position with -1 indicating
+   * the instance parameter. If no types are provided then the types default to
+   * `Object`.
+   */
+  Type getParameterType(int pos) { none() }
+
+  /**
+   * Gets the return type of this callable. If no type is provided then the type
+   * defaults to `Object`.
+   */
+  Type getReturnType() { none() }
+}
+
+private newtype TSummarizedCallableBase =
+  TSimpleCallable(Callable c) { c.isSourceDeclaration() } or
+  TSyntheticCallable(SyntheticCallable c)
+
+/**
+ * A callable that may have a flow summary. This is either a regular `Callable`
+ * or a `SyntheticCallable`.
+ */
+class SummarizedCallableBase extends TSummarizedCallableBase {
+  /** Gets a textual representation of this callable. */
+  string toString() { result = this.asCallable().toString() or result = this.asSyntheticCallable() }
+
+  /** Gets the source location for this callable. */
+  Location getLocation() {
+    result = this.asCallable().getLocation()
+    or
+    result.hasLocationInfo("", 0, 0, 0, 0) and
+    this instanceof TSyntheticCallable
+  }
+
+  /** Gets this callable cast as a `Callable`. */
+  Callable asCallable() { this = TSimpleCallable(result) }
+
+  /** Gets this callable cast as a `SyntheticCallable`. */
+  SyntheticCallable asSyntheticCallable() { this = TSyntheticCallable(result) }
+
+  /** Gets a call that targets this callable. */
+  Call getACall() {
+    result.getCallee().getSourceDeclaration() = this.asCallable()
+    or
+    result = this.asSyntheticCallable().getACall()
+  }
+
+  /**
+   * Gets the type of the parameter at the specified position with -1 indicating
+   * the instance parameter.
+   */
+  Type getParameterType(int pos) {
+    result = this.asCallable().getParameterType(pos)
+    or
+    pos = -1 and result = this.asCallable().getDeclaringType()
+    or
+    result = this.asSyntheticCallable().getParameterType(pos)
+    or
+    exists(SyntheticCallable sc | sc = this.asSyntheticCallable() |
+      Impl::Private::summaryParameterNodeRange(this, pos) and
+      not exists(sc.getParameterType(pos)) and
+      result instanceof TypeObject
+    )
+  }
+
+  /** Gets the return type of this callable. */
+  Type getReturnType() {
+    result = this.asCallable().getReturnType()
+    or
+    exists(SyntheticCallable sc | sc = this.asSyntheticCallable() |
+      result = sc.getReturnType()
+      or
+      not exists(sc.getReturnType()) and
+      result instanceof TypeObject
+    )
+  }
+}
+
 class SummarizedCallable = Impl::Public::SummarizedCallable;
 
+/**
+ * An adapter class to add the flow summaries specified on `SyntheticCallable`
+ * to `SummarizedCallable`.
+ */
+private class SummarizedSyntheticCallableAdapter extends SummarizedCallable, TSyntheticCallable {
+  override predicate propagatesFlow(
+    SummaryComponentStack input, SummaryComponentStack output, boolean preservesValue
+  ) {
+    this.asSyntheticCallable().propagatesFlow(input, output, preservesValue)
+  }
+}
+
 class RequiredSummaryComponentStack = Impl::Public::RequiredSummaryComponentStack;

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowDispatch.qll

@@ -9,9 +9,7 @@ private import semmle.code.java.dispatch.internal.Unification
 
 private module DispatchImpl {
   private predicate hasHighConfidenceTarget(Call c) {
-    exists(SummarizedCallable sc |
-      sc = c.getCallee().getSourceDeclaration() and not sc.isAutoGenerated()
-    )
+    exists(SummarizedCallable sc | sc.getACall() = c and not sc.isAutoGenerated())
     or
     exists(Callable srcTgt |
       srcTgt = VirtualDispatch::viableCallable(c) and
@@ -30,7 +28,7 @@ private module DispatchImpl {
   DataFlowCallable viableCallable(DataFlowCall c) {
     result.asCallable() = sourceDispatch(c.asCall())
     or
-    result.asSummarizedCallable() = c.asCall().getCallee().getSourceDeclaration()
+    result.asSummarizedCallable().getACall() = c.asCall()
   }
 
   /**
@@ -144,7 +142,7 @@ private module DispatchImpl {
         not Unification::failsUnification(t, t2)
       )
       or
-      result.asSummarizedCallable() = def
+      result.asSummarizedCallable().getACall() = ma
     )
   }
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowNodes.qll

@@ -463,11 +463,7 @@ module Private {
       c.asSummarizedCallable() = sc and pos = pos_
     }
 
-    Type getTypeImpl() {
-      result = sc.getParameter(pos_).getType()
-      or
-      pos_ = -1 and result = sc.getDeclaringType()
-    }
+    Type getTypeImpl() { result = sc.getParameterType(pos_) }
   }
 }
 

java/ql/lib/semmle/code/java/dataflow/internal/DataFlowPrivate.qll

@@ -241,12 +241,6 @@ class DataFlowCallable extends TDataFlowCallable {
 
   Field asFieldScope() { this = TFieldScope(result) }
 
-  RefType getDeclaringType() {
-    result = this.asCallable().getDeclaringType() or
-    result = this.asSummarizedCallable().getDeclaringType() or
-    result = this.asFieldScope().getDeclaringType()
-  }
-
   string toString() {
     result = this.asCallable().toString() or
     result = "Synthetic: " + this.asSummarizedCallable().toString() or

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -9,12 +9,9 @@ private import DataFlowUtil
 private import FlowSummaryImpl::Private
 private import FlowSummaryImpl::Public
 private import semmle.code.java.dataflow.ExternalFlow
+private import semmle.code.java.dataflow.FlowSummary as FlowSummary
 
-private module FlowSummaries {
-  private import semmle.code.java.dataflow.FlowSummary as F
-}
-
-class SummarizedCallableBase = Callable;
+class SummarizedCallableBase = FlowSummary::SummarizedCallableBase;
 
 DataFlowCallable inject(SummarizedCallable c) { result.asSummarizedCallable() = c }
 
@@ -67,26 +64,28 @@ private boolean isGenerated(string provenance) {
  * `input`, output specification `output`, kind `kind`, and a flag `generated`
  * stating whether the summary is autogenerated.
  */
-predicate summaryElement(Callable c, string input, string output, string kind, boolean generated) {
+predicate summaryElement(
+  SummarizedCallableBase c, string input, string output, string kind, boolean generated
+) {
   exists(
     string namespace, string type, boolean subtypes, string name, string signature, string ext,
     string provenance
   |
     summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and
     generated = isGenerated(provenance) and
-    c = interpretElement(namespace, type, subtypes, name, signature, ext)
+    c.asCallable() = interpretElement(namespace, type, subtypes, name, signature, ext)
   )
 }
 
 /**
  * Holds if a negative flow summary exists for `c`, which means that there is no
  * flow through `c`. The flag `generated` states whether the summary is autogenerated.
  */
-predicate negativeSummaryElement(Callable c, boolean generated) {
+predicate negativeSummaryElement(SummarizedCallableBase c, boolean generated) {
   exists(string namespace, string type, string name, string signature, string provenance |
     negativeSummaryModel(namespace, type, name, signature, provenance) and
     generated = isGenerated(provenance) and
-    c = interpretElement(namespace, type, false, name, signature, "")
+    c.asCallable() = interpretElement(namespace, type, false, name, signature, "")
   )
 }