Go: Implement new data flow interface

hvitved · hvitved · commit e4a4c1816632 · 2024-03-13T14:41:57.000+01:00
diff --git a/go/ql/lib/semmle/go/DiagnosticsReporting.qll b/go/ql/lib/semmle/go/DiagnosticsReporting.qll
@@ -1,6 +1,7 @@
 /** Provides classes for working with errors and warnings recorded during extraction. */
 
 import go
+private import semmle.go.internal.Locations
 
 /** Gets the SARIF severity level that indicates an error. */
 private int getErrorSeverity() { result = 2 }
@@ -29,7 +30,7 @@ private class Diagnostic extends @diagnostic {
    * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
    */
   predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
-    exists(Location l | diagnostics(this, _, _, _, _, l) | l.hasLocationInfo(path, sl, sc, el, ec))
+    getDiagnosticLocation(this).hasLocationInfo(path, sl, sc, el, ec)
   }
 
   string toString() { result = this.getMessage() }
diff --git a/go/ql/lib/semmle/go/Files.qll b/go/ql/lib/semmle/go/Files.qll
@@ -50,8 +50,6 @@ class Folder extends Container, Impl::Folder {
 class ExtractedOrExternalFile extends Container, Impl::File, Documentable, ExprParent,
   GoModExprParent, DeclParent, ScopeNode
 {
-  override Location getLocation() { has_location(this, result) }
-
   /** Gets the number of lines in this file. */
   int getNumberOfLines() { numlines(this, result, _, _) }
 
diff --git a/go/ql/lib/semmle/go/HTML.qll b/go/ql/lib/semmle/go/HTML.qll
@@ -15,8 +15,6 @@ module HTML {
   class Element extends Locatable, @xmlelement {
     Element() { exists(HtmlFile f | xmlElements(this, _, _, _, f)) }
 
-    override Location getLocation() { xmllocations(this, result) }
-
     /**
      * Gets the name of this HTML element.
      *
@@ -97,8 +95,6 @@ module HTML {
   class Attribute extends Locatable, @xmlattribute {
     Attribute() { xmlAttrs(this, _, _, _, _, any(HtmlFile f)) }
 
-    override Location getLocation() { xmllocations(this, result) }
-
     /**
      * Gets the element to which this attribute belongs.
      */
@@ -180,8 +176,6 @@ module HTML {
      * Holds if this text node is inside a `CDATA` tag.
      */
     predicate isCData() { xmlChars(this, _, _, _, 1, _) }
-
-    override Location getLocation() { xmllocations(this, result) }
   }
 
   /**
@@ -203,7 +197,5 @@ module HTML {
     string getText() { result = this.toString().regexpCapture("(?s)<!--(.*)-->", 1) }
 
     override string toString() { xmlComments(this, result, _, _) }
-
-    override Location getLocation() { xmllocations(this, result) }
   }
 }
diff --git a/go/ql/lib/semmle/go/Locations.qll b/go/ql/lib/semmle/go/Locations.qll
@@ -1,28 +1,31 @@
 /** Provides classes for working with locations and program elements that have locations. */
 
 import go
+private import internal.Locations
 
 /**
  * A location as given by a file, a start line, a start column,
  * an end line, and an end column.
  *
+ * This class is restricted to locations created by the extractor.
+ *
  * For more information about locations see [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
  */
-class Location extends @location {
+class DbLocation extends TDbLocation {
   /** Gets the file for this location. */
-  File getFile() { locations_default(this, result, _, _, _, _) }
+  File getFile() { dbLocationInfo(this, result, _, _, _, _) }
 
   /** Gets the 1-based line number (inclusive) where this location starts. */
-  int getStartLine() { locations_default(this, _, result, _, _, _) }
+  int getStartLine() { dbLocationInfo(this, _, result, _, _, _) }
 
   /** Gets the 1-based column number (inclusive) where this location starts. */
-  int getStartColumn() { locations_default(this, _, _, result, _, _) }
+  int getStartColumn() { dbLocationInfo(this, _, _, result, _, _) }
 
   /** Gets the 1-based line number (inclusive) where this location ends. */
-  int getEndLine() { locations_default(this, _, _, _, result, _) }
+  int getEndLine() { dbLocationInfo(this, _, _, _, result, _) }
 
   /** Gets the 1-based column number (inclusive) where this location ends. */
-  int getEndColumn() { locations_default(this, _, _, _, _, result) }
+  int getEndColumn() { dbLocationInfo(this, _, _, _, _, result) }
 
   /** Gets the number of lines covered by this location. */
   int getNumLines() { result = this.getEndLine() - this.getStartLine() + 1 }
@@ -46,19 +49,21 @@ class Location extends @location {
     string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
     exists(File f |
-      locations_default(this, f, startline, startcolumn, endline, endcolumn) and
+      dbLocationInfo(this, f, startline, startcolumn, endline, endcolumn) and
       filepath = f.getAbsolutePath()
     )
   }
 }
 
+final class Location = LocationImpl;
+
 /** A program element with a location. */
 class Locatable extends @locatable {
   /** Gets the file this program element comes from. */
   File getFile() { result = this.getLocation().getFile() }
 
   /** Gets this element's location. */
-  Location getLocation() { has_location(this, result) }
+  final DbLocation getLocation() { result = getLocatableLocation(this) }
 
   /** Gets the number of lines covered by this element. */
   int getNumLines() { result = this.getLocation().getNumLines() }
diff --git a/go/ql/lib/semmle/go/dataflow/DataFlow.qll b/go/ql/lib/semmle/go/dataflow/DataFlow.qll
@@ -24,7 +24,7 @@ import go
 module DataFlow {
   private import semmle.go.dataflow.internal.DataFlowImplSpecific
   private import codeql.dataflow.DataFlow
-  import DataFlowMake<GoDataFlow>
+  import DataFlowMake<Location, GoDataFlow>
   import semmle.go.dataflow.internal.DataFlowImpl1
   import Properties
 }
diff --git a/go/ql/lib/semmle/go/dataflow/TaintTracking.qll b/go/ql/lib/semmle/go/dataflow/TaintTracking.qll
@@ -13,7 +13,8 @@ module TaintTracking {
   import semmle.go.dataflow.internal.tainttracking1.TaintTrackingParameter::Public
   private import semmle.go.dataflow.internal.DataFlowImplSpecific
   private import semmle.go.dataflow.internal.TaintTrackingImplSpecific
+  private import semmle.go.Locations
   private import codeql.dataflow.TaintTracking
-  import TaintFlowMake<GoDataFlow, GoTaintTracking>
+  import TaintFlowMake<Location, GoDataFlow, GoTaintTracking>
   import semmle.go.dataflow.internal.tainttracking1.TaintTrackingImpl
 }
diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImpl.qll
@@ -1,3 +1,4 @@
 private import DataFlowImplSpecific
 private import codeql.dataflow.internal.DataFlowImpl
-import MakeImpl<GoDataFlow>
+private import semmle.go.Locations
+import MakeImpl<Location, GoDataFlow>
diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplCommon.qll
@@ -1,3 +1,4 @@
 private import DataFlowImplSpecific
 private import codeql.dataflow.internal.DataFlowImplCommon
-import MakeImplCommon<GoDataFlow>
+private import semmle.go.Locations
+import MakeImplCommon<Location, GoDataFlow>
diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplConsistency.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplConsistency.qll
@@ -7,7 +7,8 @@ private import go
 private import DataFlowImplSpecific
 private import TaintTrackingImplSpecific
 private import codeql.dataflow.internal.DataFlowImplConsistency
+private import semmle.go.dataflow.internal.DataFlowNodes
 
-private module Input implements InputSig<GoDataFlow> { }
+private module Input implements InputSig<Location, GoDataFlow> { }
 
-module Consistency = MakeConsistency<GoDataFlow, GoTaintTracking, Input>;
+module Consistency = MakeConsistency<Location, GoDataFlow, GoTaintTracking, Input>;
diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowImplSpecific.qll
@@ -3,6 +3,7 @@
  */
 
 private import codeql.dataflow.DataFlow
+private import semmle.go.Locations
 
 module Private {
   import DataFlowPrivate
@@ -13,7 +14,7 @@ module Public {
   import DataFlowUtil
 }
 
-module GoDataFlow implements InputSig {
+module GoDataFlow implements InputSig<Location> {
   import Private
   import Public
 
diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowNodes.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowNodes.qll
@@ -157,6 +157,14 @@ module Public {
       endcolumn = 0
     }
 
+    /** Gets the location of this node. */
+    Location getLocation() {
+      exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+        this.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and
+        result.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+      )
+    }
+
     /** Gets the file in which this node appears. */
     File getFile() { this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _) }
 
diff --git a/go/ql/lib/semmle/go/dataflow/internal/DataFlowPrivate.qll b/go/ql/lib/semmle/go/dataflow/internal/DataFlowPrivate.qll
@@ -251,8 +251,6 @@ class DataFlowType extends TDataFlowType {
   string toString() { result = "" }
 }
 
-class DataFlowLocation = Location;
-
 private newtype TDataFlowCallable =
   TCallable(Callable c) or
   TFileScope(File f) or
diff --git a/go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll b/go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll
@@ -15,7 +15,7 @@ private module FlowSummaries {
   private import semmle.go.dataflow.FlowSummary as F
 }
 
-module Input implements InputSig<DataFlowImplSpecific::GoDataFlow> {
+module Input implements InputSig<Location, DataFlowImplSpecific::GoDataFlow> {
   class SummarizedCallableBase = Callable;
 
   ArgumentPosition callbackSelfParameterPosition() { result = -1 }
@@ -83,7 +83,7 @@ module Input implements InputSig<DataFlowImplSpecific::GoDataFlow> {
   }
 }
 
-private import Make<DataFlowImplSpecific::GoDataFlow, Input> as Impl
+private import Make<Location, DataFlowImplSpecific::GoDataFlow, Input> as Impl
 
 private module StepsInput implements Impl::Private::StepsInputSig {
   DataFlowCall getACall(Public::SummarizedCallable sc) {
@@ -95,7 +95,7 @@ private module StepsInput implements Impl::Private::StepsInputSig {
 }
 
 module SourceSinkInterpretationInput implements
-  Impl::Private::External::SourceSinkInterpretationInputSig<Location>
+  Impl::Private::External::SourceSinkInterpretationInputSig
 {
   class Element = SourceOrSinkElement;
 
@@ -264,7 +264,7 @@ module Private {
 
   module External {
     import Impl::Private::External
-    import Impl::Private::External::SourceSinkInterpretation<Location, SourceSinkInterpretationInput>
+    import Impl::Private::External::SourceSinkInterpretation<SourceSinkInterpretationInput>
   }
 
   /**
diff --git a/go/ql/lib/semmle/go/dataflow/internal/TaintTrackingImplSpecific.qll b/go/ql/lib/semmle/go/dataflow/internal/TaintTrackingImplSpecific.qll
@@ -4,7 +4,8 @@
 
 private import codeql.dataflow.TaintTracking
 private import DataFlowImplSpecific
+private import semmle.go.Locations
 
-module GoTaintTracking implements InputSig<GoDataFlow> {
+module GoTaintTracking implements InputSig<Location, GoDataFlow> {
   import TaintTrackingUtil
 }
diff --git a/go/ql/lib/semmle/go/internal/Locations.qll b/go/ql/lib/semmle/go/internal/Locations.qll
@@ -0,0 +1,143 @@
+/** Provides classes for working with locations and program elements that have locations. */
+
+import go
+
+// Should _not_ be cached, as that would require the data flow stage to be evaluated
+// in order to evaluate the AST stage. Ideally, we would cache each injector separately,
+// but that's not possible. Instead, we cache all predicates that need the injectors
+// to be tuple numbered.
+newtype TLocation =
+  TDbLocation(@location loc) or
+  TSynthLocation(string filepath, int startline, int startcolumn, int endline, int endcolumn) {
+    any(DataFlow::Node n).hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and
+    // avoid overlap with existing DB locations
+    not exists(File f |
+      locations_default(_, f, startline, startcolumn, endline, endcolumn) and
+      f.getAbsolutePath() = filepath
+    )
+  }
+
+/**
+ * A location as given by a file, a start line, a start column,
+ * an end line, and an end column.
+ *
+ * For more information about locations see [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+ */
+abstract class LocationImpl extends TLocation {
+  /** Gets the file for this location. */
+  abstract File getFile();
+
+  /** Gets the 1-based line number (inclusive) where this location starts. */
+  abstract int getStartLine();
+
+  /** Gets the 1-based column number (inclusive) where this location starts. */
+  abstract int getStartColumn();
+
+  /** Gets the 1-based line number (inclusive) where this location ends. */
+  abstract int getEndLine();
+
+  /** Gets the 1-based column number (inclusive) where this location ends. */
+  abstract int getEndColumn();
+
+  /** Gets the number of lines covered by this location. */
+  int getNumLines() { result = this.getEndLine() - this.getStartLine() + 1 }
+
+  /** Gets a textual representation of this element. */
+  string toString() {
+    exists(string filepath, int startline, int startcolumn, int endline, int endcolumn |
+      this.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and
+      result = filepath + "@" + startline + ":" + startcolumn + ":" + endline + ":" + endcolumn
+    )
+  }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/).
+   */
+  abstract predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  );
+}
+
+class DbLocationImpl extends LocationImpl instanceof DbLocation {
+  private @location loc;
+
+  DbLocationImpl() { this = TDbLocation(loc) }
+
+  override File getFile() { result = DbLocation.super.getFile() }
+
+  override int getStartLine() { result = DbLocation.super.getStartLine() }
+
+  override int getStartColumn() { result = DbLocation.super.getStartColumn() }
+
+  override int getEndLine() { result = DbLocation.super.getEndLine() }
+
+  override int getEndColumn() { result = DbLocation.super.getEndColumn() }
+
+  override predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    DbLocation.super.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
+  }
+}
+
+class SynthLocationImpl extends LocationImpl, TSynthLocation {
+  override File getFile() { synthLocationInfo(this, result.getAbsolutePath(), _, _, _, _) }
+
+  override int getStartLine() { synthLocationInfo(this, _, result, _, _, _) }
+
+  override int getStartColumn() { synthLocationInfo(this, _, _, result, _, _) }
+
+  override int getEndLine() { synthLocationInfo(this, _, _, _, result, _) }
+
+  override int getEndColumn() { synthLocationInfo(this, _, _, _, _, result) }
+
+  override predicate hasLocationInfo(
+    string filepath, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    synthLocationInfo(this, filepath, startline, startcolumn, endline, endcolumn)
+  }
+}
+
+cached
+private module Cached {
+  cached
+  DbLocation getLocatableLocation(@locatable l) {
+    exists(@location loc |
+      has_location(l, loc) or
+      xmllocations(l, loc)
+    |
+      result = TDbLocation(loc)
+    )
+  }
+
+  cached
+  DbLocation getDiagnosticLocation(@diagnostic d) {
+    exists(@location loc |
+      diagnostics(d, _, _, _, _, loc) and
+      result = TDbLocation(loc)
+    )
+  }
+
+  cached
+  predicate dbLocationInfo(
+    DbLocation l, File f, int startline, int startcolumn, int endline, int endcolumn
+  ) {
+    exists(@location loc |
+      l = TDbLocation(loc) and
+      locations_default(loc, f, startline, startcolumn, endline, endcolumn)
+    )
+  }
+}
+
+import Cached
+
+cached
+private predicate synthLocationInfo(
+  SynthLocationImpl l, string filepath, int startline, int startcolumn, int endline, int endcolumn
+) {
+  l = TSynthLocation(filepath, startline, startcolumn, endline, endcolumn)
+}
diff --git a/go/ql/src/Security/CWE-338/InsecureRandomness.ql b/go/ql/src/Security/CWE-338/InsecureRandomness.ql
diff --git a/go/ql/test/TestUtilities/InlineFlowTest.qll b/go/ql/test/TestUtilities/InlineFlowTest.qll
diff --git a/go/ql/test/TestUtilities/internal/InlineExpectationsTestImpl.qll b/go/ql/test/TestUtilities/internal/InlineExpectationsTestImpl.qll
diff --git a/go/ql/test/extractor-tests/diagnostics/Diagnostics.ql b/go/ql/test/extractor-tests/diagnostics/Diagnostics.ql

Original file line number	Diff line number	Diff line change
`@@ -50,8 +50,6 @@ class Folder extends Container, Impl::Folder {`
`50`	`50`	`class ExtractedOrExternalFile extends Container, Impl::File, Documentable, ExprParent,`
`51`	`51`	`GoModExprParent, DeclParent, ScopeNode`
`52`	`52`	`{`
`53`		`- override Location getLocation() { has_location(this, result) }`
`54`		`-`
`55`	`53`	`/** Gets the number of lines in this file. */`
`56`	`54`	`int getNumberOfLines() { numlines(this, result, _, _) }`
`57`	`55`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ import go`
`24`	`24`	`module DataFlow {`
`25`	`25`	`private import semmle.go.dataflow.internal.DataFlowImplSpecific`
`26`	`26`	`private import codeql.dataflow.DataFlow`
`27`		`- import DataFlowMake<GoDataFlow>`
	`27`	`+ import DataFlowMake<Location, GoDataFlow>`
`28`	`28`	`import semmle.go.dataflow.internal.DataFlowImpl1`
`29`	`29`	`import Properties`
`30`	`30`	`}`