Go: Cleanup and renaming.

Author: michaelnebel

go/ql/lib/semmle/go/dataflow/ExternalFlow.qll

@@ -1,19 +1,20 @@
 /**
  * INTERNAL use only. This is an experimental API subject to change without notice.
  *
- * Provides classes and predicates for dealing with flow models specified in CSV format.
+ * Provides classes and predicates for dealing with MaD flow models specified
+ * in data extensions and CSV format.
  *
  * The CSV specification has the following columns:
  * - Sources:
- *   `namespace; type; subtypes; name; signature; ext; output; kind`
+ *   `package; type; subtypes; name; signature; ext; output; kind; provenance`
  * - Sinks:
- *   `namespace; type; subtypes; name; signature; ext; input; kind`
+ *   `package; type; subtypes; name; signature; ext; input; kind; provenance`
  * - Summaries:
- *   `namespace; type; subtypes; name; signature; ext; input; output; kind`
+ *   `package; type; subtypes; name; signature; ext; input; output; kind; provenance`
  *
  * The interpretation of a row is similar to API-graphs with a left-to-right
  * reading.
- * 1. The `namespace` column selects a package.
+ * 1. The `package` column selects a package.
  * 2. The `type` column selects a type within that package.
  * 3. The `subtypes` is a boolean that indicates whether to jump to an
  *    arbitrary subtype of that type.
@@ -76,45 +77,6 @@ private module Frameworks {
   private import semmle.go.frameworks.Stdlib
 }
 
-/**
- * A unit class for adding additional source model rows.
- *
- * Extend this class to add additional source definitions.
- */
-class SourceModelCsv extends Unit {
-  /** Holds if `row` specifies a source definition. */
-  abstract predicate row(string row);
-}
-
-/**
- * A unit class for adding additional sink model rows.
- *
- * Extend this class to add additional sink definitions.
- */
-class SinkModelCsv extends Unit {
-  /** Holds if `row` specifies a sink definition. */
-  abstract predicate row(string row);
-}
-
-/**
- * A unit class for adding additional summary model rows.
- *
- * Extend this class to add additional flow summary definitions.
- */
-class SummaryModelCsv extends Unit {
-  /** Holds if `row` specifies a summary definition. */
-  abstract predicate row(string row);
-}
-
-/** Holds if `row` is a source model. */
-predicate sourceModel(string row) { any(SourceModelCsv s).row(row) }
-
-/** Holds if `row` is a sink model. */
-predicate sinkModel(string row) { any(SinkModelCsv s).row(row) }
-
-/** Holds if `row` is a summary model. */
-predicate summaryModel(string row) { any(SummaryModelCsv s).row(row) }
-
 /** Holds if a source model exists for the given parameters. */
 predicate sourceModel = Extensions::sourceModel/9;
 
@@ -124,45 +86,45 @@ predicate sinkModel = Extensions::sinkModel/9;
 /** Holds if a summary model exists for the given parameters. */
 predicate summaryModel = Extensions::summaryModel/10;
 
-/** Holds if `package` have CSV framework coverage. */
-private predicate packageHasCsvCoverage(string package) {
+/** Holds if `package` have MaD framework coverage. */
+private predicate packageHasMaDCoverage(string package) {
   sourceModel(package, _, _, _, _, _, _, _, _) or
   sinkModel(package, _, _, _, _, _, _, _, _) or
   summaryModel(package, _, _, _, _, _, _, _, _, _)
 }
 
 /**
- * Holds if `package` and `subpkg` have CSV framework coverage and `subpkg`
+ * Holds if `package` and `subpkg` have MaD framework coverage and `subpkg`
  * is a subpackage of `package`.
  */
 private predicate packageHasASubpackage(string package, string subpkg) {
-  packageHasCsvCoverage(package) and
-  packageHasCsvCoverage(subpkg) and
+  packageHasMaDCoverage(package) and
+  packageHasMaDCoverage(subpkg) and
   subpkg.prefix(subpkg.indexOf(".")) = package
 }
 
 /**
- * Holds if `package` has CSV framework coverage and it is not a subpackage of
- * any other package with CSV framework coverage.
+ * Holds if `package` has MaD framework coverage and it is not a subpackage of
+ * any other package with MaD framework coverage.
  */
 private predicate canonicalPackage(string package) {
-  packageHasCsvCoverage(package) and not packageHasASubpackage(_, package)
+  packageHasMaDCoverage(package) and not packageHasASubpackage(_, package)
 }
 
 /**
- * Holds if `package` and `subpkg` have CSV framework coverage, `subpkg` is a
+ * Holds if `package` and `subpkg` have MaD framework coverage, `subpkg` is a
  * subpackage of `package` (or they are the same), and `package` is not a
- * subpackage of any other package with CSV framework coverage.
+ * subpackage of any other package with MaD framework coverage.
  */
 private predicate canonicalPackageHasASubpackage(string package, string subpkg) {
   canonicalPackage(package) and
   (subpkg = package or packageHasASubpackage(package, subpkg))
 }
 
 /**
- * Holds if CSV framework coverage of `package` is `n` api endpoints of the
+ * Holds if MaD framework coverage of `package` is `n` api endpoints of the
  * kind `(kind, part)`, and `pkgs` is the number of subpackages of `package`
- * which have CSV framework coverage (including `package` itself).
+ * which have MaD framework coverage (including `package` itself).
  */
 predicate modelCoverage(string package, int pkgs, string kind, string part, int n) {
   pkgs = strictcount(string subpkg | canonicalPackageHasASubpackage(package, subpkg)) and
@@ -193,8 +155,8 @@ predicate modelCoverage(string package, int pkgs, string kind, string part, int
   )
 }
 
-/** Provides a query predicate to check the CSV data for validation errors. */
-module CsvValidation {
+/** Provides a query predicate to check the MaD models for validation errors. */
+module ModelValidation {
   private string getInvalidModelInput() {
     exists(string pred, AccessPath input, string part |
       sinkModel(_, _, _, _, _, _, input, _, _) and pred = "sink"
@@ -227,57 +189,25 @@ module CsvValidation {
   }
 
   private string getInvalidModelKind() {
-    exists(string row, string kind | summaryModel(row) |
-      kind = row.splitAt(";", 8) and
+    exists(string kind | summaryModel(_, _, _, _, _, _, _, _, kind, _) |
       not kind = ["taint", "value"] and
       result = "Invalid kind \"" + kind + "\" in summary model."
     )
   }
 
-  private string getInvalidModelSubtype() {
-    exists(string pred, string row |
-      sourceModel(row) and pred = "source"
-      or
-      sinkModel(row) and pred = "sink"
-      or
-      summaryModel(row) and pred = "summary"
-    |
-      exists(string b |
-        b = row.splitAt(";", 2) and
-        not b = ["true", "false"] and
-        result = "Invalid boolean \"" + b + "\" in " + pred + " model."
-      )
-    )
-  }
-
-  private string getInvalidModelColumnCount() {
-    exists(string pred, string row, int expect |
-      sourceModel(row) and expect = 8 and pred = "source"
-      or
-      sinkModel(row) and expect = 8 and pred = "sink"
-      or
-      summaryModel(row) and expect = 9 and pred = "summary"
-    |
-      exists(int cols |
-        cols = 1 + max(int n | exists(row.splitAt(";", n))) and
-        cols != expect and
-        result =
-          "Wrong number of columns in " + pred + " model row, expected " + expect + ", got " + cols +
-            "."
-      )
-    )
-  }
-
   private string getInvalidModelSignature() {
-    exists(string pred, string namespace, string type, string name, string signature, string ext |
-      sourceModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "source"
+    exists(
+      string pred, string package, string type, string name, string signature, string ext,
+      string provenance
+    |
+      sourceModel(package, type, _, name, signature, ext, _, _, provenance) and pred = "source"
       or
-      sinkModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "sink"
+      sinkModel(package, type, _, name, signature, ext, _, _, provenance) and pred = "sink"
       or
-      summaryModel(namespace, type, _, name, signature, ext, _, _, _, _) and pred = "summary"
+      summaryModel(package, type, _, name, signature, ext, _, _, _, provenance) and pred = "summary"
     |
-      not namespace.regexpMatch("[a-zA-Z0-9_\\./]*") and
-      result = "Dubious namespace \"" + namespace + "\" in " + pred + " model."
+      not package.regexpMatch("[a-zA-Z0-9_\\./]*") and
+      result = "Dubious package \"" + package + "\" in " + pred + " model."
       or
       not type.regexpMatch("[a-zA-Z0-9_\\$<>]*") and
       result = "Dubious type \"" + type + "\" in " + pred + " model."
@@ -290,26 +220,29 @@ module CsvValidation {
       or
       not ext.regexpMatch("|Annotated") and
       result = "Unrecognized extra API graph element \"" + ext + "\" in " + pred + " model."
+      or
+      not provenance = ["manual", "generated"] and
+      result = "Unrecognized provenance description \"" + provenance + "\" in " + pred + " model."
     )
   }
 
-  /** Holds if some row in a CSV-based flow model appears to contain typos. */
+  /** Holds if some row in a MaD flow model appears to contain typos. */
   query predicate invalidModelRow(string msg) {
     msg =
       [
         getInvalidModelSignature(), getInvalidModelInput(), getInvalidModelOutput(),
-        getInvalidModelSubtype(), getInvalidModelColumnCount(), getInvalidModelKind()
+        getInvalidModelKind()
       ]
   }
 }
 
 pragma[nomagic]
 private predicate elementSpec(
-  string namespace, string type, boolean subtypes, string name, string signature, string ext
+  string package, string type, boolean subtypes, string name, string signature, string ext
 ) {
-  sourceModel(namespace, type, subtypes, name, signature, ext, _, _, _) or
-  sinkModel(namespace, type, subtypes, name, signature, ext, _, _, _) or
-  summaryModel(namespace, type, subtypes, name, signature, ext, _, _, _, _)
+  sourceModel(package, type, subtypes, name, signature, ext, _, _, _) or
+  sinkModel(package, type, subtypes, name, signature, ext, _, _, _) or
+  summaryModel(package, type, subtypes, name, signature, ext, _, _, _, _)
 }
 
 private string paramsStringPart(Function f, int i) {
@@ -405,7 +338,7 @@ predicate parseContent(string component, DataFlow::Content content) {
 cached
 private module Cached {
   /**
-   * Holds if `node` is specified as a source with the given kind in a CSV flow
+   * Holds if `node` is specified as a source with the given kind in a MaD flow
    * model.
    */
   cached
@@ -414,7 +347,7 @@ private module Cached {
   }
 
   /**
-   * Holds if `node` is specified as a sink with the given kind in a CSV flow
+   * Holds if `node` is specified as a sink with the given kind in a MaD flow
    * model.
    */
   cached

go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -65,11 +65,9 @@ DataFlowType getSyntheticGlobalType(SummaryComponent::SyntheticGlobal sg) { any(
 predicate summaryElement(
   SummarizedCallableBase c, string input, string output, string kind, string provenance
 ) {
-  exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
-  |
-    summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and
-    c.asFunction() = interpretElement(namespace, type, subtypes, name, signature, ext).asEntity()
+  exists(string package, string type, boolean subtypes, string name, string signature, string ext |
+    summaryModel(package, type, subtypes, name, signature, ext, input, output, kind, provenance) and
+    c.asFunction() = interpretElement(package, type, subtypes, name, signature, ext).asEntity()
   )
 }
 
@@ -154,11 +152,9 @@ class SourceOrSinkElement extends TSourceOrSinkElement {
  * `output`, kind `kind`, and provenance `provenance`.
  */
 predicate sourceElement(SourceOrSinkElement e, string output, string kind, string provenance) {
-  exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
-  |
-    sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, provenance) and
-    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  exists(string package, string type, boolean subtypes, string name, string signature, string ext |
+    sourceModel(package, type, subtypes, name, signature, ext, output, kind, provenance) and
+    e = interpretElement(package, type, subtypes, name, signature, ext)
   )
 }
 
@@ -167,11 +163,9 @@ predicate sourceElement(SourceOrSinkElement e, string output, string kind, strin
  * `input`, kind `kind` and provenance `provenance`.
  */
 predicate sinkElement(SourceOrSinkElement e, string input, string kind, string provenance) {
-  exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
-  |
-    sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, provenance) and
-    e = interpretElement(namespace, type, subtypes, name, signature, ext)
+  exists(string package, string type, boolean subtypes, string name, string signature, string ext |
+    sinkModel(package, type, subtypes, name, signature, ext, input, kind, provenance) and
+    e = interpretElement(package, type, subtypes, name, signature, ext)
   )
 }
 

go/ql/lib/semmle/go/security/FlowSources.qll

@@ -23,7 +23,12 @@ module UntrustedFlowSource {
    */
   abstract class Range extends DataFlow::Node { }
 
-  class CsvRemoteSource extends Range {
-    CsvRemoteSource() { ExternalFlow::sourceNode(this, "remote") }
+  /**
+   * A source of data that is controlled by an untrusted user.
+   */
+  class MaDRemoteSource extends Range {
+    MaDRemoteSource() { ExternalFlow::sourceNode(this, "remote") }
   }
+
+  deprecated class CsvRemoteSource = MaDRemoteSource;
 }

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/completetest.ql

@@ -4,7 +4,7 @@
 
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 import semmle.go.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 import TestUtilities.InlineFlowTest
 

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/sinks.ql

@@ -1,6 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 
 from DataFlow::Node node, string kind
 where sinkNode(node, kind)

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/srcs.ql

@@ -1,6 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 
 from DataFlow::Node node, string kind
 where sourceNode(node, kind)

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlow/steps.ql

@@ -1,6 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 import semmle.go.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 
 from DataFlow::Node node1, DataFlow::Node node2

go/ql/test/library-tests/semmle/go/dataflow/ExternalFlowVarArgs/Flows.ql

@@ -1,6 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 import TestUtilities.InlineExpectationsTest
 
 class DataConfiguration extends DataFlow::Configuration {

go/ql/test/library-tests/semmle/go/dataflow/VarArgsWithFunctionModels/Flows.ql

@@ -1,6 +1,6 @@
 import go
 import semmle.go.dataflow.ExternalFlow
-import CsvValidation
+import ModelValidation
 // import DataFlow::PartialPathGraph
 import TestUtilities.InlineExpectationsTest