Merge pull request github#6030 from smowton/smowton/admin/test-generator

Add test-generator script + add generated models for Spring summary steps

Author: aschackmull

csharp/ql/src/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -642,6 +642,13 @@ module Private {
       )
     }
 
+    /**
+     * Holds if `spec` specifies summary component stack `stack`.
+     */
+    predicate interpretSpec(string spec, SummaryComponentStack stack) {
+      interpretSpec(spec, 0, stack)
+    }
+
     private predicate interpretSpec(string spec, int idx, SummaryComponentStack stack) {
       exists(string c |
         relevantSpec(spec) and
@@ -680,8 +687,8 @@ module Private {
       ) {
         exists(string inSpec, string outSpec, string kind |
           summaryElement(this, inSpec, outSpec, kind) and
-          interpretSpec(inSpec, 0, input) and
-          interpretSpec(outSpec, 0, output)
+          interpretSpec(inSpec, input) and
+          interpretSpec(outSpec, output)
         |
           kind = "value" and preservesValue = true
           or

java/ql/src/semmle/code/java/dataflow/ExternalFlow.qll

@@ -436,19 +436,25 @@ predicate summaryModel(
   string namespace, string type, boolean subtypes, string name, string signature, string ext,
   string input, string output, string kind
 ) {
-  exists(string row |
-    summaryModel(row) and
-    row.splitAt(";", 0) = namespace and
-    row.splitAt(";", 1) = type and
-    row.splitAt(";", 2) = subtypes.toString() and
-    subtypes = [true, false] and
-    row.splitAt(";", 3) = name and
-    row.splitAt(";", 4) = signature and
-    row.splitAt(";", 5) = ext and
-    row.splitAt(";", 6) = input and
-    row.splitAt(";", 7) = output and
-    row.splitAt(";", 8) = kind
-  )
+  summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, _)
+}
+
+/** Holds if a summary model `row` exists for the given parameters. */
+predicate summaryModel(
+  string namespace, string type, boolean subtypes, string name, string signature, string ext,
+  string input, string output, string kind, string row
+) {
+  summaryModel(row) and
+  row.splitAt(";", 0) = namespace and
+  row.splitAt(";", 1) = type and
+  row.splitAt(";", 2) = subtypes.toString() and
+  subtypes = [true, false] and
+  row.splitAt(";", 3) = name and
+  row.splitAt(";", 4) = signature and
+  row.splitAt(";", 5) = ext and
+  row.splitAt(";", 6) = input and
+  row.splitAt(";", 7) = output and
+  row.splitAt(";", 8) = kind
 }
 
 private predicate relevantPackage(string package) {

java/ql/src/semmle/code/java/dataflow/internal/DataFlowUtil.qll

@@ -173,6 +173,13 @@ class Content extends TContent {
   /** Gets a textual representation of this element. */
   abstract string toString();
 
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   */
   predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
     path = "" and sl = 0 and sc = 0 and el = 0 and ec = 0
   }

java/ql/src/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -642,6 +642,13 @@ module Private {
       )
     }
 
+    /**
+     * Holds if `spec` specifies summary component stack `stack`.
+     */
+    predicate interpretSpec(string spec, SummaryComponentStack stack) {
+      interpretSpec(spec, 0, stack)
+    }
+
     private predicate interpretSpec(string spec, int idx, SummaryComponentStack stack) {
       exists(string c |
         relevantSpec(spec) and
@@ -680,8 +687,8 @@ module Private {
       ) {
         exists(string inSpec, string outSpec, string kind |
           summaryElement(this, inSpec, outSpec, kind) and
-          interpretSpec(inSpec, 0, input) and
-          interpretSpec(outSpec, 0, output)
+          interpretSpec(inSpec, input) and
+          interpretSpec(outSpec, output)
         |
           kind = "value" and preservesValue = true
           or

java/ql/src/utils/GenerateFlowTestCase.py

@@ -0,0 +1,185 @@
+#!/usr/bin/python3
+
+import errno
+import json
+import os
+import os.path
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+
+if any(s == "--help" for s in sys.argv):
+  print("""Usage:
+GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir
+
+This generates test cases exercising function model specifications found in specsToTest.csv
+producing files Test.java, test.ql and test.expected in outdir.
+
+projectPom.xml should be a Maven pom sufficient to resolve the classes named in specsToTest.csv.
+Typically this means supplying a skeleton POM <dependencies> section that retrieves whatever jars
+contain the needed classes.
+
+Requirements: `mvn` and `codeql` should both appear on your path.
+
+After test generation completes, any lines in specsToTest.csv that didn't produce tests are output.
+If this happens, check the spelling of class and method names, and the syntax of input and output specifications.
+""")
+  sys.exit(0)
+
+if len(sys.argv) != 4:
+  print("Usage: GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir", file=sys.stderr)
+  print("specsToTest.csv should contain CSV rows describing method taint-propagation specifications to test", file=sys.stderr)
+  print("projectPom.xml should import dependencies sufficient to resolve the types used in specsToTest.csv", file=sys.stderr)
+  sys.exit(1)
+
+try:
+  os.makedirs(sys.argv[3])
+except Exception as e:
+  if e.errno != errno.EEXIST:
+    print("Failed to create output directory %s: %s" % (sys.argv[3], e))
+    sys.exit(1)
+
+resultJava = os.path.join(sys.argv[3], "Test.java")
+resultQl = os.path.join(sys.argv[3], "test.ql")
+
+if os.path.exists(resultJava) or os.path.exists(resultQl):
+  print("Won't overwrite existing files '%s' or '%s'" % (resultJava, resultQl), file = sys.stderr)
+  sys.exit(1)
+
+workDir = tempfile.mkdtemp()
+
+# Make a database that touches all types whose methods we want to test:
+print("Creating Maven project")
+projectDir = os.path.join(workDir, "mavenProject")
+os.makedirs(projectDir)
+
+try:
+  shutil.copyfile(sys.argv[2], os.path.join(projectDir, "pom.xml"))
+except Exception as e:
+  print("Failed to read project POM %s: %s" % (sys.argv[2], e), file = sys.stderr)
+  sys.exit(1)
+
+commentRegex = re.compile("^\s*(//|#)")
+def isComment(s):
+  return commentRegex.match(s) is not None
+
+try:
+  with open(sys.argv[1], "r") as f:
+    specs = [l for l in f if not isComment(l)]
+except Exception as e:
+  print("Failed to open %s: %s\n" % (sys.argv[1], e))
+  sys.exit(1)
+
+projectTestPkgDir = os.path.join(projectDir, "src", "main", "java", "test")
+projectTestFile = os.path.join(projectTestPkgDir, "Test.java")
+
+os.makedirs(projectTestPkgDir)
+
+def qualifiedOuterNameFromCsvRow(row):
+  cells = row.split(";")
+  if len(cells) < 2:
+    return None
+  return cells[0] + "." + cells[1].replace("$", ".")
+
+with open(projectTestFile, "w") as testJava:
+  testJava.write("package test;\n\npublic class Test {\n\n")
+
+  for i, spec in enumerate(specs):
+    outerName = qualifiedOuterNameFromCsvRow(spec)
+    if outerName is None:
+      print("A taint specification has the wrong format: should be 'package;classname;methodname....'", file = sys.stderr)
+      print("Mis-formatted row: " + spec, file = sys.stderr)
+      sys.exit(1)
+    testJava.write("\t%s obj%d = null;\n" % (outerName, i))
+
+  testJava.write("}")
+
+print("Creating project database")
+cmd = ["codeql", "database", "create", "--language=java", "db"]
+ret = subprocess.call(cmd, cwd = projectDir)
+if ret != 0:
+  print("Failed to create project database. Check that '%s' is a valid POM that pulls in all necessary dependencies, and '%s' specifies valid classes and methods." % (sys.argv[2], sys.argv[1]), file = sys.stderr)
+  print("Failed command was: %s (cwd: %s)" % (shlex.join(cmd), projectDir), file = sys.stderr)
+  sys.exit(1)
+
+print("Creating test-generation query")
+queryDir = os.path.join(workDir, "query")
+os.makedirs(queryDir)
+qlFile = os.path.join(queryDir, "gen.ql")
+with open(os.path.join(queryDir, "qlpack.yml"), "w") as f:
+  f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql-java")
+with open(qlFile, "w") as f:
+  f.write("import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
+  f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
+  f.write("\n\t\t]\n\t}\n}\n")
+
+print("Generating tests")
+generatedBqrs = os.path.join(queryDir, "out.bqrs")
+cmd = ['codeql', 'query', 'run', qlFile, '--database', os.path.join(projectDir, "db"), '--output', generatedBqrs]
+ret = subprocess.call(cmd)
+if ret != 0:
+  print("Failed to generate tests. Failed command was: " + shlex.join(cmd))
+  sys.exit(1)
+
+generatedJson = os.path.join(queryDir, "out.json")
+cmd = ['codeql', 'bqrs', 'decode', generatedBqrs, '--format=json', '--output', generatedJson]
+ret = subprocess.call(cmd)
+if ret != 0:
+  print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
+  sys.exit(1)
+
+def getTuples(queryName, jsonResult, fname):
+  if queryName not in jsonResult or "tuples" not in jsonResult[queryName]:
+    print("Failed to read generated tests: expected key '%s' with a 'tuples' subkey in file '%s'" % (queryName, fname), file = sys.stderr)
+    sys.exit(1)
+  return jsonResult[queryName]["tuples"]
+
+with open(generatedJson, "r") as f:
+  generateOutput = json.load(f)
+  expectedTables = ("getTestCase", "getASupportMethodModel", "missingSummaryModelCsv", "getAParseFailure")
+
+  testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
+    tuple([getTuples(k, generateOutput, generatedJson) for k in expectedTables])
+
+  if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
+    print("Expected exactly one getTestCase result with one column (got: %s)" % json.dumps(testCaseRows), file = sys.stderr)
+  if any(len(row) != 1 for row in supportModelRows):
+    print("Expected exactly one column in getASupportMethodModel relation (got: %s)" % json.dumps(supportModelRows), file = sys.stderr)
+  if any(len(row) != 2 for row in parseFailureRows):
+    print("Expected exactly two columns in parseFailureRows relation (got: %s)" % json.dumps(parseFailureRows), file = sys.stderr)
+
+  if len(missingSummaryModelCsvRows) != 0:
+    print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" + "\n".join(r[0] for r in missingSummaryModelCsvRows))
+    sys.exit(1)
+  if len(parseFailureRows) != 0:
+    print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file = sys.stderr)
+    sys.exit(1)
+
+with open(resultJava, "w") as f:
+  f.write(generateOutput["getTestCase"]["tuples"][0][0])
+
+scriptPath = os.path.dirname(sys.argv[0])
+
+def copyfile(fromName, toFileHandle):
+  with open(os.path.join(scriptPath, fromName), "r") as fromFileHandle:
+    shutil.copyfileobj(fromFileHandle, toFileHandle)
+
+with open(resultQl, "w") as f:
+  copyfile("testHeader.qlfrag", f)
+  if len(supportModelRows) != 0:
+    copyfile("testModelsHeader.qlfrag", f)
+    f.write(", ".join('"%s"' % modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
+    copyfile("testModelsFooter.qlfrag", f)
+  copyfile("testFooter.qlfrag", f)
+
+# Make an empty .expected file, since this is an inline-exectations test
+with open(os.path.join(sys.argv[3], "test.expected"), "w"):
+  pass
+
+cmd = ['codeql', 'query', 'format', '-qq', '-i', resultQl]
+subprocess.call(cmd)
+
+shutil.rmtree(workDir)