Merge pull request github#6373 from joefarebrother/test-gen-improvements

Java: Test generator improvements

Author: joefarebrother

java/ql/src/utils/GenerateFlowTestCase.py

@@ -12,8 +12,8 @@
 import tempfile
 
 if any(s == "--help" for s in sys.argv):
-  print("""Usage:
-GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir
+    print("""Usage:
+GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir [--force]
 
 This generates test cases exercising function model specifications found in specsToTest.csv
 producing files Test.java, test.ql and test.expected in outdir.
@@ -22,32 +22,41 @@
 Typically this means supplying a skeleton POM <dependencies> section that retrieves whatever jars
 contain the needed classes.
 
+If --force is present, existing files may be overwritten.
+
 Requirements: `mvn` and `codeql` should both appear on your path.
 
 After test generation completes, any lines in specsToTest.csv that didn't produce tests are output.
 If this happens, check the spelling of class and method names, and the syntax of input and output specifications.
 """)
-  sys.exit(0)
+    sys.exit(0)
+
+force = False
+if "--force" in sys.argv:
+    sys.argv.remove("--force")
+    force = True
 
 if len(sys.argv) != 4:
-  print("Usage: GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir", file=sys.stderr)
-  print("specsToTest.csv should contain CSV rows describing method taint-propagation specifications to test", file=sys.stderr)
-  print("projectPom.xml should import dependencies sufficient to resolve the types used in specsToTest.csv", file=sys.stderr)
-  sys.exit(1)
+    print(
+        "Usage: GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir [--force]", file=sys.stderr)
+    print("specsToTest.csv should contain CSV rows describing method taint-propagation specifications to test", file=sys.stderr)
+    print("projectPom.xml should import dependencies sufficient to resolve the types used in specsToTest.csv", file=sys.stderr)
+    sys.exit(1)
 
 try:
-  os.makedirs(sys.argv[3])
+    os.makedirs(sys.argv[3])
 except Exception as e:
-  if e.errno != errno.EEXIST:
-    print("Failed to create output directory %s: %s" % (sys.argv[3], e))
-    sys.exit(1)
+    if e.errno != errno.EEXIST:
+        print("Failed to create output directory %s: %s" % (sys.argv[3], e))
+        sys.exit(1)
 
 resultJava = os.path.join(sys.argv[3], "Test.java")
 resultQl = os.path.join(sys.argv[3], "test.ql")
 
-if os.path.exists(resultJava) or os.path.exists(resultQl):
-  print("Won't overwrite existing files '%s' or '%s'" % (resultJava, resultQl), file = sys.stderr)
-  sys.exit(1)
+if not force and (os.path.exists(resultJava) or os.path.exists(resultQl)):
+    print("Won't overwrite existing files '%s' or '%s'" %
+          (resultJava, resultQl), file=sys.stderr)
+    sys.exit(1)
 
 workDir = tempfile.mkdtemp()
 
@@ -57,129 +66,159 @@
 os.makedirs(projectDir)
 
 try:
-  shutil.copyfile(sys.argv[2], os.path.join(projectDir, "pom.xml"))
+    shutil.copyfile(sys.argv[2], os.path.join(projectDir, "pom.xml"))
 except Exception as e:
-  print("Failed to read project POM %s: %s" % (sys.argv[2], e), file = sys.stderr)
-  sys.exit(1)
+    print("Failed to read project POM %s: %s" %
+          (sys.argv[2], e), file=sys.stderr)
+    sys.exit(1)
 
 commentRegex = re.compile("^\s*(//|#)")
+
+
 def isComment(s):
-  return commentRegex.match(s) is not None
+    return commentRegex.match(s) is not None
+
 
 try:
-  with open(sys.argv[1], "r") as f:
-    specs = [l for l in f if not isComment(l)]
+    with open(sys.argv[1], "r") as f:
+        specs = [l for l in f if not isComment(l)]
 except Exception as e:
-  print("Failed to open %s: %s\n" % (sys.argv[1], e))
-  sys.exit(1)
+    print("Failed to open %s: %s\n" % (sys.argv[1], e))
+    sys.exit(1)
 
 projectTestPkgDir = os.path.join(projectDir, "src", "main", "java", "test")
 projectTestFile = os.path.join(projectTestPkgDir, "Test.java")
 
 os.makedirs(projectTestPkgDir)
 
+
 def qualifiedOuterNameFromCsvRow(row):
-  cells = row.split(";")
-  if len(cells) < 2:
-    return None
-  return cells[0] + "." + cells[1].replace("$", ".")
+    cells = row.split(";")
+    if len(cells) < 2:
+        return None
+    return cells[0] + "." + cells[1].replace("$", ".")
+
 
 with open(projectTestFile, "w") as testJava:
-  testJava.write("package test;\n\npublic class Test {\n\n")
+    testJava.write("package test;\n\npublic class Test {\n\n")
 
-  for i, spec in enumerate(specs):
-    outerName = qualifiedOuterNameFromCsvRow(spec)
-    if outerName is None:
-      print("A taint specification has the wrong format: should be 'package;classname;methodname....'", file = sys.stderr)
-      print("Mis-formatted row: " + spec, file = sys.stderr)
-      sys.exit(1)
-    testJava.write("\t%s obj%d = null;\n" % (outerName, i))
+    for i, spec in enumerate(specs):
+        outerName = qualifiedOuterNameFromCsvRow(spec)
+        if outerName is None:
+            print("A taint specification has the wrong format: should be 'package;classname;methodname....'", file=sys.stderr)
+            print("Mis-formatted row: " + spec, file=sys.stderr)
+            sys.exit(1)
+        testJava.write("\t%s obj%d = null;\n" % (outerName, i))
 
-  testJava.write("}")
+    testJava.write("}")
 
 print("Creating project database")
 cmd = ["codeql", "database", "create", "--language=java", "db"]
-ret = subprocess.call(cmd, cwd = projectDir)
+ret = subprocess.call(cmd, cwd=projectDir)
 if ret != 0:
-  print("Failed to create project database. Check that '%s' is a valid POM that pulls in all necessary dependencies, and '%s' specifies valid classes and methods." % (sys.argv[2], sys.argv[1]), file = sys.stderr)
-  print("Failed command was: %s (cwd: %s)" % (shlex.join(cmd), projectDir), file = sys.stderr)
-  sys.exit(1)
+    print("Failed to create project database. Check that '%s' is a valid POM that pulls in all necessary dependencies, and '%s' specifies valid classes and methods." % (
+        sys.argv[2], sys.argv[1]), file=sys.stderr)
+    print("Failed command was: %s (cwd: %s)" %
+          (shlex.join(cmd), projectDir), file=sys.stderr)
+    sys.exit(1)
 
 print("Creating test-generation query")
 queryDir = os.path.join(workDir, "query")
 os.makedirs(queryDir)
 qlFile = os.path.join(queryDir, "gen.ql")
 with open(os.path.join(queryDir, "qlpack.yml"), "w") as f:
-  f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql-java")
+    f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql-java")
 with open(qlFile, "w") as f:
-  f.write("import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
-  f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
-  f.write("\n\t\t]\n\t}\n}\n")
+    f.write(
+        "import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
+    f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
+    f.write("\n\t\t]\n\t}\n}\n")
 
 print("Generating tests")
 generatedBqrs = os.path.join(queryDir, "out.bqrs")
-cmd = ['codeql', 'query', 'run', qlFile, '--database', os.path.join(projectDir, "db"), '--output', generatedBqrs]
+cmd = ['codeql', 'query', 'run', qlFile, '--database',
+       os.path.join(projectDir, "db"), '--output', generatedBqrs]
 ret = subprocess.call(cmd)
 if ret != 0:
-  print("Failed to generate tests. Failed command was: " + shlex.join(cmd))
-  sys.exit(1)
+    print("Failed to generate tests. Failed command was: " + shlex.join(cmd))
+    sys.exit(1)
 
 generatedJson = os.path.join(queryDir, "out.json")
-cmd = ['codeql', 'bqrs', 'decode', generatedBqrs, '--format=json', '--output', generatedJson]
+cmd = ['codeql', 'bqrs', 'decode', generatedBqrs,
+       '--format=json', '--output', generatedJson]
 ret = subprocess.call(cmd)
 if ret != 0:
-  print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
-  sys.exit(1)
-
-def getTuples(queryName, jsonResult, fname):
-  if queryName not in jsonResult or "tuples" not in jsonResult[queryName]:
-    print("Failed to read generated tests: expected key '%s' with a 'tuples' subkey in file '%s'" % (queryName, fname), file = sys.stderr)
+    print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
     sys.exit(1)
-  return jsonResult[queryName]["tuples"]
 
-with open(generatedJson, "r") as f:
-  generateOutput = json.load(f)
-  expectedTables = ("getTestCase", "getASupportMethodModel", "missingSummaryModelCsv", "getAParseFailure")
 
-  testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
-    tuple([getTuples(k, generateOutput, generatedJson) for k in expectedTables])
+def getTuples(queryName, jsonResult, fname):
+    if queryName not in jsonResult or "tuples" not in jsonResult[queryName]:
+        print("Failed to read generated tests: expected key '%s' with a 'tuples' subkey in file '%s'" % (
+            queryName, fname), file=sys.stderr)
+        sys.exit(1)
+    return jsonResult[queryName]["tuples"]
 
-  if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
-    print("Expected exactly one getTestCase result with one column (got: %s)" % json.dumps(testCaseRows), file = sys.stderr)
-  if any(len(row) != 1 for row in supportModelRows):
-    print("Expected exactly one column in getASupportMethodModel relation (got: %s)" % json.dumps(supportModelRows), file = sys.stderr)
-  if any(len(row) != 2 for row in parseFailureRows):
-    print("Expected exactly two columns in parseFailureRows relation (got: %s)" % json.dumps(parseFailureRows), file = sys.stderr)
 
-  if len(missingSummaryModelCsvRows) != 0:
-    print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" + "\n".join(r[0] for r in missingSummaryModelCsvRows))
-    sys.exit(1)
-  if len(parseFailureRows) != 0:
-    print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file = sys.stderr)
-    sys.exit(1)
+with open(generatedJson, "r") as f:
+    generateOutput = json.load(f)
+    expectedTables = ("getTestCase", "getASupportMethodModel",
+                      "missingSummaryModelCsv", "getAParseFailure", "noTestCaseGenerated")
+
+    testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows, noTestCaseGeneratedRows = \
+        tuple([getTuples(k, generateOutput, generatedJson)
+               for k in expectedTables])
+
+    if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
+        print("Expected exactly one getTestCase result with one column (got: %s)" %
+              json.dumps(testCaseRows), file=sys.stderr)
+    if any(len(row) != 1 for row in supportModelRows):
+        print("Expected exactly one column in getASupportMethodModel relation (got: %s)" %
+              json.dumps(supportModelRows), file=sys.stderr)
+    if any(len(row) != 2 for row in parseFailureRows):
+        print("Expected exactly two columns in parseFailureRows relation (got: %s)" %
+              json.dumps(parseFailureRows), file=sys.stderr)
+    if any(len(row) != 1 for row in noTestCaseGeneratedRows):
+        print("Expected exactly one column in noTestCaseGenerated relation (got: %s)" %
+              json.dumps(noTestCaseGeneratedRows), file=sys.stderr)
+
+    if len(missingSummaryModelCsvRows) != 0:
+        print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" +
+              "\n".join(r[0] for r in missingSummaryModelCsvRows))
+        sys.exit(1)
+    if len(parseFailureRows) != 0:
+        print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" %
+              "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file=sys.stderr)
+        sys.exit(1)
+    if len(noTestCaseGeneratedRows) != 0:
+        print("The following CSV rows failed to generate any test case due to a limitation of the query. Other test cases will still be generated:\n" +
+              "\n".join(r[0] for r in noTestCaseGeneratedRows))
 
 with open(resultJava, "w") as f:
-  f.write(generateOutput["getTestCase"]["tuples"][0][0])
+    f.write(generateOutput["getTestCase"]["tuples"][0][0])
 
 scriptPath = os.path.dirname(sys.argv[0])
 
+
 def copyfile(fromName, toFileHandle):
-  with open(os.path.join(scriptPath, fromName), "r") as fromFileHandle:
-    shutil.copyfileobj(fromFileHandle, toFileHandle)
+    with open(os.path.join(scriptPath, fromName), "r") as fromFileHandle:
+        shutil.copyfileobj(fromFileHandle, toFileHandle)
+
 
 with open(resultQl, "w") as f:
-  copyfile("testHeader.qlfrag", f)
-  if len(supportModelRows) != 0:
-    copyfile("testModelsHeader.qlfrag", f)
-    f.write(", ".join('"%s"' % modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
-    copyfile("testModelsFooter.qlfrag", f)
-  copyfile("testFooter.qlfrag", f)
+    copyfile("testHeader.qlfrag", f)
+    if len(supportModelRows) != 0:
+        copyfile("testModelsHeader.qlfrag", f)
+        f.write(", ".join('"%s"' %
+                          modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
+        copyfile("testModelsFooter.qlfrag", f)
+    copyfile("testFooter.qlfrag", f)
 
 # Make an empty .expected file, since this is an inline-exectations test
 with open(os.path.join(sys.argv[3], "test.expected"), "w"):
-  pass
+    pass
 
 cmd = ['codeql', 'query', 'format', '-qq', '-i', resultQl]
 subprocess.call(cmd)
 
-shutil.rmtree(workDir)
+shutil.rmtree(workDir)

java/ql/src/utils/GenerateFlowTestCase.qll

@@ -62,13 +62,26 @@ query string getAParseFailure(string reason) {
   )
 }
 
+/**
+ * Gets a CSV row for which a test was requested and was correctly parsed,
+ * but for which no test case could be generated due to a limitation of the query.
+ */
+query string noTestCaseGenerated() {
+  any(TargetSummaryModelCsv target).row(result) and
+  any(SummaryModelCsv model).row(result) and
+  not exists(getAParseFailure(_)) and
+  not exists(any(TestCase tc).getATestSnippetForRow(result))
+}
+
 private class CallableToTest extends Callable {
   CallableToTest() {
     exists(
       string namespace, string type, boolean subtypes, string name, string signature, string ext
     |
       summaryModel(namespace, type, subtypes, name, signature, ext, _, _, _) and
-      this = interpretElement(namespace, type, subtypes, name, signature, ext)
+      this = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      this.isPublic() and
+      getRootType(this.getDeclaringType()).isPublic()
     )
   }
 }
@@ -148,7 +161,10 @@ string contentToken(Content c) {
 RefType getRootType(RefType t) {
   if t instanceof NestedType
   then result = getRootType(t.(NestedType).getEnclosingType())
-  else result = t
+  else
+    if t instanceof Array
+    then result = getRootType(t.(Array).getElementType())
+    else result = t
 }
 
 /**
@@ -485,18 +501,22 @@ predicate isImportable(Type t) {
  * if we cannot import it due to a name clash.
  */
 string getShortNameIfPossible(Type t) {
-  getRootSourceDeclaration(t) = any(TestCase tc).getADesiredImport() and
-  if t instanceof RefType
-  then
-    exists(RefType replaced, string nestedName |
-      replaced = replaceTypeVariable(t).getSourceDeclaration() and
-      nestedName = replaced.nestedName().replaceAll("$", ".")
-    |
-      if isImportable(getRootSourceDeclaration(t))
-      then result = nestedName
-      else result = replaced.getPackage().getName() + "." + nestedName
-    )
-  else result = t.getName()
+  if t instanceof Array
+  then result = getShortNameIfPossible(t.(Array).getElementType()) + "[]"
+  else (
+    getRootSourceDeclaration(t) = any(TestCase tc).getADesiredImport() and
+    if t instanceof RefType
+    then
+      exists(RefType replaced, string nestedName |
+        replaced = replaceTypeVariable(t).getSourceDeclaration() and
+        nestedName = replaced.nestedName().replaceAll("$", ".")
+      |
+        if isImportable(getRootSourceDeclaration(t))
+        then result = nestedName
+        else result = replaced.getPackage().getName() + "." + nestedName
+      )
+    else result = t.getName()
+  )
 }
 
 /**
@@ -533,7 +553,7 @@ query string getTestCase() {
   result =
     "package generatedtest;\n\n" + concat(getAnImportStatement() + "\n") +
       "\n// Test case generated by GenerateFlowTestCase.ql\npublic class Test {\n\n" +
-      concat("\t" + getASupportMethod() + "\n") + "\n\tpublic void test() {\n\n" +
+      concat("\t" + getASupportMethod() + "\n") + "\n\tpublic void test() throws Exception {\n\n" +
       concat(string row, string snippet |
         snippet = any(TestCase tc).getATestSnippetForRow(row)
       |