Merge pull request #11116 from github/redsun82/swift-expect-failure-in-qltest

Swift: allow expecting failure in qltest.sh

Author: redsun82

swift/tools/qltest.sh

@@ -12,8 +12,12 @@ SDK="$CODEQL_EXTRACTOR_SWIFT_ROOT/qltest/$CODEQL_PLATFORM/sdk"
 for src in *.swift; do
   opts=(-sdk "$SDK" -c -primary-file "$src")
   opts+=($(sed -n '1 s=//codeql-extractor-options:==p' $src))
+  expected_status=$(sed -n 's=//codeql-extractor-expected-status:\s*==p' $src)
+  expected_status=${expected_status:-0}
   echo -e "calling extractor with flags: ${opts[@]}\n" >> $QLTEST_LOG
-  if ! "$EXTRACTOR" "${opts[@]}" >> $QLTEST_LOG 2>&1; then
+  "$EXTRACTOR" "${opts[@]}" >> $QLTEST_LOG 2>&1
+  actual_status=$?
+  if [[ $actual_status != $expected_status ]]; then
     FAILED=1
   fi
 done

swift/tools/test/qltest/expected_failure_codes/a.swift

@@ -0,0 +1 @@
+//codeql-extractor-expected-status: 42

swift/tools/test/qltest/expected_failure_codes/b.swift

@@ -0,0 +1 @@
+//codeql-extractor-expected-status: 1

swift/tools/test/qltest/expected_failure_codes/test.py

@@ -0,0 +1,11 @@
+from swift.tools.test.qltest.utils import *
+
+set_dummy_extractor("""
+if [[ " $@ " =~ a.swift ]]; then exit 42; fi
+if [[ " $@ " =~ b.swift ]]; then exit 1; fi
+""")
+run_qltest()
+assert_extractor_executed_with(
+    "a.swift",
+    "b.swift",
+)

swift/tools/test/qltest/unexpected_return_code/a.swift

@@ -0,0 +1 @@
+//codeql-extractor-expected-status: 1

swift/tools/test/qltest/unexpected_return_code/b.swift

@@ -0,0 +1,9 @@
+from swift.tools.test.qltest.utils import *
+
+set_dummy_extractor()
+run_qltest(expected_returncode=1)
+assert_extractor_executed_with(
+    "a.swift",
+    "b.swift",
+    "c.swift",
+)