Skip to content

Commit c733606

Browse files
oetroetr
committed
fix: clear last finding before starting a fuzzing run
Prior to this commit, if last input in regression mode resulted in a finding, the first fuzzer input (null bytes) always produced a finding and a crashfile with 0 bytes. This commit clears last finding before starting the fuzzing run
1 parent 1840ec2 commit c733606

File tree

6 files changed

+245
-0
lines changed

6 files changed

+245
-0
lines changed

‎examples/junit/src/test/java/com/example/FuzzTestWithCrashTest.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright 2025 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example;
18+
19+
import com.code_intelligence.jazzer.api.FuzzedDataProvider;
20+
import com.code_intelligence.jazzer.junit.FuzzTest;
21+
import java.util.Objects;
22+
import java.util.regex.Pattern;
23+
import java.util.regex.PatternSyntaxException;
24+
25+
public class FuzzTestWithCrashTest {
26+
@FuzzTest(maxDuration = "10s")
27+
void crashFuzz(FuzzedDataProvider d) {
28+
String input = d.consumeRemainingAsString();
29+
if (Objects.equals(input, "crash")) {
30+
byte[] bytes =
31+
new byte[] {
32+
0xa, 0x5c, 0x45, 0x5d, 0x5c, 0x45, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d, 0x5d,
33+
};
34+
String data = new String(bytes, java.nio.charset.StandardCharsets.UTF_8);
35+
try {
36+
Pattern.matches("\\Q" + data + "\\E", "foobar");
37+
} catch (PatternSyntaxException ignored) {
38+
}
39+
}
40+
}
41+
}

‎examples/junit/src/test/resources/com/example/FuzzTestWithCrashTestInputs/crashFuzz/crash‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
crash

‎src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,13 @@ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext ext
172172
"Regression tests are run instead of fuzzing since JAZZER_FUZZ has not been set to a"
173173
+ " non-empty value");
174174
}
175+
176+
// Clear the last finding before starting a fuzzing run.
177+
try {
178+
getLastFindingField().set(null, null);
179+
} catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException ignore) {
180+
}
181+
175182
// Only fuzz the first @FuzzTest that makes it here.
176183
if (FuzzTestExtensions.fuzzTestMethod.compareAndSet(
177184
null, extensionContext.getRequiredTestMethod())

‎src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,31 @@ java_test(
6666
],
6767
)
6868

69+
java_test(
70+
name = "FuzzingAfterRegressionCrashTest",
71+
srcs = ["FuzzingAfterRegressionCrashTest.java"],
72+
env = {
73+
"JAZZER_FUZZ": "1",
74+
},
75+
test_class = "com.code_intelligence.jazzer.junit.FuzzingAfterRegressionCrashTest",
76+
runtime_deps = [
77+
"//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar",
78+
"@maven//:org_junit_jupiter_junit_jupiter_engine",
79+
],
80+
deps = [
81+
"//src/main/java/com/code_intelligence/jazzer/api:hooks",
82+
"//src/main/java/com/code_intelligence/jazzer/junit:common_exceptions",
83+
"//src/test/java/com/code_intelligence/jazzer/junit:test-method",
84+
"@maven//:com_google_truth_extensions_truth_java8_extension",
85+
"@maven//:com_google_truth_truth",
86+
"@maven//:junit_junit",
87+
"@maven//:org_assertj_assertj_core",
88+
"@maven//:org_junit_platform_junit_platform_engine",
89+
"@maven//:org_junit_platform_junit_platform_testkit",
90+
"@maven//:org_opentest4j_opentest4j",
91+
],
92+
)
93+
6994
[
7095
java_test(
7196
name = "FuzzingWithCrashTest" + JAZZER_FUZZ,

‎src/test/java/com/code_intelligence/jazzer/junit/FuzzingAfterRegressionCrashTest.java‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
/*
2+
* Copyright 2024 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.code_intelligence.jazzer.junit;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
21+
import static org.junit.platform.testkit.engine.EventConditions.container;
22+
import static org.junit.platform.testkit.engine.EventConditions.displayName;
23+
import static org.junit.platform.testkit.engine.EventConditions.event;
24+
import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
25+
import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
26+
import static org.junit.platform.testkit.engine.EventConditions.test;
27+
import static org.junit.platform.testkit.engine.EventConditions.type;
28+
import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
29+
import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED;
30+
import static org.junit.platform.testkit.engine.EventType.FINISHED;
31+
import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED;
32+
import static org.junit.platform.testkit.engine.EventType.STARTED;
33+
import static org.junit.platform.testkit.engine.TestExecutionResultConditions.cause;
34+
import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
35+
36+
import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow;
37+
import java.io.IOException;
38+
import java.nio.file.Files;
39+
import java.nio.file.Path;
40+
import java.util.List;
41+
import java.util.stream.Collectors;
42+
import java.util.stream.Stream;
43+
import org.junit.Before;
44+
import org.junit.Rule;
45+
import org.junit.Test;
46+
import org.junit.platform.testkit.engine.EngineExecutionResults;
47+
import org.junit.platform.testkit.engine.EngineTestKit;
48+
import org.junit.rules.TemporaryFolder;
49+
50+
public class FuzzingAfterRegressionCrashTest {
51+
private static final String ENGINE = "engine:junit-jupiter";
52+
private static final String INVOCATION = "test-template-invocation:#";
53+
private static final String CLAZZ_NAME = "com.example.FuzzTestWithCrashTest";
54+
private static final String CLAZZ = "class:" + CLAZZ_NAME;
55+
private static final TestMethod CRASH_FUZZ =
56+
new TestMethod(CLAZZ_NAME, "crashFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)");
57+
58+
@Rule public TemporaryFolder temp = new TemporaryFolder();
59+
Path baseDir;
60+
61+
@Before
62+
public void setup() throws IOException {
63+
baseDir = temp.getRoot().toPath();
64+
}
65+
66+
private EngineExecutionResults executeTests() {
67+
return EngineTestKit.engine("junit-jupiter")
68+
.selectors(selectClass(CLAZZ_NAME))
69+
.configurationParameter(
70+
"jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*")
71+
.configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
72+
.execute();
73+
}
74+
75+
@Test
76+
public void fuzzingAfterRegressionCrashTest() throws IOException {
77+
EngineExecutionResults results = executeTests();
78+
results
79+
.containerEvents()
80+
.assertEventsMatchExactly(
81+
event(type(STARTED), container(ENGINE)),
82+
event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
83+
event(
84+
type(STARTED),
85+
container(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId()))),
86+
event(
87+
type(REPORTING_ENTRY_PUBLISHED),
88+
container(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId()))),
89+
event(
90+
type(REPORTING_ENTRY_PUBLISHED),
91+
container(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId()))),
92+
event(
93+
type(FINISHED),
94+
container(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId())),
95+
finishedSuccessfully()),
96+
event(
97+
type(FINISHED),
98+
container(uniqueIdSubstrings(ENGINE, CLAZZ)),
99+
finishedSuccessfully()),
100+
event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
101+
102+
results
103+
.testEvents()
104+
.assertEventsMatchLooselyInOrder(
105+
event(
106+
type(DYNAMIC_TEST_REGISTERED),
107+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId())),
108+
displayName("<empty input>")),
109+
event(
110+
type(STARTED),
111+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId(), INVOCATION)),
112+
displayName("<empty input>")),
113+
event(
114+
type(FINISHED),
115+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId(), INVOCATION)),
116+
displayName("<empty input>"),
117+
finishedSuccessfully()),
118+
event(
119+
type(DYNAMIC_TEST_REGISTERED),
120+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId())),
121+
displayName("crash")),
122+
event(
123+
type(STARTED),
124+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId(), INVOCATION)),
125+
displayName("crash")),
126+
event(
127+
type(FINISHED),
128+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId(), INVOCATION)),
129+
displayName("crash"),
130+
finishedWithFailure(
131+
instanceOf(FuzzTestFindingException.class),
132+
cause(instanceOf(FuzzerSecurityIssueLow.class)))),
133+
event(
134+
type(DYNAMIC_TEST_REGISTERED),
135+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId())),
136+
displayName("Fuzzing...")),
137+
event(
138+
type(STARTED),
139+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId(), INVOCATION)),
140+
displayName("Fuzzing...")),
141+
event(
142+
type(FINISHED),
143+
test(uniqueIdSubstrings(ENGINE, CLAZZ, CRASH_FUZZ.getDescriptorId(), INVOCATION)),
144+
displayName("Fuzzing..."),
145+
finishedWithFailure(
146+
instanceOf(FuzzTestFindingException.class),
147+
cause(instanceOf(FuzzerSecurityIssueLow.class)))));
148+
149+
try (Stream<Path> stream =
150+
Files.list(baseDir).filter(f -> f.getFileName().toString().startsWith("crash-"))) {
151+
List<Path> entries = stream.collect(Collectors.toList());
152+
153+
// Ensure that there is at least one crash file.
154+
assertThat(entries).hasSize(1);
155+
156+
// Ensure that no crash file has 0-bytes (aka. <empty input>).
157+
// This is to catch the bug where if the last input in regression mode caused a crash, Jazzer
158+
// would register a crash for the very first input in fuzzing mode, which is an empty input.
159+
entries.forEach(
160+
entry -> {
161+
try {
162+
assertThat(Files.size(entry)).isGreaterThan(0);
163+
} catch (IOException e) {
164+
throw new RuntimeException(e);
165+
}
166+
});
167+
}
168+
}
169+
}

‎src/test/java/com/code_intelligence/jazzer/junit/RegressionTestTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import static com.google.common.truth.Truth.assertThat;
2020
import static org.junit.Assume.assumeTrue;
21+
import static org.junit.platform.engine.discovery.ClassNameFilter.excludeClassNamePatterns;
2122
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectPackage;
2223
import static org.junit.platform.testkit.engine.EventConditions.container;
2324
import static org.junit.platform.testkit.engine.EventConditions.displayName;
@@ -69,6 +70,7 @@ public class RegressionTestTest {
6970
private static EngineExecutionResults executeTests() {
7071
return EngineTestKit.engine("junit-jupiter")
7172
.selectors(selectPackage("com.example"))
73+
.filters(excludeClassNamePatterns("com.example.FuzzTestWithCrashTest"))
7274
.configurationParameter(
7375
"jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*")
7476
.configurationParameter("jazzer.mutator_framework", "false")

0 commit comments

Comments
 (0)