Including SEED_DIR argument when fuzzing with zest: Fuzzing stopped due to guidance exception: java.lang.IllegalArgumentException: Input is either empty or nothing was requested from the input generator.
MuxiLyuLucy opened this issue · 4 comments
I am trying to fuzz using JQF + Zest using this command:
Usage: ./jqf-zest [-c CLASSPATH] [-v] TEST_CLASS TEST_METHOD [OUT_DIR [SEED_FILES...]]
The seed file is in the format of string. The test method takes InputStream as its parameter.
Here is the error message when I run jqf-zest:
.Fuzzing stopped due to guidance exception: java.lang.IllegalArgumentException: Input is either empty or nothing was requested from the input generator.
E
Time: ...
There was 1 failure:
1) ...
edu.berkeley.cs.jqf.fuzz.guidance.GuidanceException: java.lang.IllegalArgumentException: Input is either empty or nothing was requested from the input generator.
at edu.berkeley.cs.jqf.fuzz.junit.quickcheck.FuzzStatement.evaluate(FuzzStatement.java:181)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing.run(GuidedFuzzing.java:213)
at edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing.run(GuidedFuzzing.java:159)
at edu.berkeley.cs.jqf.fuzz.junit.GuidedFuzzing.run(GuidedFuzzing.java:123)
at edu.berkeley.cs.jqf.fuzz.ei.ZestDriver.main(ZestDriver.java:77)
Caused by: java.lang.IllegalArgumentException: Input is either empty or nothing was requested from the input generator.
at edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance$LinearInput.gc(ZestGuidance.java:1287)
at edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance.lambda$handleResult$5(ZestGuidance.java:766)
at edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance.conditionallySynchronize(ZestGuidance.java:1060)
at edu.berkeley.cs.jqf.fuzz.ei.ZestGuidance.handleResult(ZestGuidance.java:736)
at edu.berkeley.cs.jqf.fuzz.junit.quickcheck.FuzzStatement.evaluate(FuzzStatement.java:176)
... 16 more
FAILURES!!!
Tests run: 1, Failures: 1
What is the test method that you are fuzzing? It looks like you are not doing anything with the string, and so the InputStream is not being queried at all. You will get the error above if you have a test method like this:
@Fuzz
public void doNothing(InputStream in) {
// Do nothing with `in`
}
Because the InputStream can be of arbitrary length, Zest will not be able to mutate streams if zero bytes are consumed by the test method.
PS: Are you getting the error ONLY when providing SEED_DIR but the fuzzing works fine if you don't provide any seeds? Do you have any empty seed files?
Thank you for your help! This is our test method:
package htmlpublisher;
import hudson.model.FreeStyleProject;
import hudson.tasks.Shell;
import org.htmlunit.AlertHandler;
import org.htmlunit.Page;
import org.junit.Rule;
import org.junit.Test;
import org.jvnet.hudson.test.JenkinsRule;
import com.code_intelligence.jazzer.api.BugDetectors;
import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; // we need this to trigger the jazzer sanitizer
import org.junit.Before;
import org.junit.After;
import java.util.ArrayList;
import java.util.List;
import java.nio.charset.StandardCharsets;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import java.io.*;
import org.junit.runner.RunWith;
import static org.junit.Assert.*;
import com.pholser.junit.quickcheck.*;
import com.pholser.junit.quickcheck.generator.*;
import edu.berkeley.cs.jqf.fuzz.*;
@RunWith(JQF.class)
public class Security3302FuzzHarnessForJQF {
private static CustomJenkinsRule j;
private static void setUp() throws Throwable {
j = new CustomJenkinsRule();
j.setTestDescription(Description.createTestDescription(Security3302FuzzHarnessForJQF.class, "fuzzerTest"));
j.before();
}
public static void tearDown() throws Throwable {
if (j != null) {
j.after();
}
}
// TODO: use LLM to automatically write the generator
@Fuzz
public void fuzzerTestOneInput(InputStream data) throws Throwable {
setUp();
String jobName = new String(data.readAllBytes(), StandardCharsets.UTF_8);
try {
System.out.println("jobName: " + jobName);
FreeStyleProject job = j.jenkins.createProject(FreeStyleProject.class, jobName);
job.getBuildersList().add(new Shell("date > index.html"));
HtmlPublisherTarget target = new HtmlPublisherTarget(
"HTML Report",
"",
"index.html",
true,
false,
false
);
target.setUseWrapperFileDirectly(true);
target.setEscapeUnderscores(true);
target.setReportTitles("");
target.setIncludes("**/*");
List<HtmlPublisherTarget> reportTargets = new ArrayList<>();
reportTargets.add(target);
job.getPublishersList().add(new HtmlPublisher(reportTargets));
j.buildAndAssertSuccess(job);
JenkinsRule.WebClient client = j.createWebClient();
// Create an alert handler to check for any alerts
Alerter alerter = new Alerter();
client.setAlertHandler(alerter);
client.goTo("job/" + jobName + "/HTML_20Report/");
// Check that the alerter has not been triggered
client.waitForBackgroundJavaScript(2000);
if (!alerter.messages.isEmpty()) {
throw new FuzzerSecurityIssueMedium(); // jazzer sanitizer
}
} catch (Exception e) {
if (e instanceof FuzzerSecurityIssueMedium) {
System.out.println("Jazzer sanitizer triggered by jobName: " + jobName);
// throw e;
assertTrue("Exception FuzzerSecurityIssueMedium Raise!", false);
}
System.err.println("Caught exception: " + e.getMessage());
} finally {
tearDown();
}
}
private static class Alerter implements AlertHandler {
List<String> messages = new ArrayList<>();
@Override
public void handleAlert(Page page, String message) {
messages.add(message);
}
}
private static class CustomJenkinsRule extends JenkinsRule {
public void setTestDescription(Description description) {
this.testDescription = description;
}
}
}
And we are experiencing the same problem BOTH with and without SEED_DIR.
We don't have any empty seed files.
That's strange! I'm wondering if it is because something is happening during the setUp()
phase (e.g., an exception) that is preventing the call to data.readAllBytes()
and therefore not querying the InputStream.
Can I suggest you try to use a different strategy to get string inputs from JQF? Try using the ArbitraryStringLengthGenerator
(this class is in the examples
module, so if you can't import that package feel free to copy this class into your project) and then change your test driver to expect a String
type input as follows: https://github.com/rohanpadhye/JQF/blob/jqf-2.0/examples/src/test/java/edu/berkeley/cs/jqf/examples/commons/MathTest.java#L42
The ArbitraryStringLengthGenerator
is just a wrapper around InputStreamGenerator
that reads all the bytes eagerly before test execution, so it should hopefully make your error go away and also allow having human-readable seed files and corpus.
Thank you so much! I have tried ArbitraryStringLengthGenerator and it works.
package htmlpublisher;
import hudson.model.FreeStyleProject;
import hudson.tasks.Shell;
import org.htmlunit.AlertHandler;
import org.htmlunit.Page;
import org.junit.Rule;
import org.junit.Test;
import org.jvnet.hudson.test.JenkinsRule;
import com.code_intelligence.jazzer.api.BugDetectors;
import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium;
import org.junit.Before;
import org.junit.After;
import java.util.ArrayList;
import java.util.List;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import org.junit.runner.RunWith;
import com.pholser.junit.quickcheck.*;
import com.pholser.junit.quickcheck.generator.*;
import com.pholser.junit.quickcheck.random.SourceOfRandomness;
import edu.berkeley.cs.jqf.fuzz.junit.quickcheck.InputStreamGenerator;
import edu.berkeley.cs.jqf.fuzz.*;
import htmlpublisher.ArbitraryLengthStringGenerator;
import static org.junit.Assert.*;
@RunWith(JQF.class)
public class Security3302FuzzHarnessForJQF {
private static CustomJenkinsRule j;
private static void setUp() throws Throwable {
j = new CustomJenkinsRule();
j.setTestDescription(Description.createTestDescription(Security3302FuzzHarnessForJQF.class, "fuzzerTest"));
j.before();
}
public static void tearDown() throws Throwable {
if (j != null) {
j.after();
}
}
@Fuzz
public void fuzzerTestOneInput(@From(ArbitraryLengthStringGenerator.class) String jobName) throws Throwable {
setUp();
try {
System.out.println("jobName: " + jobName);
FreeStyleProject job = j.jenkins.createProject(FreeStyleProject.class, jobName);
job.getBuildersList().add(new Shell("date > index.html"));
HtmlPublisherTarget target = new HtmlPublisherTarget(
"HTML Report",
"",
"index.html",
true,
false,
false
);
target.setUseWrapperFileDirectly(true);
target.setEscapeUnderscores(true);
target.setReportTitles("");
target.setIncludes("**/*");
List<HtmlPublisherTarget> reportTargets = new ArrayList<>();
reportTargets.add(target);
job.getPublishersList().add(new HtmlPublisher(reportTargets));
j.buildAndAssertSuccess(job);
JenkinsRule.WebClient client = j.createWebClient();
// Create an alert handler to check for any alerts
Alerter alerter = new Alerter();
client.setAlertHandler(alerter);
client.goTo("job/" + jobName + "/HTML_20Report/");
// Check that the alerter has not been triggered
client.waitForBackgroundJavaScript(2000);
if (!alerter.messages.isEmpty()) {
throw new FuzzerSecurityIssueMedium(); // jazzer sanitizer
}
} catch (Exception e) {
if (e instanceof FuzzerSecurityIssueMedium) {
System.out.println("Jazzer sanitizer triggered by jobName: " + jobName);
// throw e;
assertTrue("Exception FuzzerSecurityIssueMedium Raise!", false);
}
System.err.println("Caught exception: " + e.getMessage());
} finally {
tearDown();
}
}
private static class Alerter implements AlertHandler {
List<String> messages = new ArrayList<>();
@Override
public void handleAlert(Page page, String message) {
messages.add(message);
}
}
private static class CustomJenkinsRule extends JenkinsRule {
public void setTestDescription(Description description) {
this.testDescription = description;
}
}
}
I will also look into setUp() and try to figure out why InputSteam didn't work.