From 32933c4e6edfa00e0e3123f6ec54c9ac286950ba Mon Sep 17 00:00:00 2001 From: Sascha Knoop Date: Sat, 15 Feb 2025 21:50:35 +0100 Subject: [PATCH 1/2] SonarQube report as Java class --- pom.xml | 5 + ...Benchmark_1.2-ZAPweekly-20150824-18000.xml | 0 .../Benchmark_1.2-ZAPweekly-20160905.xml | 0 .../Benchmark_1.2-findbugs-v3.0.1-92.xml | 0 .../Benchmark_1.2-findsecbugs-v1.4.0-110.xml | 0 .../Benchmark_1.2-findsecbugs-v1.4.3-118.xml | 0 .../Benchmark_1.2-findsecbugs-v1.4.4-253.xml | 0 .../Benchmark_1.2-findsecbugs-v1.4.5-129.xml | 0 .../Benchmark_1.2-findsecbugs-v1.4.6-122.xml | 0 .../{ => old}/Benchmark_1.2-pmd-v5.2.3-11.xml | 0 ...chmark_1.2-sonar-Java-Plugin-v3.14-330.xml | 0 ...Benchmark_1.2-visualcodegrepper-v2.2.0.xml | 0 scripts/runSonarQube.sh | 18 ++- .../report/sonarqube/SonarReport.java | 124 ++++++++++++++++++ .../sonarqube/dto/KeepAsJsonDeserializer.java | 30 +++++ .../report/sonarqube/dto/SonarQubeResult.java | 36 +++++ 16 files changed, 203 insertions(+), 10 deletions(-) rename results/{ => old}/Benchmark_1.2-ZAPweekly-20150824-18000.xml (100%) rename results/{ => old}/Benchmark_1.2-ZAPweekly-20160905.xml (100%) rename results/{ => old}/Benchmark_1.2-findbugs-v3.0.1-92.xml (100%) rename results/{ => old}/Benchmark_1.2-findsecbugs-v1.4.0-110.xml (100%) rename results/{ => old}/Benchmark_1.2-findsecbugs-v1.4.3-118.xml (100%) rename results/{ => old}/Benchmark_1.2-findsecbugs-v1.4.4-253.xml (100%) rename results/{ => old}/Benchmark_1.2-findsecbugs-v1.4.5-129.xml (100%) rename results/{ => old}/Benchmark_1.2-findsecbugs-v1.4.6-122.xml (100%) rename results/{ => old}/Benchmark_1.2-pmd-v5.2.3-11.xml (100%) rename results/{ => old}/Benchmark_1.2-sonar-Java-Plugin-v3.14-330.xml (100%) rename results/{ => old}/Benchmark_1.2-visualcodegrepper-v2.2.0.xml (100%) create mode 100644 src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java create mode 100644 src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java create mode 100644 src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java diff --git a/pom.xml b/pom.xml index 2236ef6233..bf2e377cd2 100644 --- a/pom.xml +++ b/pom.xml @@ -851,6 +851,11 @@ 1.4.01 + + com.fasterxml.jackson.core + jackson-databind + 2.17.2 + diff --git a/results/Benchmark_1.2-ZAPweekly-20150824-18000.xml b/results/old/Benchmark_1.2-ZAPweekly-20150824-18000.xml similarity index 100% rename from results/Benchmark_1.2-ZAPweekly-20150824-18000.xml rename to results/old/Benchmark_1.2-ZAPweekly-20150824-18000.xml diff --git a/results/Benchmark_1.2-ZAPweekly-20160905.xml b/results/old/Benchmark_1.2-ZAPweekly-20160905.xml similarity index 100% rename from results/Benchmark_1.2-ZAPweekly-20160905.xml rename to results/old/Benchmark_1.2-ZAPweekly-20160905.xml diff --git a/results/Benchmark_1.2-findbugs-v3.0.1-92.xml b/results/old/Benchmark_1.2-findbugs-v3.0.1-92.xml similarity index 100% rename from results/Benchmark_1.2-findbugs-v3.0.1-92.xml rename to results/old/Benchmark_1.2-findbugs-v3.0.1-92.xml diff --git a/results/Benchmark_1.2-findsecbugs-v1.4.0-110.xml b/results/old/Benchmark_1.2-findsecbugs-v1.4.0-110.xml similarity index 100% rename from results/Benchmark_1.2-findsecbugs-v1.4.0-110.xml rename to results/old/Benchmark_1.2-findsecbugs-v1.4.0-110.xml diff --git a/results/Benchmark_1.2-findsecbugs-v1.4.3-118.xml b/results/old/Benchmark_1.2-findsecbugs-v1.4.3-118.xml similarity index 100% rename from results/Benchmark_1.2-findsecbugs-v1.4.3-118.xml rename to results/old/Benchmark_1.2-findsecbugs-v1.4.3-118.xml diff --git a/results/Benchmark_1.2-findsecbugs-v1.4.4-253.xml b/results/old/Benchmark_1.2-findsecbugs-v1.4.4-253.xml similarity index 100% rename from results/Benchmark_1.2-findsecbugs-v1.4.4-253.xml rename to results/old/Benchmark_1.2-findsecbugs-v1.4.4-253.xml diff --git a/results/Benchmark_1.2-findsecbugs-v1.4.5-129.xml b/results/old/Benchmark_1.2-findsecbugs-v1.4.5-129.xml similarity index 100% rename from results/Benchmark_1.2-findsecbugs-v1.4.5-129.xml rename to results/old/Benchmark_1.2-findsecbugs-v1.4.5-129.xml diff --git a/results/Benchmark_1.2-findsecbugs-v1.4.6-122.xml b/results/old/Benchmark_1.2-findsecbugs-v1.4.6-122.xml similarity index 100% rename from results/Benchmark_1.2-findsecbugs-v1.4.6-122.xml rename to results/old/Benchmark_1.2-findsecbugs-v1.4.6-122.xml diff --git a/results/Benchmark_1.2-pmd-v5.2.3-11.xml b/results/old/Benchmark_1.2-pmd-v5.2.3-11.xml similarity index 100% rename from results/Benchmark_1.2-pmd-v5.2.3-11.xml rename to results/old/Benchmark_1.2-pmd-v5.2.3-11.xml diff --git a/results/Benchmark_1.2-sonar-Java-Plugin-v3.14-330.xml b/results/old/Benchmark_1.2-sonar-Java-Plugin-v3.14-330.xml similarity index 100% rename from results/Benchmark_1.2-sonar-Java-Plugin-v3.14-330.xml rename to results/old/Benchmark_1.2-sonar-Java-Plugin-v3.14-330.xml diff --git a/results/Benchmark_1.2-visualcodegrepper-v2.2.0.xml b/results/old/Benchmark_1.2-visualcodegrepper-v2.2.0.xml similarity index 100% rename from results/Benchmark_1.2-visualcodegrepper-v2.2.0.xml rename to results/old/Benchmark_1.2-visualcodegrepper-v2.2.0.xml diff --git a/scripts/runSonarQube.sh b/scripts/runSonarQube.sh index 2847ad7007..761294d373 100755 --- a/scripts/runSonarQube.sh +++ b/scripts/runSonarQube.sh @@ -29,10 +29,10 @@ sonar_user="admin" sonar_default_password="admin" sonar_password="P4ssword!!!!" -echo "Creating temporary SonarQube instance..." +docker pull sonarqube +docker pull sonarsource/sonar-scanner-cli -#docker pull sonarqube -#docker pull sonarsource/sonar-scanner-cli +echo "Creating temporary SonarQube instance..." # start local sonarqube docker run --rm -d --name "$container_name" -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p "$sonar_external_port:$sonar_internal_port" sonarqube @@ -44,6 +44,7 @@ while [[ "$(curl --connect-timeout 5 --max-time 5 --retry 60 --retry-delay 0 --r sleep 3 done +echo "" echo "Waiting for SonarQube to become ready..." while [[ "$(curl --silent "$sonar_host/api/system/status" | jq -r '.status')" != "UP" ]]; do @@ -51,6 +52,7 @@ while [[ "$(curl --silent "$sonar_host/api/system/status" | jq -r '.status')" != sleep 3 done +echo "" echo "SonarQube ready. Setting up instance..." # change default password @@ -82,15 +84,11 @@ while [[ "$(curl --silent -u "$sonar_token:" "$sonar_host/api/ce/component?compo sleep 3 done +echo "" echo "Generating report..." -benchmark_version=$(scripts/getBenchmarkVersion.sh) -sonarqube_version=$(curl --silent -u "$sonar_token:" "$sonar_host/api/server/version") -result_file="results/Benchmark_$benchmark_version-sonarqube-v$sonarqube_version.json" - -sonar-report --sonarurl "$sonar_host" --sonarcomponent="$sonar_project" --sonarusername "$sonar_user" --sonarpassword 'P4ssword!!!!' --allbugs --no-rules-in-report --save-report-json "$result_file" +mvn exec:java -Dexec.mainClass="org.owasp.benchmark.report.sonarqube.SonarReport" -echo "Result file written to $result_file" echo "Shutting down SonarQube..." -docker stop "$container_name" +#docker stop "$container_name" diff --git a/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java b/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java new file mode 100644 index 0000000000..21c8f47319 --- /dev/null +++ b/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java @@ -0,0 +1,124 @@ +package org.owasp.benchmark.report.sonarqube; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.owasp.benchmark.report.sonarqube.dto.SonarQubeResult; + +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static java.lang.String.join; +import static java.nio.charset.Charset.defaultCharset; +import static org.apache.commons.io.FileUtils.writeStringToFile; +import static org.apache.commons.io.IOUtils.readLines; + +public class SonarReport { + private static final String SONAR_USER = "admin"; + private static final String SONAR_PASSWORD = "P4ssword!!!!"; + private static final String SONAR_PROJECT = "benchmark"; + public static final String SONAR_HOST = "ubuntu-server"; + public static final String SONAR_PORT = "9876"; + + private static final int PAGE_SIZE = 500; + + private static final String sonarAuth = Base64.getEncoder() + .encodeToString((SONAR_USER + ":" + SONAR_PASSWORD) + .getBytes()); + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + String allJavaRules = String.join(",", allJavaRules()); + List issues = new ArrayList<>(); + List hotspots = new ArrayList<>(); + + forAllPagesAt( + "issues/search?componentKeys=" + SONAR_PROJECT + "&types=VULNERABILITY&&rules=" + allJavaRules, + (result -> issues.addAll(result.issues)) + ); + forAllPagesAt( + "hotspots/search?projectKey=" + SONAR_PROJECT, + (result -> hotspots.addAll(result.hotspots)) + ); + + writeStringToFile( + new File("results/" + resultFilename() + ".json"), + formattedJson(issues, hotspots), + defaultCharset() + ); + } + + private static String resultFilename() throws Exception { + return "Benchmark_" + benchmarkVersion() + "-sonarqube-v" + apiCall("server/version"); + } + + private static String benchmarkVersion() throws Exception { + return DocumentBuilderFactory + .newInstance() + .newDocumentBuilder() + .parse(new File("pom.xml")) + .getElementsByTagName("version") + .item(0) + .getTextContent(); + } + + private static Set allJavaRules() throws IOException { + Set javaRuleIds = new HashSet<>(); + + forAllPagesAt("rules/search", (result) -> result + .rules + .stream().filter(rule -> rule.ruleId.startsWith("java:")) + .forEach(rule -> javaRuleIds.add(rule.ruleId))); + + return javaRuleIds; + } + + private static void forAllPagesAt(String apiPath, Consumer pageHandlerCallback) throws IOException { + int pages; + int page = 1; + + do { + SonarQubeResult result = objectMapper.readValue( + apiCall(apiPath + pagingSuffix(page, apiPath)), + SonarQubeResult.class + ); + + pages = (result.paging.resultCount / PAGE_SIZE) + 1; + + pageHandlerCallback.accept(result); + + page++; + } while ((page - 1) < pages); + } + + private static String pagingSuffix(int page, String apiPath) { + return (apiPath.contains("?") ? "&" : "?") + "p=" + page + "&ps=" + PAGE_SIZE; + } + + private static String apiCall(String apiPath) throws IOException { + URL url = new URL("http://" + SONAR_HOST + ":" + SONAR_PORT + "/api/" + apiPath); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", "Basic " + sonarAuth); + + return join("\n", readLines(connection.getInputStream(), defaultCharset())); + } + + private static String formattedJson(List issues, List hotspots) throws JsonProcessingException { + String sb = "{\"issues\":[" + join(",", issues) + "],\"hotspots\":[" + join(",", hotspots) + "]}"; + + return objectMapper + .writerWithDefaultPrettyPrinter() + .writeValueAsString(objectMapper.readValue(sb, Object.class)); + } +} diff --git a/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java new file mode 100644 index 0000000000..e97dce7f5b --- /dev/null +++ b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java @@ -0,0 +1,30 @@ +package org.owasp.benchmark.report.sonarqube.dto; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.ObjectCodec; +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Credits to Roy Truelove + */ +public class KeepAsJsonDeserializer extends JsonDeserializer> { + + @Override + public List deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { + ObjectCodec codec = jp.getCodec(); + TreeNode entries = codec.readTree(jp); + List result = new ArrayList<>(); + + for (int i = 0; i < entries.size(); i++) { + result.add(codec.readTree(codec.treeAsTokens(entries.get(i))).toString()); + } + + return result; + } +} diff --git a/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java new file mode 100644 index 0000000000..be798743db --- /dev/null +++ b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java @@ -0,0 +1,36 @@ +package org.owasp.benchmark.report.sonarqube.dto; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class SonarQubeResult { + + public Paging paging; + + public List rules; + + @JsonDeserialize(using = KeepAsJsonDeserializer.class) + public List issues; + + @JsonDeserialize(using = KeepAsJsonDeserializer.class) + public List hotspots; + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Paging { + + @JsonAlias("total") + public int resultCount; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Rule { + + @JsonAlias("key") + public String ruleId; + + } +} From 283e0e61f05b9578e2fcda4a2ec8bd38b16a35f8 Mon Sep 17 00:00:00 2001 From: Sascha Knoop Date: Sun, 2 Mar 2025 23:11:28 +0100 Subject: [PATCH 2/2] spotless --- .../report/sonarqube/SonarReport.java | 87 ++++++++++--------- .../sonarqube/dto/KeepAsJsonDeserializer.java | 5 +- .../report/sonarqube/dto/SonarQubeResult.java | 2 - 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java b/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java index 21c8f47319..5498447d0f 100644 --- a/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java +++ b/src/main/java/org/owasp/benchmark/report/sonarqube/SonarReport.java @@ -1,10 +1,12 @@ package org.owasp.benchmark.report.sonarqube; +import static java.lang.String.join; +import static java.nio.charset.Charset.defaultCharset; +import static org.apache.commons.io.FileUtils.writeStringToFile; +import static org.apache.commons.io.IOUtils.readLines; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.owasp.benchmark.report.sonarqube.dto.SonarQubeResult; - -import javax.xml.parsers.DocumentBuilderFactory; import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; @@ -15,11 +17,8 @@ import java.util.List; import java.util.Set; import java.util.function.Consumer; - -import static java.lang.String.join; -import static java.nio.charset.Charset.defaultCharset; -import static org.apache.commons.io.FileUtils.writeStringToFile; -import static org.apache.commons.io.IOUtils.readLines; +import javax.xml.parsers.DocumentBuilderFactory; +import org.owasp.benchmark.report.sonarqube.dto.SonarQubeResult; public class SonarReport { private static final String SONAR_USER = "admin"; @@ -30,9 +29,8 @@ public class SonarReport { private static final int PAGE_SIZE = 500; - private static final String sonarAuth = Base64.getEncoder() - .encodeToString((SONAR_USER + ":" + SONAR_PASSWORD) - .getBytes()); + private static final String sonarAuth = + Base64.getEncoder().encodeToString((SONAR_USER + ":" + SONAR_PASSWORD).getBytes()); private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -42,19 +40,19 @@ public static void main(String[] args) throws Exception { List hotspots = new ArrayList<>(); forAllPagesAt( - "issues/search?componentKeys=" + SONAR_PROJECT + "&types=VULNERABILITY&&rules=" + allJavaRules, - (result -> issues.addAll(result.issues)) - ); + "issues/search?componentKeys=" + + SONAR_PROJECT + + "&types=VULNERABILITY&&rules=" + + allJavaRules, + (result -> issues.addAll(result.issues))); forAllPagesAt( - "hotspots/search?projectKey=" + SONAR_PROJECT, - (result -> hotspots.addAll(result.hotspots)) - ); + "hotspots/search?projectKey=" + SONAR_PROJECT, + (result -> hotspots.addAll(result.hotspots))); writeStringToFile( - new File("results/" + resultFilename() + ".json"), - formattedJson(issues, hotspots), - defaultCharset() - ); + new File("results/" + resultFilename() + ".json"), + formattedJson(issues, hotspots), + defaultCharset()); } private static String resultFilename() throws Exception { @@ -62,35 +60,36 @@ private static String resultFilename() throws Exception { } private static String benchmarkVersion() throws Exception { - return DocumentBuilderFactory - .newInstance() - .newDocumentBuilder() - .parse(new File("pom.xml")) - .getElementsByTagName("version") - .item(0) - .getTextContent(); + return DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new File("pom.xml")) + .getElementsByTagName("version") + .item(0) + .getTextContent(); } private static Set allJavaRules() throws IOException { Set javaRuleIds = new HashSet<>(); - forAllPagesAt("rules/search", (result) -> result - .rules - .stream().filter(rule -> rule.ruleId.startsWith("java:")) - .forEach(rule -> javaRuleIds.add(rule.ruleId))); + forAllPagesAt( + "rules/search", + (result) -> + result.rules.stream() + .filter(rule -> rule.ruleId.startsWith("java:")) + .forEach(rule -> javaRuleIds.add(rule.ruleId))); return javaRuleIds; } - private static void forAllPagesAt(String apiPath, Consumer pageHandlerCallback) throws IOException { + private static void forAllPagesAt(String apiPath, Consumer pageHandlerCallback) + throws IOException { int pages; int page = 1; do { - SonarQubeResult result = objectMapper.readValue( - apiCall(apiPath + pagingSuffix(page, apiPath)), - SonarQubeResult.class - ); + SonarQubeResult result = + objectMapper.readValue( + apiCall(apiPath + pagingSuffix(page, apiPath)), SonarQubeResult.class); pages = (result.paging.resultCount / PAGE_SIZE) + 1; @@ -114,11 +113,17 @@ private static String apiCall(String apiPath) throws IOException { return join("\n", readLines(connection.getInputStream(), defaultCharset())); } - private static String formattedJson(List issues, List hotspots) throws JsonProcessingException { - String sb = "{\"issues\":[" + join(",", issues) + "],\"hotspots\":[" + join(",", hotspots) + "]}"; + private static String formattedJson(List issues, List hotspots) + throws JsonProcessingException { + String sb = + "{\"issues\":[" + + join(",", issues) + + "],\"hotspots\":[" + + join(",", hotspots) + + "]}"; return objectMapper - .writerWithDefaultPrettyPrinter() - .writeValueAsString(objectMapper.readValue(sb, Object.class)); + .writerWithDefaultPrettyPrinter() + .writeValueAsString(objectMapper.readValue(sb, Object.class)); } } diff --git a/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java index e97dce7f5b..67678e9986 100644 --- a/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java +++ b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/KeepAsJsonDeserializer.java @@ -5,14 +5,11 @@ import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; - import java.io.IOException; import java.util.ArrayList; import java.util.List; -/** - * Credits to Roy Truelove - */ +/** Credits to Roy Truelove */ public class KeepAsJsonDeserializer extends JsonDeserializer> { @Override diff --git a/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java index be798743db..39806b0fda 100644 --- a/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java +++ b/src/main/java/org/owasp/benchmark/report/sonarqube/dto/SonarQubeResult.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; - import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) @@ -31,6 +30,5 @@ public static class Rule { @JsonAlias("key") public String ruleId; - } }