diff --git a/create_excel.py b/create_excel.py new file mode 100644 index 00000000..6a1565f8 --- /dev/null +++ b/create_excel.py @@ -0,0 +1,12 @@ +import openpyxl + +workbook = openpyxl.Workbook() +sheet = workbook.active +sheet.title = "Sheet1" +sheet.cell(row=1, column=1, value="ID") +sheet.cell(row=1, column=2, value="Value") +sheet.cell(row=2, column=1, value="1") +sheet.cell(row=2, column=2, value="10") +sheet.cell(row=3, column=1, value="2") +sheet.cell(row=3, column=2, value="20") +workbook.save("src/test/resources/test.xlsx") diff --git a/create_excel_multi.py b/create_excel_multi.py new file mode 100644 index 00000000..5d08e665 --- /dev/null +++ b/create_excel_multi.py @@ -0,0 +1,21 @@ +import openpyxl + +workbook = openpyxl.Workbook() +sheet1 = workbook.active +sheet1.title = "Sheet1" +sheet1.cell(row=1, column=1, value="ID") +sheet1.cell(row=1, column=2, value="Value") +sheet1.cell(row=2, column=1, value="1") +sheet1.cell(row=2, column=2, value="10") +sheet1.cell(row=3, column=1, value="2") +sheet1.cell(row=3, column=2, value="20") + +sheet2 = workbook.create_sheet("Sheet2") +sheet2.cell(row=1, column=1, value="ID") +sheet2.cell(row=1, column=2, value="Value") +sheet2.cell(row=2, column=1, value="3") +sheet2.cell(row=2, column=2, value="30") +sheet2.cell(row=3, column=1, value="4") +sheet2.cell(row=3, column=2, value="40") + +workbook.save("src/test/resources/test_multi.xlsx") diff --git a/create_excel_multi_inconsistent.py b/create_excel_multi_inconsistent.py new file mode 100644 index 00000000..dcdbfcf0 --- /dev/null +++ b/create_excel_multi_inconsistent.py @@ -0,0 +1,17 @@ +import openpyxl + +workbook = openpyxl.Workbook() +sheet1 = workbook.active +sheet1.title = "Sheet1" +sheet1.cell(row=1, column=1, value="ID") +sheet1.cell(row=1, column=2, value="Value") +sheet1.cell(row=2, column=1, value="1") +sheet1.cell(row=2, column=2, value="10") + +sheet2 = workbook.create_sheet("Sheet2") +sheet2.cell(row=1, column=1, value="ID") +sheet2.cell(row=1, column=2, value="Value2") +sheet2.cell(row=2, column=1, value="3") +sheet2.cell(row=2, column=2, value="30") + +workbook.save("src/test/resources/test_multi_inconsistent.xlsx") diff --git a/pom.xml b/pom.xml index aba68c38..2b925031 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,21 @@ org.jenkins-ci.plugins jackson2-api + + org.apache.poi + poi + 5.2.3 + + + org.apache.poi + poi-ooxml + 5.2.3 + + + org.apache.xmlbeans + xmlbeans + 5.1.1 + @@ -156,6 +171,12 @@ tests test + + org.assertj + assertj-core + 3.23.1 + test + diff --git a/src/main/java/io/jenkins/plugins/reporter/provider/Csv.java b/src/main/java/io/jenkins/plugins/reporter/provider/Csv.java index 4417152d..d5ba66c0 100644 --- a/src/main/java/io/jenkins/plugins/reporter/provider/Csv.java +++ b/src/main/java/io/jenkins/plugins/reporter/provider/Csv.java @@ -7,12 +7,9 @@ import hudson.Extension; import io.jenkins.plugins.reporter.Messages; -import io.jenkins.plugins.reporter.model.Item; -import io.jenkins.plugins.reporter.model.Provider; import io.jenkins.plugins.reporter.model.ReportDto; import io.jenkins.plugins.reporter.model.ReportParser; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.jenkinsci.Symbol; import org.kohsuke.stapler.DataBoundConstructor; @@ -24,7 +21,7 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -public class Csv extends Provider { +public class Csv extends Tabular { private static final long serialVersionUID = 9141170397250309265L; @@ -48,31 +45,20 @@ public ReportParser createParser() { /** Descriptor for this provider. */ @Symbol("csv") @Extension - public static class Descriptor extends Provider.ProviderDescriptor { + public static class Descriptor extends ProviderDescriptor { /** Creates the descriptor instance. */ public Descriptor() { super(ID); } } - public static class CsvCustomParser extends ReportParser { + public static class CsvCustomParser extends Tabular.TabularParser { private static final long serialVersionUID = -8689695008930386640L; - private final String id; - - private List parserMessages; - public CsvCustomParser(String id) { - super(); - this.id = id; - this.parserMessages = new ArrayList(); - } - - public String getId() { - return id; + super(id); } - private char detectDelimiter(File file) throws IOException { // List of possible delimiters @@ -106,7 +92,6 @@ private char detectDelimiter(File file) throws IOException { return detectedDelimiter; } - @Override public ReportDto parse(File file) throws IOException { // Get delimiter @@ -125,135 +110,10 @@ public ReportDto parse(File file) throws IOException { .with(schema) .readValues(file); - ReportDto report = new ReportDto(); - report.setId(getId()); - report.setItems(new ArrayList<>()); - final List header = it.next(); final List> rows = it.readAll(); - int rowCount = 0; - final int headerColumnCount = header.size(); - int colIdxValueStart = 0; - - if (headerColumnCount >= 2) { - rowCount = rows.size(); - } else { - parserMessages.add(String.format("skipped file - First line has %d elements", headerColumnCount + 1)); - } - - /** Parse all data rows */ - for (int rowIdx = 0; rowIdx < rowCount; rowIdx++) { - String parentId = "report"; - List row = rows.get(rowIdx); - Item last = null; - boolean lastItemAdded = false; - LinkedHashMap result = new LinkedHashMap<>(); - boolean emptyFieldFound = false; - int rowSize = row.size(); - - /** Parse untill first data line is found to get data and value field */ - if (colIdxValueStart == 0) { - /** Col 0 is assumed to be string */ - for (int colIdx = rowSize - 1; colIdx > 1; colIdx--) { - String value = row.get(colIdx); - - if (NumberUtils.isCreatable(value)) { - colIdxValueStart = colIdx; - } else { - if (colIdxValueStart > 0) { - parserMessages - .add(String.format("Found data - fields number = %d - numeric fields = %d", - colIdxValueStart, rowSize - colIdxValueStart)); - } - break; - } - } - } - - String valueId = ""; - /** Parse line if first data line is OK and line has more element than header */ - if ((colIdxValueStart > 0) && (rowSize >= headerColumnCount)) { - /** Check line and header size matching */ - for (int colIdx = 0; colIdx < headerColumnCount; colIdx++) { - String id = header.get(colIdx); - String value = row.get(colIdx); - - /** Check value fields */ - if ((colIdx < colIdxValueStart)) { - /** Test if text item is a value or empty */ - if ((NumberUtils.isCreatable(value)) || (StringUtils.isBlank(value))) { - /** Empty field found - message */ - if (colIdx == 0) { - parserMessages - .add(String.format("skipped line %d - First column item empty - col = %d ", - rowIdx + 2, colIdx + 1)); - break; - } else { - emptyFieldFound = true; - /** Continue next column parsing */ - continue; - } - } else { - /** Check if field values are present after empty cells */ - if (emptyFieldFound) { - parserMessages.add(String.format("skipped line %d Empty field in col = %d ", - rowIdx + 2, colIdx + 1)); - break; - } - } - valueId += value; - Optional parent = report.findItem(parentId, report.getItems()); - Item item = new Item(); - lastItemAdded = false; - item.setId(valueId); - item.setName(value); - String finalValueId = valueId; - if (parent.isPresent()) { - Item p = parent.get(); - if (!p.hasItems()) { - p.setItems(new ArrayList<>()); - } - if (p.getItems().stream().noneMatch(i -> i.getId().equals(finalValueId))) { - p.addItem(item); - lastItemAdded = true; - } - } else { - if (report.getItems().stream().noneMatch(i -> i.getId().equals(finalValueId))) { - report.getItems().add(item); - lastItemAdded = true; - } - } - parentId = valueId; - last = item; - } else { - Number val = 0; - if (NumberUtils.isCreatable(value)) { - val = NumberUtils.createNumber(value); - } - result.put(id, val.intValue()); - } - } - } else { - /** Skip file if first data line has no value field */ - if (colIdxValueStart == 0) { - parserMessages.add(String.format("skipped line %d - First data row not found", rowIdx + 2)); - continue; - } else { - parserMessages - .add(String.format("skipped line %d - line has fewer element than title", rowIdx + 2)); - continue; - } - } - /** If last item was created, it will be added to report */ - if (lastItemAdded) { - last.setResult(result); - } else { - parserMessages.add(String.format("ignored line %d - Same fields already exists", rowIdx + 2)); - } - } - // report.setParserLog(parserMessages); - return report; + return parse(header, rows); } } } \ No newline at end of file diff --git a/src/main/java/io/jenkins/plugins/reporter/provider/Excel.java b/src/main/java/io/jenkins/plugins/reporter/provider/Excel.java new file mode 100644 index 00000000..9fc0ffe8 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/reporter/provider/Excel.java @@ -0,0 +1,109 @@ +package io.jenkins.plugins.reporter.provider; + +import hudson.Extension; +import io.jenkins.plugins.reporter.Messages; +import io.jenkins.plugins.reporter.model.ReportDto; +import io.jenkins.plugins.reporter.model.ReportParser; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundConstructor; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class Excel extends Tabular { + + private static final long serialVersionUID = 1L; + + private static final String ID = "excel"; + + @DataBoundConstructor + public Excel() { + super(); + // empty constructor required for stapler + } + + @Override + public ReportParser createParser() { + return new ExcelParser(getActualId()); + } + + /** Descriptor for this provider. */ + @Symbol("excel") + @Extension + public static class Descriptor extends ProviderDescriptor { + /** Creates the descriptor instance. */ + public Descriptor() { + super(ID); + } + } + + public static class ExcelParser extends Tabular.TabularParser { + + private static final long serialVersionUID = 1L; + + public ExcelParser(String id) { + super(id); + } + + @Override + public ReportDto parse(File file) throws IOException { + try (Workbook workbook = WorkbookFactory.create(file)) { + Sheet sheet = workbook.getSheetAt(0); + List> data = new ArrayList<>(); + for (Row row : sheet) { + List rowData = new ArrayList<>(); + for (Cell cell : row) { + switch (cell.getCellType()) { + case STRING: + rowData.add(cell.getRichStringCellValue().getString()); + break; + case NUMERIC: + if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) { + rowData.add(cell.getDateCellValue().toString()); + } else { + rowData.add(String.valueOf(cell.getNumericCellValue())); + } + break; + case BOOLEAN: + rowData.add(String.valueOf(cell.getBooleanCellValue())); + break; + case FORMULA: + rowData.add(cell.getCellFormula()); + break; + default: + rowData.add(null); + } + } + data.add(rowData); + } + + if (data.isEmpty()) { + return new ReportDto(); + } + + List header = data.get(0); + List> rows = data.subList(1, data.size()); + List> cellTypes = new ArrayList<>(); + + for (Row row : sheet) { + List rowCellTypes = new ArrayList<>(); + for (Cell cell : row) { + rowCellTypes.add(cell.getCellType().getCode()); + } + cellTypes.add(rowCellTypes); + } + + return parse(header, rows, cellTypes); + } + } + } +} diff --git a/src/main/java/io/jenkins/plugins/reporter/provider/ExcelMulti.java b/src/main/java/io/jenkins/plugins/reporter/provider/ExcelMulti.java new file mode 100644 index 00000000..5ad7a8bb --- /dev/null +++ b/src/main/java/io/jenkins/plugins/reporter/provider/ExcelMulti.java @@ -0,0 +1,114 @@ +package io.jenkins.plugins.reporter.provider; + +import hudson.Extension; +import io.jenkins.plugins.reporter.Messages; +import io.jenkins.plugins.reporter.model.ReportDto; +import io.jenkins.plugins.reporter.model.ReportParser; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundConstructor; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class ExcelMulti extends Tabular { + + private static final long serialVersionUID = 1L; + + private static final String ID = "excelMulti"; + + @DataBoundConstructor + public ExcelMulti() { + super(); + // empty constructor required for stapler + } + + @Override + public ReportParser createParser() { + return new ExcelMultiParser(getActualId()); + } + + /** Descriptor for this provider. */ + @Symbol("excelMulti") + @Extension + public static class Descriptor extends ProviderDescriptor { + /** Creates the descriptor instance. */ + public Descriptor() { + super(ID); + } + } + + public static class ExcelMultiParser extends Tabular.TabularParser { + + private static final long serialVersionUID = 1L; + + public ExcelMultiParser(String id) { + super(id); + } + + @Override + public ReportDto parse(File file) throws IOException { + try (Workbook workbook = WorkbookFactory.create(file)) { + List> allRows = new ArrayList<>(); + List header = null; + + for (Sheet sheet : workbook) { + List> sheetData = new ArrayList<>(); + for (Row row : sheet) { + List rowData = new ArrayList<>(); + for (Cell cell : row) { + switch (cell.getCellType()) { + case STRING: + rowData.add(cell.getRichStringCellValue().getString()); + break; + case NUMERIC: + if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) { + rowData.add(cell.getDateCellValue().toString()); + } else { + rowData.add(String.valueOf(cell.getNumericCellValue())); + } + break; + case BOOLEAN: + rowData.add(String.valueOf(cell.getBooleanCellValue())); + break; + case FORMULA: + rowData.add(cell.getCellFormula()); + break; + default: + rowData.add(null); + } + } + sheetData.add(rowData); + } + + if (!sheetData.isEmpty()) { + if (header == null) { + header = sheetData.get(0); + allRows.addAll(sheetData.subList(1, sheetData.size())); + } else { + List currentHeader = sheetData.get(0); + if (!header.equals(currentHeader)) { + throw new IOException("Headers are not consistent across sheets"); + } + allRows.addAll(sheetData.subList(1, sheetData.size())); + } + } + } + + if (header == null) { + return new ReportDto(); + } + + return parse(header, allRows); + } + } + } +} diff --git a/src/main/java/io/jenkins/plugins/reporter/provider/Tabular.java b/src/main/java/io/jenkins/plugins/reporter/provider/Tabular.java new file mode 100644 index 00000000..f6fcaf48 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/reporter/provider/Tabular.java @@ -0,0 +1,163 @@ +package io.jenkins.plugins.reporter.provider; + +import io.jenkins.plugins.reporter.model.Item; +import io.jenkins.plugins.reporter.model.Provider; +import io.jenkins.plugins.reporter.model.ReportDto; +import io.jenkins.plugins.reporter.model.ReportParser; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; + +public abstract class Tabular extends Provider { + + private static final long serialVersionUID = 2427895324546453L; + + public static abstract class TabularParser extends ReportParser { + + private static final long serialVersionUID = -8689695008930386640L; + + private final String id; + + private List parserMessages; + + public TabularParser(String id) { + super(); + this.id = id; + this.parserMessages = new ArrayList(); + } + + public String getId() { + return id; + } + + public ReportDto parse(List header, List> rows) { + return parse(header, rows, null); + } + + public ReportDto parse(List header, List> rows, List> cellTypes) { + ReportDto report = new ReportDto(); + report.setId(getId()); + report.setItems(new ArrayList<>()); + + int rowCount = 0; + final int headerColumnCount = header.size(); + + if (headerColumnCount >= 2) { + rowCount = rows.size(); + } else { + parserMessages.add(String.format("skipped file - First line has %d elements", headerColumnCount + 1)); + } + + /** Parse all data rows */ + for (int rowIdx = 0; rowIdx < rowCount; rowIdx++) { + String parentId = "report"; + List row = rows.get(rowIdx); + Item last = null; + boolean lastItemAdded = false; + LinkedHashMap result = new LinkedHashMap<>(); + boolean emptyFieldFound = false; + int rowSize = row.size(); + int colIdxValueStart = 0; + + /** Parse untill first data line is found to get data and value field */ + if (colIdxValueStart == 0) { + /** Col 0 is assumed to be string */ + for (int colIdx = rowSize - 1; colIdx >= 0; colIdx--) { + if (cellTypes != null && cellTypes.get(rowIdx + 1).get(colIdx) == 0) { + colIdxValueStart = colIdx; + } else if (colIdxValueStart > 0) { + break; + } + } + } + + String valueId = ""; + /** Parse line if first data line is OK and line has more element than header */ + if ((colIdxValueStart > 0) && (rowSize >= headerColumnCount)) { + /** Check line and header size matching */ + for (int colIdx = 0; colIdx < headerColumnCount; colIdx++) { + String id = header.get(colIdx); + String value = row.get(colIdx); + + /** Check value fields */ + if ((colIdx < colIdxValueStart)) { + /** Test if text item is a value or empty */ + if ((NumberUtils.isCreatable(value)) || (StringUtils.isBlank(value))) { + /** Empty field found - message */ + if (colIdx == 0) { + parserMessages + .add(String.format("skipped line %d - First column item empty - col = %d ", + rowIdx + 2, colIdx + 1)); + break; + } else { + emptyFieldFound = true; + /** Continue next column parsing */ + continue; + } + } else { + /** Check if field values are present after empty cells */ + if (emptyFieldFound) { + parserMessages.add(String.format("skipped line %d Empty field in col = %d ", + rowIdx + 2, colIdx + 1)); + break; + } + } + valueId += value; + Optional parent = report.findItem(parentId, report.getItems()); + Item item = new Item(); + lastItemAdded = false; + item.setId(valueId); + item.setName(value); + String finalValueId = valueId; + if (parent.isPresent()) { + Item p = parent.get(); + if (!p.hasItems()) { + p.setItems(new ArrayList<>()); + } + if (p.getItems().stream().noneMatch(i -> i.getId().equals(finalValueId))) { + p.addItem(item); + lastItemAdded = true; + } + } else { + if (report.getItems().stream().noneMatch(i -> i.getId().equals(finalValueId))) { + report.getItems().add(item); + lastItemAdded = true; + } + } + parentId = valueId; + last = item; + } else { + Number val = 0; + if (NumberUtils.isCreatable(value)) { + val = NumberUtils.createNumber(value); + } + result.put(id, val.intValue()); + } + } + } else { + /** Skip file if first data line has no value field */ + if (colIdxValueStart == 0) { + parserMessages.add(String.format("skipped line %d - First data row not found", rowIdx + 2)); + continue; + } else { + parserMessages + .add(String.format("skipped line %d - line has fewer element than title", rowIdx + 2)); + continue; + } + } + /** If last item was created, it will be added to report */ + if (lastItemAdded) { + last.setResult(result); + } else { + parserMessages.add(String.format("ignored line %d - Same fields already exists", rowIdx + 2)); + } + } + // report.setParserLog(parserMessages); + return report; + } + } +} diff --git a/src/test/java/io/jenkins/plugins/reporter/provider/ExcelMultiTest.java b/src/test/java/io/jenkins/plugins/reporter/provider/ExcelMultiTest.java new file mode 100644 index 00000000..2731adb1 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/reporter/provider/ExcelMultiTest.java @@ -0,0 +1,35 @@ +package io.jenkins.plugins.reporter.provider; + +import io.jenkins.plugins.reporter.model.ReportDto; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExcelMultiTest { + + @Test + public void shouldParseExcelFile() throws IOException { + ExcelMulti.ExcelMultiParser parser = new ExcelMulti.ExcelMultiParser("excelMulti"); + File file = new File("src/test/resources/test_multi.xlsx"); + ReportDto report = parser.parse(file); + assertThat(report.getItems()).hasSize(4); + assertThat(report.getItems().get(0).getName()).isEqualTo("1.0"); + assertThat(report.getItems().get(0).getResult().get("Value")).isEqualTo(10); + assertThat(report.getItems().get(1).getName()).isEqualTo("2.0"); + assertThat(report.getItems().get(1).getResult().get("Value")).isEqualTo(20); + assertThat(report.getItems().get(2).getName()).isEqualTo("3.0"); + assertThat(report.getItems().get(2).getResult().get("Value")).isEqualTo(30); + assertThat(report.getItems().get(3).getName()).isEqualTo("4.0"); + assertThat(report.getItems().get(3).getResult().get("Value")).isEqualTo(40); + } + + @Test(expected = IOException.class) + public void shouldThrowExceptionForInconsistentHeaders() throws IOException { + ExcelMulti.ExcelMultiParser parser = new ExcelMulti.ExcelMultiParser("excelMulti"); + File file = new File("src/test/resources/test_multi_inconsistent.xlsx"); + parser.parse(file); + } +} diff --git a/src/test/java/io/jenkins/plugins/reporter/provider/ExcelTest.java b/src/test/java/io/jenkins/plugins/reporter/provider/ExcelTest.java new file mode 100644 index 00000000..86b4e0d1 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/reporter/provider/ExcelTest.java @@ -0,0 +1,27 @@ +package io.jenkins.plugins.reporter.provider; + +import io.jenkins.plugins.reporter.model.ReportDto; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ExcelTest { + + @Test + public void shouldParseExcelFile() throws IOException { + Excel.ExcelParser parser = new Excel.ExcelParser("excel"); + File file = new File("src/test/resources/test.xlsx"); + ReportDto report = parser.parse(file); + + System.out.println(report.getItems()); + + assertThat(report.getItems()).hasSize(2); + assertThat(report.getItems().get(0).getName()).isEqualTo("1.0"); + assertThat(report.getItems().get(0).getResult().get("Value")).isEqualTo(10); + assertThat(report.getItems().get(1).getName()).isEqualTo("2.0"); + assertThat(report.getItems().get(1).getResult().get("Value")).isEqualTo(20); + } +} diff --git a/src/test/resources/test.xlsx b/src/test/resources/test.xlsx new file mode 100644 index 00000000..d6d6b912 Binary files /dev/null and b/src/test/resources/test.xlsx differ diff --git a/src/test/resources/test_multi.xlsx b/src/test/resources/test_multi.xlsx new file mode 100644 index 00000000..eb86b0e8 Binary files /dev/null and b/src/test/resources/test_multi.xlsx differ diff --git a/src/test/resources/test_multi_inconsistent.xlsx b/src/test/resources/test_multi_inconsistent.xlsx new file mode 100644 index 00000000..44e7c5a2 Binary files /dev/null and b/src/test/resources/test_multi_inconsistent.xlsx differ