diff --git a/build.gradle.kts b/build.gradle.kts index f28130f..3b45eae 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,8 +15,8 @@ ignitionModule { fileName.set("Ignition-Extensions.modl") id.set("org.imdc.extensions.IgnitionExtensions") moduleVersion.set("${project.version}") - moduleDescription.set("Useful but niche extensions to Ignition for power users") + license.set("LICENSE.md") requiredIgnitionVersion.set(libs.versions.ignition.get()) projectScopes.putAll( diff --git a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt index 3ba17ac..c8962ed 100644 --- a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt +++ b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt @@ -6,9 +6,19 @@ import com.inductiveautomation.ignition.common.script.PyArgParser import com.inductiveautomation.ignition.common.script.builtin.KeywordArgs import com.inductiveautomation.ignition.common.script.hints.ScriptFunction import com.inductiveautomation.ignition.common.util.DatasetBuilder +import org.apache.poi.ss.usermodel.CellType.BOOLEAN +import org.apache.poi.ss.usermodel.CellType.FORMULA +import org.apache.poi.ss.usermodel.CellType.NUMERIC +import org.apache.poi.ss.usermodel.CellType.STRING +import org.apache.poi.ss.usermodel.DateUtil +import org.apache.poi.ss.usermodel.WorkbookFactory import org.python.core.Py import org.python.core.PyFunction import org.python.core.PyObject +import java.io.File +import java.math.BigDecimal +import java.util.Date +import kotlin.math.max object DatasetExtensions { @Suppress("unused") @@ -35,9 +45,7 @@ object DatasetExtensions { List(dataset.columnCount) { Any::class.java } } - val builder = DatasetBuilder.newBuilder() - .colNames(dataset.columnNames) - .colTypes(columnTypes) + val builder = DatasetBuilder.newBuilder().colNames(dataset.columnNames).colTypes(columnTypes) for (row in dataset.rowIndices) { val columnValues = Array(dataset.columnCount) { col -> @@ -103,41 +111,59 @@ object DatasetExtensions { @Suppress("unused") @ScriptFunction(docBundlePrefix = "DatasetExtensions") @KeywordArgs( - names = ["dataset", "output"], - types = [Dataset::class, Appendable::class], + names = ["dataset", "output", "includeTypes"], + types = [Dataset::class, Appendable::class, Boolean::class], ) fun print(args: Array, keywords: Array) { val parsedArgs = PyArgParser.parseArgs( args, keywords, - arrayOf("dataset", "output"), - arrayOf(Dataset::class.java, PyObject::class.java), + arrayOf("dataset", "output", "includeTypes"), + Array(3) { PyObject::class.java }, "print", ) val dataset = parsedArgs.requirePyObject("dataset").toJava() val appendable = parsedArgs.getPyObject("output") .orElse(Py.getSystemState().stdout) .let(::PyObjectAppendable) + val includeTypes = parsedArgs.getBoolean("includeTypes").orElse(false) - return appendable.printDataset(dataset) + return printDataset(appendable, dataset, includeTypes) } - internal fun Appendable.printDataset(dataset: Dataset, separator: String = "|") { + internal fun printDataset(appendable: Appendable, dataset: Dataset, includeTypes: Boolean = false) { + val typeNames = List(dataset.columnCount) { column -> + if (includeTypes) { + dataset.getColumnType(column).simpleName + } else { + "" + } + } + val columnWidths = IntArray(dataset.columnCount) { column -> maxOf( + // longest value in a row if (dataset.rowCount > 0) { dataset.rowIndices.maxOf { row -> dataset[row, column].toString().length } } else { 0 }, - dataset.getColumnName(column).length, + // longest value in a header + if (includeTypes) { + dataset.getColumnName(column).length + typeNames[column].length + 3 // 3 = two parens and a space + } else { + dataset.getColumnName(column).length + }, + // absolute minimum width for markdown table (and human eyeballs) 3, ) } + val separator = "|" + fun Sequence.joinToBuffer() { joinTo( - buffer = this@printDataset, + buffer = appendable, separator = " $separator ", prefix = "$separator ", postfix = " $separator\n", @@ -148,7 +174,16 @@ object DatasetExtensions { sequence { yield("Row") for (column in dataset.columnIndices) { - yield(dataset.getColumnName(column).padStart(columnWidths[column])) + val headerValue = buildString { + append(dataset.getColumnName(column)) + if (includeTypes) { + append(" (").append(typeNames[column]).append(")") + } + while (length < columnWidths[column]) { + insert(0, ' ') + } + } + yield(headerValue) } }.joinToBuffer() @@ -171,4 +206,140 @@ object DatasetExtensions { }.joinToBuffer() } } + + @Suppress("unused") + @ScriptFunction(docBundlePrefix = "DatasetExtensions") + @KeywordArgs( + names = ["input", "headerRow", "sheetNumber", "firstRow", "lastRow", "firstColumn", "lastColumn"], + types = [ByteArray::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class], + ) + fun fromExcel(args: Array, keywords: Array): Dataset { + val parsedArgs = PyArgParser.parseArgs( + args, + keywords, + arrayOf( + "input", + "headerRow", + "sheetNumber", + "firstRow", + "lastRow", + "firstColumn", + "lastColumn", + ), + Array(7) { Any::class.java }, + "fromExcel", + ) + + when (val input = parsedArgs.requirePyObject("input").toJava()) { + is String -> WorkbookFactory.create(File(input)) + is ByteArray -> WorkbookFactory.create(input.inputStream().buffered()) + else -> throw Py.TypeError("Unable to create Workbook from input; should be string or binary data. Got ${input::class.simpleName} instead.") + }.use { workbook -> + val sheetNumber = parsedArgs.getInteger("sheetNumber").orElse(0) + val sheet = workbook.getSheetAt(sheetNumber) + + val headerRow = parsedArgs.getInteger("headerRow").orElse(-1) + val firstRow = parsedArgs.getInteger("firstRow").orElseGet { max(sheet.firstRowNum, headerRow + 1) } + val lastRow = parsedArgs.getInteger("lastRow").orElseGet { sheet.lastRowNum } + + val dataRange = firstRow..lastRow + + if (firstRow >= lastRow) { + throw Py.ValueError("firstRow ($firstRow) must be less than lastRow ($lastRow)") + } + if (headerRow >= 0 && headerRow in dataRange) { + throw Py.ValueError("headerRow must not be in firstRow..lastRow ($dataRange)") + } + + val columnRow = sheet.getRow(if (headerRow >= 0) headerRow else firstRow) + val firstColumn = parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() } + val lastColumn = + parsedArgs.getInteger("lastColumn").map { it + 1 }.orElseGet { columnRow.lastCellNum.toInt() } + if (firstColumn >= lastColumn) { + throw Py.ValueError("firstColumn ($firstColumn) must be less than lastColumn ($lastColumn)") + } + + val columnCount = lastColumn - firstColumn + + val dataset = DatasetBuilder() + dataset.colNames( + List(columnCount) { + if (headerRow >= 0) { + columnRow.getCell(it + firstColumn).toString() + } else { + "Col $it" + } + }, + ) + + var typesSet = false + val columnTypes = mutableListOf>() + + for (i in dataRange) { + if (i == headerRow) { + continue + } + + val row = sheet.getRow(i) + + val rowValues = Array(columnCount) { j -> + val cell = row.getCell(j + firstColumn) + + when (cell?.cellType.takeUnless { it == FORMULA } ?: cell.cachedFormulaResultType) { + NUMERIC -> { + if (DateUtil.isCellDateFormatted(cell)) { + if (!typesSet) { + columnTypes.add(Date::class.java) + } + cell.dateCellValue + } else { + val numericCellValue = cell.numericCellValue + if (BigDecimal(numericCellValue).scale() == 0) { + if (!typesSet) { + columnTypes.add(Int::class.javaObjectType) + } + numericCellValue.toInt() + } else { + if (!typesSet) { + columnTypes.add(Double::class.javaObjectType) + } + numericCellValue + } + } + } + + STRING -> { + if (!typesSet) { + columnTypes.add(String::class.java) + } + cell.stringCellValue + } + + BOOLEAN -> { + if (!typesSet) { + columnTypes.add(Boolean::class.javaObjectType) + } + cell.booleanCellValue + } + + else -> { + if (!typesSet) { + columnTypes.add(Any::class.java) + } + null + } + } + } + + if (!typesSet) { + typesSet = true + dataset.colTypes(columnTypes) + } + + dataset.addRow(*rowValues) + } + + return dataset.build() + } + } } diff --git a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties index 555a33a..a4deb76 100644 --- a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties +++ b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties @@ -12,4 +12,15 @@ filter.returns=A modified dataset. print.desc=Prints a dataset to standard output, or the provided buffer. print.param.dataset=The dataset to print. Must not be null. print.param.output=The output destination. Defaults to sys.stdout. +print.param.includeTypes=If True, includes the type of the column in the first row. Defaults to False. print.returns=None. + +fromExcel.desc=Creates a dataset by reading select cells from an Excel spreadsheet. +fromExcel.param.input=The Excel document to read - either the path to a file on disk, or a byte array with the contents of a file. +fromExcel.param.headerRow=The row number to use for the column names in the output dataset. +fromExcel.param.sheetNumber=The sheet number (zero-indexed) in the Excel document to extract data from. +fromExcel.param.firstRow=The first row (zero-indexed) in the Excel document to retrieve data from. If not supplied, the first non-empty row will be used. +fromExcel.param.lastRow=The last row (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty row will be used. +fromExcel.param.firstColumn=The first column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the first non-empty column will be used. +fromExcel.param.lastColumn=The last column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty column will be used. +fromExcel.returns=A Dataset created from the Excel document. Types are assumed based on the first row of input data. diff --git a/common/src/main/resources/org/imdc/extensions/common/UtilitiesExtensions.properties b/common/src/main/resources/org/imdc/extensions/common/UtilitiesExtensions.properties index 4171733..c5ce55e 100644 --- a/common/src/main/resources/org/imdc/extensions/common/UtilitiesExtensions.properties +++ b/common/src/main/resources/org/imdc/extensions/common/UtilitiesExtensions.properties @@ -1,8 +1,10 @@ getContext.desc=Returns the current scope's context object directly. getContext.returns=The current scope's context. + deepCopy.desc=Deep copies the inner object structure into plain Python lists, dictionaries, and primitives. deepCopy.param.object=The object to convert. deepCopy.returns=A plain Python primitive object. + evalExpression.desc=Evaluates the supplied expression. Provide keyword arguments to populate values to curly braces. evalExpression.param.expression=The expression to evaluate. evalExpression.returns=A QualifiedValue with the result of the provided expression. diff --git a/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt b/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt index ce4235b..f52a0d0 100644 --- a/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt +++ b/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt @@ -3,9 +3,12 @@ package org.imdc.extensions.common import com.inductiveautomation.ignition.common.BasicDataset import com.inductiveautomation.ignition.common.Dataset import com.inductiveautomation.ignition.common.util.DatasetBuilder -import io.kotest.assertions.asClue +import io.kotest.assertions.withClue import io.kotest.matchers.shouldBe +import org.imdc.extensions.common.DatasetExtensions.printDataset import org.python.core.Py +import kotlin.io.path.createTempFile +import kotlin.io.path.writeBytes @Suppress("PyUnresolvedReferences", "PyInterpreter") class DatasetExtensionsTests : JythonTest( @@ -17,8 +20,38 @@ class DatasetExtensionsTests : JythonTest( .addRow(1, 3.14, "pi") .addRow(2, 6.28, "tau") .build() + + val excelSample = + DatasetExtensionsTests::class.java.getResourceAsStream("sample.xlsx")!!.readAllBytes() + val tempXlsx = createTempFile(suffix = "xlsx").also { + it.writeBytes(excelSample) + it.toFile().deleteOnExit() + } + globals["xlsxBytes"] = excelSample + globals["xlsxFile"] = tempXlsx.toString() + + val xlsSample = + DatasetExtensionsTests::class.java.getResourceAsStream("sample.xls")!!.readAllBytes() + val tempXls = createTempFile(suffix = "xls").also { + it.writeBytes(xlsSample) + it.toFile().deleteOnExit() + } + globals["xlsBytes"] = xlsSample + globals["xlsFile"] = tempXls.toString() }, ) { + private fun Dataset.asClue(assertions: (Dataset) -> Unit) { + withClue( + lazy { + buildString { + printDataset(this, this@asClue, true) + } + }, + ) { + assertions(this) + } + } + init { context("Map tests") { test("Null dataset") { @@ -93,10 +126,10 @@ class DatasetExtensionsTests : JythonTest( } context("Print tests") { - test("Basic test") { - with(DatasetExtensions) { + context("Basic dataset") { + test("Without types") { buildString { - printDataset(globals["dataset"]) + printDataset(this, globals["dataset"]) } shouldBe """ | Row | a | b | c | | --- | --- | ---- | --- | @@ -105,23 +138,155 @@ class DatasetExtensionsTests : JythonTest( """.trimIndent() } + + test("With types") { + buildString { + printDataset(this, globals["dataset"], includeTypes = true) + } shouldBe """ + | Row | a (Integer) | b (Double) | c (String) | + | --- | ----------- | ---------- | ---------- | + | 0 | 1 | 3.14 | pi | + | 1 | 2 | 6.28 | tau | + + """.trimIndent() + } } - test("Empty dataset") { - with(DatasetExtensions) { + val emptyDataset = BasicDataset( + listOf("a", "b", "c"), + listOf(String::class.java, Int::class.java, Boolean::class.java), + ) + context("Empty dataset") { + test("Without types") { buildString { - printDataset( - BasicDataset( - listOf("a", "b", "c"), - listOf(String::class.java, String::class.java, String::class.java), - ), - ) + printDataset(this, emptyDataset) } shouldBe """ | Row | a | b | c | | --- | --- | --- | --- | """.trimIndent() } + + test("With types") { + buildString { + printDataset(this, emptyDataset, includeTypes = true) + } shouldBe """ + | Row | a (String) | b (int) | c (boolean) | + | --- | ---------- | ------- | ----------- | + + """.trimIndent() + } + } + } + + context("fromExcel") { + test("XLSX file") { + eval("utils.fromExcel(xlsxFile)").asClue { + it.rowCount shouldBe 100 + it.columnCount shouldBe 16 + } + } + test("XLS file") { + eval("utils.fromExcel(xlsFile)").asClue { + it.rowCount shouldBe 100 + it.columnCount shouldBe 16 + } + } + test("XLSX bytes") { + eval("utils.fromExcel(xlsxBytes)").asClue { + it.rowCount shouldBe 100 + it.columnCount shouldBe 16 + } + } + test("XLS bytes") { + eval("utils.fromExcel(xlsBytes)").asClue { + it.rowCount shouldBe 100 + it.columnCount shouldBe 16 + } + } + test("With headers") { + eval("utils.fromExcel(xlsxBytes, headerRow=0)").asClue { + it.rowCount shouldBe 99 + it.columnCount shouldBe 16 + it.columnNames shouldBe listOf( + "Segment", + "Country", + "Product", + "Discount Band", + "Units Sold", + "Manufacturing Price", + "Sale Price", + "Gross Sales", + "Discounts", + "Sales", + "COGS", + "Profit", + "Date", + "Month Number", + "Month Name", + "Year", + ) + } + } + test("First row") { + eval("utils.fromExcel(xlsxBytes, headerRow=0, firstRow=50)").asClue { + it.rowCount shouldBe 50 + it.columnCount shouldBe 16 + } + } + test("Last row") { + eval("utils.fromExcel(xlsxBytes, headerRow=0, lastRow=50)").asClue { + it.rowCount shouldBe 50 + it.columnCount shouldBe 16 + } + } + test("First & last row") { + eval("utils.fromExcel(xlsxBytes, headerRow=0, firstRow=5, lastRow=10)").asClue { + it.rowCount shouldBe 6 + it.columnCount shouldBe 16 + } + } + test("First column") { + eval("utils.fromExcel(xlsxBytes, headerRow=0, firstColumn=10)").asClue { + it.rowCount shouldBe 99 + it.columnCount shouldBe 6 + it.columnNames shouldBe listOf( + "COGS", + "Profit", + "Date", + "Month Number", + "Month Name", + "Year", + ) + } + } + test("Last column") { + eval("utils.fromExcel(xlsxBytes, headerRow=0, lastColumn=5)").asClue { + it.rowCount shouldBe 99 + it.columnCount shouldBe 6 + it.columnNames shouldBe listOf( + "Segment", + "Country", + "Product", + "Discount Band", + "Units Sold", + "Manufacturing Price", + ) + } + } + test("First & last column") { + eval("utils.fromExcel(xlsxBytes, headerRow=0, firstColumn=5, lastColumn=10)").asClue { + it.rowCount shouldBe 99 + it.columnCount shouldBe 6 + it.columnNames shouldBe listOf( + "Manufacturing Price", + "Sale Price", + "Gross Sales", + "Discounts", + "Sales", + "COGS", + ) + } } } } diff --git a/common/src/test/resources/org/imdc/extensions/common/sample.xls b/common/src/test/resources/org/imdc/extensions/common/sample.xls new file mode 100644 index 0000000..7ec3bda Binary files /dev/null and b/common/src/test/resources/org/imdc/extensions/common/sample.xls differ diff --git a/common/src/test/resources/org/imdc/extensions/common/sample.xlsx b/common/src/test/resources/org/imdc/extensions/common/sample.xlsx new file mode 100644 index 0000000..15fd838 Binary files /dev/null and b/common/src/test/resources/org/imdc/extensions/common/sample.xlsx differ