diff --git a/core/api/core.api b/core/api/core.api index 4784802faf..e160d90daa 100644 --- a/core/api/core.api +++ b/core/api/core.api @@ -5940,6 +5940,12 @@ public final class org/jetbrains/kotlinx/dataframe/impl/api/UnfoldKt { public static final fun unfoldImpl (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Lkotlin/reflect/KType;Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataColumn; } +public final class org/jetbrains/kotlinx/dataframe/impl/api/UpdateException : java/lang/IllegalStateException, org/jetbrains/kotlinx/dataframe/exceptions/DataFrameError { + public fun (Ljava/lang/String;Ljava/lang/Throwable;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun getMessage ()Ljava/lang/String; +} + public final class org/jetbrains/kotlinx/dataframe/impl/api/UpdateKt { public static final fun updateImpl (Lorg/jetbrains/kotlinx/dataframe/api/Update;Lkotlin/jvm/functions/Function3;)Lorg/jetbrains/kotlinx/dataframe/DataFrame; } diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/DataColumn.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/DataColumn.kt index 81822170ca..2fd836b236 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/DataColumn.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/DataColumn.kt @@ -24,6 +24,7 @@ import org.jetbrains.kotlinx.dataframe.impl.columns.addPath import org.jetbrains.kotlinx.dataframe.impl.columns.createColumnGuessingType import org.jetbrains.kotlinx.dataframe.impl.columns.toColumnKind import org.jetbrains.kotlinx.dataframe.impl.getValuesType +import org.jetbrains.kotlinx.dataframe.impl.nothingType import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import org.jetbrains.kotlinx.dataframe.util.CHUNKED_IMPL_IMPORT import org.jetbrains.kotlinx.dataframe.util.CREATE @@ -216,8 +217,16 @@ public interface DataColumn : BaseColumn { infer: Infer = Infer.None, ): DataColumn = createByType(name, values, typeOf(), infer) - /** Creates an empty [DataColumn] with given [name]. */ - public fun empty(name: String = ""): AnyCol = createValueColumn(name, emptyList(), typeOf()) + /** + * Creates an empty [DataColumn] with given [name]. + * @see emptyOf + */ + public fun empty(name: String = ""): DataColumn = + createValueColumn(name, emptyList(), nothingType).cast() + + /** Creates an empty [DataColumn] of type [T] with given [name]. */ + public inline fun emptyOf(name: String = ""): DataColumn = + createValueColumn(name, emptyList(), typeOf()).cast() // region deprecated diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/all.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/all.kt index 21e9d8444e..098523548a 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/all.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/all.kt @@ -30,6 +30,7 @@ import org.jetbrains.kotlinx.dataframe.impl.columns.TransformableColumnSet import org.jetbrains.kotlinx.dataframe.impl.columns.addPath import org.jetbrains.kotlinx.dataframe.impl.columns.onResolve import org.jetbrains.kotlinx.dataframe.impl.columns.transform +import org.jetbrains.kotlinx.dataframe.impl.nullableNothingType import org.jetbrains.kotlinx.dataframe.impl.owner import org.jetbrains.kotlinx.dataframe.util.DEPRECATED_ACCESS_API import kotlin.reflect.KProperty @@ -40,7 +41,10 @@ import kotlin.reflect.KProperty public fun DataColumn.all(predicate: Predicate): Boolean = values.all(predicate) /** Returns `true` if all [values] are `null` or [values] is empty. */ -public fun DataColumn.allNulls(): Boolean = size == 0 || all { it == null } +public fun DataColumn.allNulls(): Boolean = + size == 0 || + type() == nullableNothingType || + all { it == null } // endregion diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/colsOf.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/colsOf.kt index 9eba937201..d9624ebcba 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/colsOf.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/colsOf.kt @@ -96,6 +96,11 @@ public interface ColsOfColumnsSelectionDsl { * * This function operates solely on columns at the top-level. * + * __NOTE:__ Null-filled columns of type [Nothing?][Nothing] will be included when selecting [`colsOf`][colsOf]`()`. + * This is because [Nothing][Nothing] is considered a subtype of all other types in Kotlin. + * To exclude these columns, call `.`[filter][ColumnsSelectionDsl.filter]` { !it.`[allNulls][DataColumn.allNulls]`() }` + * after it. + * * ### Check out: [Grammar] * * #### For example: diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/ColumnDataCollector.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/ColumnDataCollector.kt index ae0e60438c..36dd2bb35a 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/ColumnDataCollector.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/ColumnDataCollector.kt @@ -3,16 +3,18 @@ package org.jetbrains.kotlinx.dataframe.impl import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.AnyRow import org.jetbrains.kotlinx.dataframe.DataColumn -import org.jetbrains.kotlinx.dataframe.DataFrame -import org.jetbrains.kotlinx.dataframe.DataRow +import org.jetbrains.kotlinx.dataframe.api.asDataColumn +import org.jetbrains.kotlinx.dataframe.api.cast import org.jetbrains.kotlinx.dataframe.api.concat import org.jetbrains.kotlinx.dataframe.api.toDataFrame import org.jetbrains.kotlinx.dataframe.impl.columns.createColumnGuessingType import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.full.isSubclassOf +import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.full.withNullability import kotlin.reflect.jvm.jvmErasure +import kotlin.reflect.typeOf public interface DataCollector { @@ -38,17 +40,29 @@ internal abstract class DataCollectorBase(initCapacity: Int) : DataCollector< data.add(value) } - protected fun createColumn(name: String, type: KType): DataColumn { - val classifier = type.classifier as KClass<*> - if (classifier.isSubclassOf(DataFrame::class) && !hasNulls) { - return DataColumn.createFrameColumn(name, data as List) as DataColumn - } - if (classifier.isSubclassOf(DataRow::class) && !hasNulls) { - val mergedDf = (data as List).map { it.toDataFrame() }.concat() - return DataColumn.createColumnGroup(name, mergedDf) as DataColumn - } - return DataColumn.createValueColumn(name, data, type.withNullability(hasNulls)) as DataColumn - } + @Suppress("UNCHECKED_CAST") + protected fun createColumn(name: String, type: KType): DataColumn = + when { + type == nothingType -> { + require(values.isEmpty()) { "Cannot create non-empty DataColumn of type Nothing" } + DataColumn.empty(name) + } + + type == nullableNothingType -> { + require(values.all { it == null }) { "Cannot create DataColumn of type Nothing? with non-null values" } + DataColumn.createValueColumn(name, values, nullableNothingType) + } + + type.isSubtypeOf(typeOf()) && !hasNulls -> + DataColumn.createFrameColumn(name, data as List) + + type.isSubtypeOf(typeOf()) && !hasNulls -> { + val mergedDf = (data as List).map { it.toDataFrame() }.concat() + DataColumn.createColumnGroup(name, mergedDf).asDataColumn() + } + + else -> DataColumn.createValueColumn(name, data, type.withNullability(hasNulls)) + }.cast() } internal open class ColumnDataCollector(initCapacity: Int = 0, val typeOf: (KClass<*>) -> KType) : @@ -65,7 +79,7 @@ internal class TypedColumnDataCollector(initCapacity: Int = 0, val type: KTyp override fun add(value: T?) { if (checkTypes && value != null && !value.javaClass.kotlin.isSubclassOf(kclass)) { throw IllegalArgumentException( - "Can not add value of class ${value.javaClass.kotlin.qualifiedName} to column of type $type. Value = $value", + "Cannot add a value of class ${value.javaClass.kotlin.qualifiedName} to a column of type $type. Value: '$value'.", ) } super.add(value) diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt index 48e03407e8..1de847d616 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt @@ -24,7 +24,7 @@ internal fun concatImpl(name: String, columns: List>): DataCol internal fun concatImpl(name: String, columns: List?>, columnSizes: List): DataColumn { when (columns.size) { 0 -> return DataColumn.empty(name).cast() - 1 -> return columns[0] ?: DataColumn.empty(name).cast() + 1 -> return columns.single() ?: DataColumn.empty(name).cast() } if (columns.all { it == null || it.isColumnGroup() }) { diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/update.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/update.kt index be27e2761c..c57081f63a 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/update.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/update.kt @@ -23,6 +23,7 @@ import org.jetbrains.kotlinx.dataframe.api.with import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.columns.FrameColumn import org.jetbrains.kotlinx.dataframe.columns.size +import org.jetbrains.kotlinx.dataframe.exceptions.DataFrameError import org.jetbrains.kotlinx.dataframe.impl.columns.AddDataRowImpl import org.jetbrains.kotlinx.dataframe.impl.createDataCollector import org.jetbrains.kotlinx.dataframe.index @@ -94,6 +95,10 @@ private fun ColumnGroup.replaceRowsIf( .asColumnGroup() .cast() +public class UpdateException(override val message: String, cause: Throwable? = null) : + IllegalStateException(message, cause), + DataFrameError + internal fun DataColumn.updateImpl( df: DataFrame, filter: RowValueFilter?, @@ -101,21 +106,25 @@ internal fun DataColumn.updateImpl( ): DataColumn { val collector = createDataCollector(size, type) val src = this - if (filter == null) { - df.indices().forEach { rowIndex -> - val row = AddDataRowImpl(rowIndex, df, collector.values) - collector.add(expression(row, src, src[rowIndex])) - } - } else { - df.indices().forEach { rowIndex -> - val row = AddDataRowImpl(rowIndex, df, collector.values) - val currentValue = row[src] - val newValue = - if (filter.invoke(row, currentValue)) expression(row, src, currentValue) else currentValue - collector.add(newValue) + try { + if (filter == null) { + df.indices().forEach { rowIndex -> + val row = AddDataRowImpl(rowIndex, df, collector.values) + collector.add(expression(row, src, src[rowIndex])) + } + } else { + df.indices().forEach { rowIndex -> + val row = AddDataRowImpl(rowIndex, df, collector.values) + val currentValue = row[src] + val newValue = + if (filter.invoke(row, currentValue)) expression(row, src, currentValue) else currentValue + collector.add(newValue) + } } + return collector.toColumn(src.name).cast() + } catch (e: Throwable) { + throw UpdateException("Could not update column '${src.name}': ${e.message}", e) } - return collector.toColumn(src.name).cast() } /** diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/constructors.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/constructors.kt index 3f225e76d1..d1bc8262d6 100644 --- a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/constructors.kt +++ b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/constructors.kt @@ -7,7 +7,6 @@ import org.jetbrains.kotlinx.dataframe.ColumnsContainer import org.jetbrains.kotlinx.dataframe.ColumnsSelector import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.DataFrame -import org.jetbrains.kotlinx.dataframe.DataRow import org.jetbrains.kotlinx.dataframe.Selector import org.jetbrains.kotlinx.dataframe.api.AddDataRow import org.jetbrains.kotlinx.dataframe.api.AddExpression @@ -43,9 +42,9 @@ import org.jetbrains.kotlinx.dataframe.index import org.jetbrains.kotlinx.dataframe.nrow import org.jetbrains.kotlinx.dataframe.util.CREATE_COLUMN import org.jetbrains.kotlinx.dataframe.util.GUESS_COLUMN_TYPE -import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.full.isSubtypeOf +import kotlin.reflect.full.starProjectedType import kotlin.reflect.full.withNullability import kotlin.reflect.typeOf @@ -262,10 +261,10 @@ internal fun createColumnGuessingType( return { value -> if (value != null && value is Number) converter(value) else value } } - return when (type.classifier!! as KClass<*>) { + return when (type.classifier?.starProjectedType) { // guessValueType can only return DataRow if all values are `AnyRow?` // or allColsMakesColGroup == true, and all values are `AnyCol` - DataRow::class -> + typeOf() -> if (allColsMakesColGroup && values.firstOrNull() is AnyCol) { val df = dataFrameOf(values as Iterable) DataColumn.createColumnGroup(name, df) @@ -276,7 +275,7 @@ internal fun createColumnGuessingType( DataColumn.createColumnGroup(name, df) }.asDataColumn().cast() - DataFrame::class -> { + typeOf() -> { val frames = values.map { when (it) { null -> DataFrame.empty() @@ -289,7 +288,7 @@ internal fun createColumnGuessingType( DataColumn.createFrameColumn(name, frames).asDataColumn().cast() } - List::class -> { + typeOf>() -> { val nullable = type.isMarkedNullable var isListOfRows: Boolean? = null val subType = type.arguments.first().type!! // List -> T diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/concat.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/concat.kt index 25472bfd36..648c1d0307 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/concat.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/concat.kt @@ -1,6 +1,8 @@ package org.jetbrains.kotlinx.dataframe.api import io.kotest.matchers.shouldBe +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.DataFrame import org.junit.Test class ConcatTests { @@ -9,7 +11,7 @@ class ConcatTests { fun `different types`() { val a by columnOf(1, 2) val b by columnOf(3.0, null) - a.concat(b) shouldBe columnOf(1, 2, 3.0, null).named("a") + a.concat(b) shouldBe columnOf(1, 2, 3.0, null).named("a") } @Test @@ -23,4 +25,28 @@ class ConcatTests { dfWithCategory.columnNames() shouldBe listOf("value", "type", "category") } + + @Test + fun `concat empty DataFrames no rows`() { + val dfWithSchema = DataFrame.emptyOf>() + (dfWithSchema concat dfWithSchema).let { concatenated -> + concatenated shouldBe dfWithSchema + concatenated.schema() shouldBe dfWithSchema.schema() + } + + val dfNothingCols = dataFrameOf( + "a" to DataColumn.empty(), + "b" to DataColumn.empty(), + ) + (dfNothingCols concat dfNothingCols).let { concatenated -> + concatenated shouldBe dfNothingCols + concatenated.schema() shouldBe dfNothingCols.schema() + } + } + + @Test + fun `concat empty DataFrames no cols`() { + val dfNoCols = DataFrame.empty(5) + (dfNoCols concat dfNoCols) shouldBe DataFrame.empty(10) + } } diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/update.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/update.kt index dfc179957b..24fe831d6b 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/update.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/update.kt @@ -1,8 +1,13 @@ package org.jetbrains.kotlinx.dataframe.api +import io.kotest.assertions.throwables.shouldThrow import io.kotest.matchers.shouldBe +import org.jetbrains.kotlinx.dataframe.DataColumn import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.impl.api.UpdateException +import org.jetbrains.kotlinx.dataframe.impl.nothingType +import org.jetbrains.kotlinx.dataframe.impl.nullableNothingType import org.jetbrains.kotlinx.dataframe.size import org.junit.Test @@ -79,4 +84,25 @@ class UpdateTests { df.fillNA(SchemaB::i).with { 42 } } + + @Test + fun `update Nothing columns`() { + val emptyDf = dataFrameOf("a" to DataColumn.empty()) + emptyDf["a"].type() shouldBe nothingType + + emptyDf.update { "a"() }.with { error("should not happen") } + .schema() shouldBe emptyDf.schema() + + val nullFilledDf = dataFrameOf("a" to columnOf(null)) + nullFilledDf["a"].type() shouldBe nullableNothingType + + // can only update with null + nullFilledDf.update { "a"() }.with { null } + .schema() shouldBe nullFilledDf.schema() + + // or 'Nothing', aka, return early/throw exception + shouldThrow { + nullFilledDf.update { "a"() }.with { error("Nothing") } + }.cause!!.message shouldBe "Nothing" + } } diff --git a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/testSets/person/DataFrameTests.kt b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/testSets/person/DataFrameTests.kt index 25f157eb0b..c6ee4554d7 100644 --- a/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/testSets/person/DataFrameTests.kt +++ b/core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/testSets/person/DataFrameTests.kt @@ -168,6 +168,7 @@ import org.jetbrains.kotlinx.dataframe.get import org.jetbrains.kotlinx.dataframe.hasNulls import org.jetbrains.kotlinx.dataframe.impl.DataFrameImpl import org.jetbrains.kotlinx.dataframe.impl.DataFrameSize +import org.jetbrains.kotlinx.dataframe.impl.api.UpdateException import org.jetbrains.kotlinx.dataframe.impl.api.convertToImpl import org.jetbrains.kotlinx.dataframe.impl.between import org.jetbrains.kotlinx.dataframe.impl.columns.isMissingColumn @@ -1712,7 +1713,7 @@ class DataFrameTests : BaseTest() { df.update("name").at(0).with { "ALICE" } } - @Test(expected = IllegalArgumentException::class) + @Test(expected = UpdateException::class) fun `update with wrong type`() { typed.update("age").with { "string" } } diff --git a/dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt b/dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt index a9953c936f..951100fe2c 100644 --- a/dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt +++ b/dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt @@ -3,6 +3,7 @@ package org.jetbrains.kotlinx.dataframe.jupyter import kotlinx.serialization.ExperimentalSerializationApi import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.api.FormattedFrame +import org.jetbrains.kotlinx.dataframe.api.allNulls import org.jetbrains.kotlinx.dataframe.api.colsOf import org.jetbrains.kotlinx.dataframe.api.getColumns import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions @@ -101,7 +102,7 @@ internal inline fun JupyterHtmlRenderer.render( } internal fun AnyFrame.hasFormattedColumns() = - this.getColumns { colsAtAnyDepth().colsOf?>() }.isNotEmpty() + this.getColumns { colsAtAnyDepth().colsOf?> { !it.allNulls() } }.isNotEmpty() private fun KotlinNotebookPluginUtils.IdeBuildNumber?.supportsDynamicNestedTables() = this != null && majorVersion >= MIN_IDE_VERSION_SUPPORT_JSON_WITH_METADATA diff --git a/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/RenderingTests.kt b/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/RenderingTests.kt index 1d8f7d3913..ce198b279f 100644 --- a/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/RenderingTests.kt +++ b/dataframe-jupyter/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/RenderingTests.kt @@ -14,6 +14,11 @@ import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import org.intellij.lang.annotations.Language +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.api.columnOf +import org.jetbrains.kotlinx.dataframe.api.dataFrameOf +import org.jetbrains.kotlinx.dataframe.api.format +import org.jetbrains.kotlinx.dataframe.api.with import org.jetbrains.kotlinx.dataframe.jupyter.SerializationKeys.DATA import org.jetbrains.kotlinx.dataframe.jupyter.SerializationKeys.KOTLIN_DATAFRAME import org.jetbrains.kotlinx.dataframe.jupyter.SerializationKeys.METADATA @@ -607,6 +612,21 @@ class RenderingTests : JupyterReplTestCase() { json.extractColumn(4, "mixed") shouldBe "1" } + // Issue #1546 + @Test + fun `hasFormattedFrame false positive`() { + val df = dataFrameOf( + "a" to columnOf(1, 2, 3, null), + "b" to DataColumn.createByInference("", listOf(null, null, null, null)), + "c" to columnOf(7, 3, 2, 65), + ) + + df.hasFormattedColumns() shouldBe false + + val formatted = dataFrameOf("a" to columnOf(df.format { "c"() }.with { background(black) })) + formatted.hasFormattedColumns() shouldBe true + } + companion object { /** * Set the system property for the IDE version needed for specific serialization testing purposes.