Nothing cols fixes

core/api/core.api

@@ -5940,6 +5940,12 @@ public final class org/jetbrains/kotlinx/dataframe/impl/api/UnfoldKt {
 	public static final fun unfoldImpl (Lorg/jetbrains/kotlinx/dataframe/DataColumn;Lkotlin/reflect/KType;Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataColumn;
 }
 
+public final class org/jetbrains/kotlinx/dataframe/impl/api/UpdateException : java/lang/IllegalStateException, org/jetbrains/kotlinx/dataframe/exceptions/DataFrameError {
+	public fun <init> (Ljava/lang/String;Ljava/lang/Throwable;)V
+	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun getMessage ()Ljava/lang/String;
+}
+
 public final class org/jetbrains/kotlinx/dataframe/impl/api/UpdateKt {
 	public static final fun updateImpl (Lorg/jetbrains/kotlinx/dataframe/api/Update;Lkotlin/jvm/functions/Function3;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 }

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/DataColumn.kt

@@ -24,6 +24,7 @@ import org.jetbrains.kotlinx.dataframe.impl.columns.addPath
 import org.jetbrains.kotlinx.dataframe.impl.columns.createColumnGuessingType
 import org.jetbrains.kotlinx.dataframe.impl.columns.toColumnKind
 import org.jetbrains.kotlinx.dataframe.impl.getValuesType
+import org.jetbrains.kotlinx.dataframe.impl.nothingType
 import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
 import org.jetbrains.kotlinx.dataframe.util.CHUNKED_IMPL_IMPORT
 import org.jetbrains.kotlinx.dataframe.util.CREATE
@@ -216,8 +217,16 @@ public interface DataColumn<out T> : BaseColumn<T> {
             infer: Infer = Infer.None,
         ): DataColumn<T> = createByType(name, values, typeOf<T>(), infer)
 
-        /** Creates an empty [DataColumn] with given [name]. */
-        public fun empty(name: String = ""): AnyCol = createValueColumn(name, emptyList<Unit>(), typeOf<Unit>())
+        /**
+         * Creates an empty [DataColumn] with given [name].
+         * @see emptyOf
+         */
+        public fun empty(name: String = ""): DataColumn<Nothing> =
+            createValueColumn(name, emptyList<Unit>(), nothingType).cast()
+
+        /** Creates an empty [DataColumn] of type [T] with given [name]. */
+        public inline fun <reified T> emptyOf(name: String = ""): DataColumn<T> =
+            createValueColumn(name, emptyList<T>(), typeOf<T>()).cast()
 
         // region deprecated
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/all.kt

@@ -30,6 +30,7 @@ import org.jetbrains.kotlinx.dataframe.impl.columns.TransformableColumnSet
 import org.jetbrains.kotlinx.dataframe.impl.columns.addPath
 import org.jetbrains.kotlinx.dataframe.impl.columns.onResolve
 import org.jetbrains.kotlinx.dataframe.impl.columns.transform
+import org.jetbrains.kotlinx.dataframe.impl.nullableNothingType
 import org.jetbrains.kotlinx.dataframe.impl.owner
 import org.jetbrains.kotlinx.dataframe.util.DEPRECATED_ACCESS_API
 import kotlin.reflect.KProperty
@@ -40,7 +41,10 @@ import kotlin.reflect.KProperty
 public fun <T> DataColumn<T>.all(predicate: Predicate<T>): Boolean = values.all(predicate)
 
 /** Returns `true` if all [values] are `null` or [values] is empty. */
-public fun <C> DataColumn<C>.allNulls(): Boolean = size == 0 || all { it == null }
+public fun <C> DataColumn<C>.allNulls(): Boolean =
+    size == 0 ||
+        type() == nullableNothingType ||
+        all { it == null }
 
 // endregion
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/colsOf.kt

@@ -96,6 +96,11 @@ public interface ColsOfColumnsSelectionDsl {
      *
      * This function operates solely on columns at the top-level.
      *
+     * __NOTE:__ Null-filled columns of type [Nothing?][Nothing] will be included when selecting [`colsOf`][colsOf]`<T?>()`.
+     *   This is because [Nothing][Nothing] is considered a subtype of all other types in Kotlin.
+     *   To exclude these columns, call `.`[filter][ColumnsSelectionDsl.filter]` { !it.`[allNulls][DataColumn.allNulls]`() }`
+     *   after it.
+     *
      * ### Check out: [Grammar]
      *
      * #### For example:

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/ColumnDataCollector.kt

@@ -3,16 +3,18 @@ package org.jetbrains.kotlinx.dataframe.impl
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.AnyRow
 import org.jetbrains.kotlinx.dataframe.DataColumn
-import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.DataRow
+import org.jetbrains.kotlinx.dataframe.api.asDataColumn
+import org.jetbrains.kotlinx.dataframe.api.cast
 import org.jetbrains.kotlinx.dataframe.api.concat
 import org.jetbrains.kotlinx.dataframe.api.toDataFrame
 import org.jetbrains.kotlinx.dataframe.impl.columns.createColumnGuessingType
 import kotlin.reflect.KClass
 import kotlin.reflect.KType
 import kotlin.reflect.full.isSubclassOf
+import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.full.withNullability
 import kotlin.reflect.jvm.jvmErasure
+import kotlin.reflect.typeOf
 
 public interface DataCollector<T> {
 
@@ -38,17 +40,29 @@ internal abstract class DataCollectorBase<T>(initCapacity: Int) : DataCollector<
         data.add(value)
     }
 
-    protected fun createColumn(name: String, type: KType): DataColumn<T> {
-        val classifier = type.classifier as KClass<*>
-        if (classifier.isSubclassOf(DataFrame::class) && !hasNulls) {
-            return DataColumn.createFrameColumn(name, data as List<AnyFrame>) as DataColumn<T>
-        }
-        if (classifier.isSubclassOf(DataRow::class) && !hasNulls) {
-            val mergedDf = (data as List<AnyRow>).map { it.toDataFrame() }.concat()
-            return DataColumn.createColumnGroup(name, mergedDf) as DataColumn<T>
-        }
-        return DataColumn.createValueColumn(name, data, type.withNullability(hasNulls)) as DataColumn<T>
-    }
+    @Suppress("UNCHECKED_CAST")
+    protected fun createColumn(name: String, type: KType): DataColumn<T> =
+        when {
+            type == nothingType -> {
+                require(values.isEmpty()) { "Cannot create non-empty DataColumn of type Nothing" }
+                DataColumn.empty(name)
+            }
+
+            type == nullableNothingType -> {
+                require(values.all { it == null }) { "Cannot create DataColumn of type Nothing? with non-null values" }
+                DataColumn.createValueColumn(name, values, nullableNothingType)
+            }
+
+            type.isSubtypeOf(typeOf<AnyFrame?>()) && !hasNulls ->
+                DataColumn.createFrameColumn(name, data as List<AnyFrame>)
+
+            type.isSubtypeOf(typeOf<AnyRow?>()) && !hasNulls -> {
+                val mergedDf = (data as List<AnyRow>).map { it.toDataFrame() }.concat()
+                DataColumn.createColumnGroup(name, mergedDf).asDataColumn()
+            }
+
+            else -> DataColumn.createValueColumn(name, data, type.withNullability(hasNulls))
+        }.cast()
 }
 
 internal open class ColumnDataCollector(initCapacity: Int = 0, val typeOf: (KClass<*>) -> KType) :
@@ -65,7 +79,7 @@ internal class TypedColumnDataCollector<T>(initCapacity: Int = 0, val type: KTyp
     override fun add(value: T?) {
         if (checkTypes && value != null && !value.javaClass.kotlin.isSubclassOf(kclass)) {
             throw IllegalArgumentException(
-                "Can not add value of class ${value.javaClass.kotlin.qualifiedName} to column of type $type. Value = $value",
+                "Cannot add a value of class ${value.javaClass.kotlin.qualifiedName} to a column of type $type. Value: '$value'.",
             )
         }
         super.add(value)

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt

@@ -24,7 +24,7 @@ internal fun <T> concatImpl(name: String, columns: List<DataColumn<T>>): DataCol
 internal fun <T> concatImpl(name: String, columns: List<DataColumn<T>?>, columnSizes: List<Int>): DataColumn<T> {
     when (columns.size) {
         0 -> return DataColumn.empty(name).cast()
-        1 -> return columns[0] ?: DataColumn.empty(name).cast()
+        1 -> return columns.single() ?: DataColumn.empty(name).cast()
     }
 
     if (columns.all { it == null || it.isColumnGroup() }) {

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/update.kt

@@ -23,6 +23,7 @@ import org.jetbrains.kotlinx.dataframe.api.with
 import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
 import org.jetbrains.kotlinx.dataframe.columns.FrameColumn
 import org.jetbrains.kotlinx.dataframe.columns.size
+import org.jetbrains.kotlinx.dataframe.exceptions.DataFrameError
 import org.jetbrains.kotlinx.dataframe.impl.columns.AddDataRowImpl
 import org.jetbrains.kotlinx.dataframe.impl.createDataCollector
 import org.jetbrains.kotlinx.dataframe.index
@@ -94,28 +95,36 @@ private fun <C, R> ColumnGroup<C>.replaceRowsIf(
         .asColumnGroup()
         .cast()
 
+public class UpdateException(override val message: String, cause: Throwable? = null) :
+    IllegalStateException(message, cause),
+    DataFrameError
+
 internal fun <T, C> DataColumn<C>.updateImpl(
     df: DataFrame<T>,
     filter: RowValueFilter<T, C>?,
     expression: (AddDataRow<T>, DataColumn<C>, C) -> C?,
 ): DataColumn<C> {
     val collector = createDataCollector<C>(size, type)
     val src = this
-    if (filter == null) {
-        df.indices().forEach { rowIndex ->
-            val row = AddDataRowImpl(rowIndex, df, collector.values)
-            collector.add(expression(row, src, src[rowIndex]))
-        }
-    } else {
-        df.indices().forEach { rowIndex ->
-            val row = AddDataRowImpl(rowIndex, df, collector.values)
-            val currentValue = row[src]
-            val newValue =
-                if (filter.invoke(row, currentValue)) expression(row, src, currentValue) else currentValue
-            collector.add(newValue)
+    try {
+        if (filter == null) {
+            df.indices().forEach { rowIndex ->
+                val row = AddDataRowImpl(rowIndex, df, collector.values)
+                collector.add(expression(row, src, src[rowIndex]))
+            }
+        } else {
+            df.indices().forEach { rowIndex ->
+                val row = AddDataRowImpl(rowIndex, df, collector.values)
+                val currentValue = row[src]
+                val newValue =
+                    if (filter.invoke(row, currentValue)) expression(row, src, currentValue) else currentValue
+                collector.add(newValue)
+            }
         }
+        return collector.toColumn(src.name).cast()
+    } catch (e: Throwable) {
+        throw UpdateException("Could not update column '${src.name}': ${e.message}", e)
     }
-    return collector.toColumn(src.name).cast()
 }
 
 /**

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/columns/constructors.kt

@@ -7,7 +7,6 @@ import org.jetbrains.kotlinx.dataframe.ColumnsContainer
 import org.jetbrains.kotlinx.dataframe.ColumnsSelector
 import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.DataRow
 import org.jetbrains.kotlinx.dataframe.Selector
 import org.jetbrains.kotlinx.dataframe.api.AddDataRow
 import org.jetbrains.kotlinx.dataframe.api.AddExpression
@@ -43,9 +42,9 @@ import org.jetbrains.kotlinx.dataframe.index
 import org.jetbrains.kotlinx.dataframe.nrow
 import org.jetbrains.kotlinx.dataframe.util.CREATE_COLUMN
 import org.jetbrains.kotlinx.dataframe.util.GUESS_COLUMN_TYPE
-import kotlin.reflect.KClass
 import kotlin.reflect.KType
 import kotlin.reflect.full.isSubtypeOf
+import kotlin.reflect.full.starProjectedType
 import kotlin.reflect.full.withNullability
 import kotlin.reflect.typeOf
 
@@ -262,10 +261,10 @@ internal fun <T> createColumnGuessingType(
         return { value -> if (value != null && value is Number) converter(value) else value }
     }
 
-    return when (type.classifier!! as KClass<*>) {
+    return when (type.classifier?.starProjectedType) {
         // guessValueType can only return DataRow if all values are `AnyRow?`
         // or allColsMakesColGroup == true, and all values are `AnyCol`
-        DataRow::class ->
+        typeOf<AnyRow>() ->
             if (allColsMakesColGroup && values.firstOrNull() is AnyCol) {
                 val df = dataFrameOf(values as Iterable<AnyCol>)
                 DataColumn.createColumnGroup(name, df)
@@ -276,7 +275,7 @@ internal fun <T> createColumnGuessingType(
                 DataColumn.createColumnGroup(name, df)
             }.asDataColumn().cast()
 
-        DataFrame::class -> {
+        typeOf<AnyFrame>() -> {
             val frames = values.map {
                 when (it) {
                     null -> DataFrame.empty()
@@ -289,7 +288,7 @@ internal fun <T> createColumnGuessingType(
             DataColumn.createFrameColumn(name, frames).asDataColumn().cast()
         }
 
-        List::class -> {
+        typeOf<List<*>>() -> {
             val nullable = type.isMarkedNullable
             var isListOfRows: Boolean? = null
             val subType = type.arguments.first().type!! // List<T> -> T

core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/concat.kt

@@ -1,6 +1,8 @@
 package org.jetbrains.kotlinx.dataframe.api
 
 import io.kotest.matchers.shouldBe
+import org.jetbrains.kotlinx.dataframe.DataColumn
+import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.junit.Test
 
 class ConcatTests {
@@ -9,7 +11,7 @@ class ConcatTests {
     fun `different types`() {
         val a by columnOf(1, 2)
         val b by columnOf(3.0, null)
-        a.concat(b) shouldBe columnOf(1, 2, 3.0, null).named("a")
+        a.concat(b) shouldBe columnOf<Number?>(1, 2, 3.0, null).named("a")
     }
 
     @Test
@@ -23,4 +25,28 @@ class ConcatTests {
 
         dfWithCategory.columnNames() shouldBe listOf("value", "type", "category")
     }
+
+    @Test
+    fun `concat empty DataFrames no rows`() {
+        val dfWithSchema = DataFrame.emptyOf<Pair<Int, String>>()
+        (dfWithSchema concat dfWithSchema).let { concatenated ->
+            concatenated shouldBe dfWithSchema
+            concatenated.schema() shouldBe dfWithSchema.schema()
+        }
+
+        val dfNothingCols = dataFrameOf(
+            "a" to DataColumn.empty(),
+            "b" to DataColumn.empty(),
+        )
+        (dfNothingCols concat dfNothingCols).let { concatenated ->
+            concatenated shouldBe dfNothingCols
+            concatenated.schema() shouldBe dfNothingCols.schema()
+        }
+    }
+
+    @Test
+    fun `concat empty DataFrames no cols`() {
+        val dfNoCols = DataFrame.empty(5)
+        (dfNoCols concat dfNoCols) shouldBe DataFrame.empty(10)
+    }
 }

core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/update.kt

@@ -1,8 +1,13 @@
 package org.jetbrains.kotlinx.dataframe.api
 
+import io.kotest.assertions.throwables.shouldThrow
 import io.kotest.matchers.shouldBe
+import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
+import org.jetbrains.kotlinx.dataframe.impl.api.UpdateException
+import org.jetbrains.kotlinx.dataframe.impl.nothingType
+import org.jetbrains.kotlinx.dataframe.impl.nullableNothingType
 import org.jetbrains.kotlinx.dataframe.size
 import org.junit.Test
 
@@ -79,4 +84,25 @@ class UpdateTests {
 
         df.fillNA(SchemaB::i).with { 42 }
     }
+
+    @Test
+    fun `update Nothing columns`() {
+        val emptyDf = dataFrameOf("a" to DataColumn.empty())
+        emptyDf["a"].type() shouldBe nothingType
+
+        emptyDf.update { "a"<Nothing>() }.with { error("should not happen") }
+            .schema() shouldBe emptyDf.schema()
+
+        val nullFilledDf = dataFrameOf("a" to columnOf(null))
+        nullFilledDf["a"].type() shouldBe nullableNothingType
+
+        // can only update with null
+        nullFilledDf.update { "a"<Nothing?>() }.with { null }
+            .schema() shouldBe nullFilledDf.schema()
+
+        // or 'Nothing', aka, return early/throw exception
+        shouldThrow<UpdateException> {
+            nullFilledDf.update { "a"<Nothing?>() }.with { error("Nothing") }
+        }.cause!!.message shouldBe "Nothing"
+    }
 }

core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/testSets/person/DataFrameTests.kt

@@ -168,6 +168,7 @@ import org.jetbrains.kotlinx.dataframe.get
 import org.jetbrains.kotlinx.dataframe.hasNulls
 import org.jetbrains.kotlinx.dataframe.impl.DataFrameImpl
 import org.jetbrains.kotlinx.dataframe.impl.DataFrameSize
+import org.jetbrains.kotlinx.dataframe.impl.api.UpdateException
 import org.jetbrains.kotlinx.dataframe.impl.api.convertToImpl
 import org.jetbrains.kotlinx.dataframe.impl.between
 import org.jetbrains.kotlinx.dataframe.impl.columns.isMissingColumn
@@ -1712,7 +1713,7 @@ class DataFrameTests : BaseTest() {
         df.update("name").at(0).with { "ALICE" }
     }
 
-    @Test(expected = IllegalArgumentException::class)
+    @Test(expected = UpdateException::class)
     fun `update with wrong type`() {
         typed.update("age").with { "string" }
     }

dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterHtmlRenderer.kt

@@ -3,6 +3,7 @@ package org.jetbrains.kotlinx.dataframe.jupyter
 import kotlinx.serialization.ExperimentalSerializationApi
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.api.FormattedFrame
+import org.jetbrains.kotlinx.dataframe.api.allNulls
 import org.jetbrains.kotlinx.dataframe.api.colsOf
 import org.jetbrains.kotlinx.dataframe.api.getColumns
 import org.jetbrains.kotlinx.dataframe.io.Base64ImageEncodingOptions
@@ -101,7 +102,7 @@ internal inline fun <reified T : Any> JupyterHtmlRenderer.render(
 }
 
 internal fun AnyFrame.hasFormattedColumns() =
-    this.getColumns { colsAtAnyDepth().colsOf<FormattedFrame<*>?>() }.isNotEmpty()
+    this.getColumns { colsAtAnyDepth().colsOf<FormattedFrame<*>?> { !it.allNulls() } }.isNotEmpty()
 
 private fun KotlinNotebookPluginUtils.IdeBuildNumber?.supportsDynamicNestedTables() =
     this != null && majorVersion >= MIN_IDE_VERSION_SUPPORT_JSON_WITH_METADATA