Merge branch 'master' into convert_kdocs

# Conflicts:
#	core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

Author: AndreiKingsley

build.gradle.kts

@@ -54,6 +54,7 @@ dependencies {
     api(projects.dataframeExcel)
     api(projects.dataframeJdbc)
     api(projects.dataframeCsv)
+    api(projects.dataframeJson)
 
     // experimental, so not included by default:
     // api(projects.dataframeOpenapi)
@@ -64,6 +65,7 @@ dependencies {
     kover(projects.dataframeOpenapi)
     kover(projects.dataframeJdbc)
     kover(projects.dataframeCsv)
+    kover(projects.dataframeJson)
     kover(projects.plugins.kotlinDataframe)
     kover(projects.dataframeJupyter)
 }

core/api/core.api

@@ -66,8 +66,6 @@ dependencies {
     api(libs.commonsCsv)
 
     implementation(libs.commonsIo)
-    implementation(libs.serialization.core)
-    implementation(libs.serialization.json)
     implementation(libs.fastDoubleParser)
 
     api(libs.kotlin.datetimeJvm)
@@ -82,6 +80,9 @@ dependencies {
     testImplementation(libs.kotlin.scriptingJvm)
     testImplementation(libs.jsoup)
     testImplementation(libs.sl4jsimple)
+    testImplementation(projects.dataframeJson)
+    testImplementation(libs.serialization.core)
+    testImplementation(libs.serialization.json)
 
     // for checking results
     testImplementation(libs.commonsStatisticsDescriptive)

core/build.gradle.kts

@@ -5,7 +5,6 @@ import org.jetbrains.kotlinx.dataframe.api.KeyValueProperty
 import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
 import org.jetbrains.kotlinx.dataframe.columns.FrameColumn
 import org.jetbrains.kotlinx.dataframe.documentation.UnifyingNumbers
-import org.jetbrains.kotlinx.dataframe.io.JSON
 
 /**
  * Annotation preprocessing will generate a DataSchema interface from the data at `path`.
@@ -73,8 +72,11 @@ public annotation class JdbcOptions(
 )
 
 public annotation class JsonOptions(
-    /** Allows the choice of how to handle type clashes when reading a JSON file. */
-    public val typeClashTactic: JSON.TypeClashTactic = JSON.TypeClashTactic.ARRAY_AND_VALUE_COLUMNS,
+    /**
+     * Allows the choice of how to handle type clashes when reading a JSON file.
+     * Must be either [JsonOptions.TypeClashTactics.ARRAY_AND_VALUE_COLUMNS] or [JsonOptions.TypeClashTactics.ANY_COLUMNS]
+     * */
+    public val typeClashTactic: String = TypeClashTactics.ARRAY_AND_VALUE_COLUMNS,
     /**
      * List of [JsonPath]s where instead of a [ColumnGroup], a [FrameColumn]<[KeyValueProperty]>
      *     will be created.
@@ -85,4 +87,9 @@ public annotation class JsonOptions(
     public val keyValuePaths: Array<String> = [],
     /** Whether to [unify the numbers that are read][UnifyingNumbers]. `true` by default. */
     public val unifyNumbers: Boolean = true,
-)
+) {
+    public object TypeClashTactics {
+        public const val ARRAY_AND_VALUE_COLUMNS: String = "ARRAY_AND_VALUE_COLUMNS"
+        public const val ANY_COLUMNS: String = "ANY_COLUMNS"
+    }
+}

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

@@ -9,7 +9,7 @@ import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
 import org.jetbrains.kotlinx.dataframe.columns.ColumnSet
 import org.jetbrains.kotlinx.dataframe.columns.ColumnsResolver
 import org.jetbrains.kotlinx.dataframe.columns.SingleColumn
-import org.jetbrains.kotlinx.dataframe.impl.columns.ColumnsList
+import org.jetbrains.kotlinx.dataframe.impl.columns.ColumnListImpl
 import kotlin.reflect.KProperty
 
 // region ColumnsSelectionDsl
@@ -211,7 +211,7 @@ public interface AndColumnsSelectionDsl {
      *   and right side of the [and][org.jetbrains.kotlinx.dataframe.api.AndColumnsSelectionDsl.and] operator.
      */
     @Interpretable("And0")
-    public infix fun <C> ColumnsResolver<C>.and(other: ColumnsResolver<C>): ColumnSet<C> = ColumnsList(this, other)
+    public infix fun <C> ColumnsResolver<C>.and(other: ColumnsResolver<C>): ColumnSet<C> = ColumnListImpl(this, other)
 
     /** ## And Operator
      * The [and][org.jetbrains.kotlinx.dataframe.api.AndColumnsSelectionDsl.and] operator allows you to combine selections of columns or simply select multiple columns at once.

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/and.kt

@@ -15,10 +15,14 @@ import org.jetbrains.kotlinx.dataframe.type
  * Creates a [FrameColumn] from [this] by splitting the dataframe into
  * smaller ones, with their number of rows at most [size].
  */
-public fun <T> DataFrame<T>.chunked(size: Int, name: String = "groups"): FrameColumn<T> {
-    val startIndices = (0 until nrow step size)
-    return this.chunkedImpl(startIndices, name)
-}
+public fun <T> DataFrame<T>.chunked(size: Int, name: String = "groups"): FrameColumn<T> =
+    chunked(
+        startIndices = 0 until nrow step size,
+        name = name,
+    )
+
+public fun <T> DataFrame<T>.chunked(startIndices: Iterable<Int>, name: String = "groups"): FrameColumn<T> =
+    chunkedImpl(startIndices, name)
 
 public fun <T> DataColumn<T>.chunked(size: Int): ValueColumn<List<T>> {
     val values = toList().chunked(size)

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/chunked.kt

@@ -10,8 +10,12 @@ import org.jetbrains.kotlinx.dataframe.columns.ColumnResolutionContext
 import org.jetbrains.kotlinx.dataframe.columns.ColumnSet
 import org.jetbrains.kotlinx.dataframe.columns.ColumnWithPath
 import org.jetbrains.kotlinx.dataframe.columns.ColumnsResolver
+import org.jetbrains.kotlinx.dataframe.columns.UnresolvedColumnsPolicy
 import org.jetbrains.kotlinx.dataframe.columns.toColumnSet
+import org.jetbrains.kotlinx.dataframe.impl.DataFrameReceiver
+import org.jetbrains.kotlinx.dataframe.impl.api.extractJoinColumns
 import org.jetbrains.kotlinx.dataframe.impl.api.joinImpl
+import org.jetbrains.kotlinx.dataframe.impl.columns.ColumnListImpl
 import kotlin.reflect.KProperty
 
 @Refine
@@ -28,6 +32,8 @@ public fun <A, B> DataFrame<A>.join(
     type: JoinType = JoinType.Inner,
 ): DataFrame<A> = join(other, type) { columns.toColumnSet() }
 
+@Refine
+@Interpretable("InnerJoin")
 public fun <A, B> DataFrame<A>.innerJoin(
     other: DataFrame<B>,
     selector: JoinColumnsSelector<A, B>? = null,
@@ -36,6 +42,8 @@ public fun <A, B> DataFrame<A>.innerJoin(
 public fun <A, B> DataFrame<A>.innerJoin(other: DataFrame<B>, vararg columns: String): DataFrame<A> =
     innerJoin(other) { columns.toColumnSet() }
 
+@Refine
+@Interpretable("LeftJoin")
 public fun <A, B> DataFrame<A>.leftJoin(
     other: DataFrame<B>,
     selector: JoinColumnsSelector<A, B>? = null,
@@ -44,6 +52,8 @@ public fun <A, B> DataFrame<A>.leftJoin(
 public fun <A, B> DataFrame<A>.leftJoin(other: DataFrame<B>, vararg columns: String): DataFrame<A> =
     leftJoin(other) { columns.toColumnSet() }
 
+@Refine
+@Interpretable("RightJoin")
 public fun <A, B> DataFrame<A>.rightJoin(
     other: DataFrame<B>,
     selector: JoinColumnsSelector<A, B>? = null,
@@ -52,6 +62,8 @@ public fun <A, B> DataFrame<A>.rightJoin(
 public fun <A, B> DataFrame<A>.rightJoin(other: DataFrame<B>, vararg columns: String): DataFrame<A> =
     rightJoin(other) { columns.toColumnSet() }
 
+@Refine
+@Interpretable("FullJoin")
 public fun <A, B> DataFrame<A>.fullJoin(
     other: DataFrame<B>,
     selector: JoinColumnsSelector<A, B>? = null,
@@ -60,6 +72,8 @@ public fun <A, B> DataFrame<A>.fullJoin(
 public fun <A, B> DataFrame<A>.fullJoin(other: DataFrame<B>, vararg columns: String): DataFrame<A> =
     fullJoin(other) { columns.toColumnSet() }
 
+@Refine
+@Interpretable("FilterJoin")
 public fun <A, B> DataFrame<A>.filterJoin(
     other: DataFrame<B>,
     selector: JoinColumnsSelector<A, B>? = null,
@@ -68,6 +82,8 @@ public fun <A, B> DataFrame<A>.filterJoin(
 public fun <A, B> DataFrame<A>.filterJoin(other: DataFrame<B>, vararg columns: String): DataFrame<A> =
     filterJoin(other) { columns.toColumnSet() }
 
+@Refine
+@Interpretable("ExcludeJoin")
 public fun <A, B> DataFrame<A>.excludeJoin(
     other: DataFrame<B>,
     selector: JoinColumnsSelector<A, B>? = null,
@@ -107,14 +123,44 @@ public interface JoinDsl<out A, out B> : ColumnsSelectionDsl<A> {
     @AccessApiOverload
     public infix fun <C> KProperty<C>.match(other: ColumnReference<C>): ColumnMatch<C> =
         ColumnMatch(toColumnAccessor(), other)
+
+    public companion object {
+        public fun <A, B> defaultJoinColumns(left: DataFrame<A>, right: DataFrame<B>): JoinColumnsSelector<A, B> =
+            {
+                left.columnNames().intersect(right.columnNames().toSet())
+                    .map { it.toColumnAccessor() }
+                    .let { ColumnListImpl(it) }
+            }
+
+        public fun <A, B> getColumns(
+            left: DataFrame<A>,
+            other: DataFrame<B>,
+            selector: JoinColumnsSelector<A, B>,
+        ): List<ColumnMatch<Any?>> {
+            val receiver = object : DataFrameReceiver<A>(left, UnresolvedColumnsPolicy.Fail), JoinDsl<A, B> {
+                override val right: DataFrame<B> = DataFrameReceiver(other, UnresolvedColumnsPolicy.Fail)
+            }
+            val columns = selector(receiver, left)
+            return columns.extractJoinColumns()
+        }
+    }
 }
 
-public class ColumnMatch<C>(public val left: ColumnReference<C>, public val right: ColumnReference<C>) : ColumnSet<C> {
+public interface ColumnMatch<C> : ColumnSet<C> {
+    public val left: ColumnReference<C>
+    public val right: ColumnReference<C>
+}
+
+internal class ColumnMatchImpl<C>(override val left: ColumnReference<C>, override val right: ColumnReference<C>) :
+    ColumnMatch<C> {
 
     override fun resolve(context: ColumnResolutionContext): List<ColumnWithPath<C>> =
         throw UnsupportedOperationException()
 }
 
+public fun <C> ColumnMatch(left: ColumnReference<C>, right: ColumnReference<C>): ColumnMatch<C> =
+    ColumnMatchImpl(left, right)
+
 public typealias JoinColumnsSelector<A, B> = JoinDsl<A, B>.(ColumnsContainer<A>) -> ColumnsResolver<*>
 
 public enum class JoinType {

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/join.kt

@@ -3,6 +3,8 @@ package org.jetbrains.kotlinx.dataframe.api
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
 import org.jetbrains.kotlinx.dataframe.Selector
+import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
+import org.jetbrains.kotlinx.dataframe.annotations.Refine
 import org.jetbrains.kotlinx.dataframe.impl.api.joinWithImpl
 
 public interface JoinedDataRow<out A, out B> : DataRow<A> {
@@ -11,27 +13,41 @@ public interface JoinedDataRow<out A, out B> : DataRow<A> {
 
 public typealias JoinExpression<A, B> = Selector<JoinedDataRow<A, B>, Boolean>
 
+@Refine
+@Interpretable("JoinWith")
 public fun <A, B> DataFrame<A>.joinWith(
     right: DataFrame<B>,
     type: JoinType = JoinType.Inner,
     joinExpression: JoinExpression<A, B>,
 ): DataFrame<A> = joinWithImpl(right, type, addNewColumns = type.addNewColumns, joinExpression)
 
+@Refine
+@Interpretable("InnerJoinWith")
 public fun <A, B> DataFrame<A>.innerJoinWith(right: DataFrame<B>, joinExpression: JoinExpression<A, B>): DataFrame<A> =
     joinWith(right, JoinType.Inner, joinExpression)
 
+@Refine
+@Interpretable("LeftJoinWith")
 public fun <A, B> DataFrame<A>.leftJoinWith(right: DataFrame<B>, joinExpression: JoinExpression<A, B>): DataFrame<A> =
     joinWith(right, JoinType.Left, joinExpression)
 
+@Refine
+@Interpretable("RightJoinWith")
 public fun <A, B> DataFrame<A>.rightJoinWith(right: DataFrame<B>, joinExpression: JoinExpression<A, B>): DataFrame<A> =
     joinWith(right, JoinType.Right, joinExpression)
 
+@Refine
+@Interpretable("FullJoinWith")
 public fun <A, B> DataFrame<A>.fullJoinWith(right: DataFrame<B>, joinExpression: JoinExpression<A, B>): DataFrame<A> =
     joinWith(right, JoinType.Full, joinExpression)
 
+@Refine
+@Interpretable("FilterJoinWith")
 public fun <A, B> DataFrame<A>.filterJoinWith(right: DataFrame<B>, joinExpression: JoinExpression<A, B>): DataFrame<A> =
     joinWithImpl(right, JoinType.Inner, addNewColumns = false, joinExpression)
 
+@Refine
+@Interpretable("ExcludeJoinWith")
 public fun <A, B> DataFrame<A>.excludeJoinWith(
     right: DataFrame<B>,
     joinExpression: JoinExpression<A, B>,

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/joinWith.kt

@@ -3,7 +3,7 @@ package org.jetbrains.kotlinx.dataframe.api
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.columns.ColumnSet
 import org.jetbrains.kotlinx.dataframe.columns.ColumnsResolver
-import org.jetbrains.kotlinx.dataframe.impl.columns.ColumnsList
+import org.jetbrains.kotlinx.dataframe.impl.columns.ColumnListImpl
 
 // region ColumnsSelectionDsl
 
@@ -71,7 +71,7 @@ public interface NoneColumnsSelectionDsl {
      *
      * @return An empty [ColumnsResolver].
      */
-    public fun none(): ColumnsResolver<*> = ColumnsList<Any?>(emptyList())
+    public fun none(): ColumnsResolver<*> = ColumnListImpl<Any?>(emptyList())
 }
 
 // endregion

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/none.kt

@@ -23,7 +23,6 @@ private const val CAST = "cast"
 private const val VERIFY = "verify" // cast(true) is obscure, i think it's better to use named argument here
 private const val READ_CSV = "readCSV"
 private const val READ_TSV = "readTSV"
-private const val READ_JSON = "readJson"
 private const val READ_JDBC = "readJdbc"
 
 public abstract class AbstractDefaultReadMethod(
@@ -82,13 +81,6 @@ public abstract class AbstractDefaultReadMethod(
     override val additionalImports: List<String> = listOf("import org.jetbrains.kotlinx.dataframe.io.$methodName")
 }
 
-internal class DefaultReadJsonMethod(path: String?, arguments: MethodArguments) :
-    AbstractDefaultReadMethod(
-        path = path,
-        arguments = arguments,
-        methodName = READ_JSON,
-    )
-
 internal class DefaultReadCsvMethod(path: String?, arguments: MethodArguments) :
     AbstractDefaultReadMethod(path, arguments, READ_CSV)