Skip to content

Commit 76091c7

Browse files
JolanrensenJolanrensen
authored
Merge pull request #603 from Kotlin/devcrocod/serialization-patch0
Replace Klaxon with kotlinx-serialization
2 parents 54db451 + c0ba411 commit 76091c7

File tree

29 files changed

+1014
-626
lines changed

29 files changed

+1014
-626
lines changed

‎build.gradle.kts

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ plugins {
1515
with(libs.plugins) {
1616
alias(kotlin.jvm)
1717
alias(publisher)
18-
alias(serialization)
18+
alias(serialization) apply false
1919
alias(jupyter.api) apply false
2020
alias(dokka)
2121
alias(kover)
@@ -71,8 +71,6 @@ fun String.findVersion(): Version {
7171

7272
// these names of outdated dependencies will not show up in the table output
7373
val dependencyUpdateExclusions = listOf(
74-
// 5.6 requires Java 11
75-
libs.klaxon.get().name,
7674
// TODO Requires more work to be updated to 1.7.0+, https://github.com/Kotlin/dataframe/issues/594
7775
libs.plugins.kover.get().pluginId,
7876
// TODO Updating requires major changes all across the project, https://github.com/Kotlin/dataframe/issues/364

‎core/build.gradle.kts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,9 @@ dependencies {
6767
implementation(libs.kotlin.stdlib.jdk8)
6868

6969
api(libs.commonsCsv)
70-
implementation(libs.klaxon)
70+
implementation(libs.serialization.core)
71+
implementation(libs.serialization.json)
72+
7173
implementation(libs.fuel)
7274

7375
api(libs.kotlin.datetimeJvm)

‎core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/io/readJson.kt

Lines changed: 84 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
11
package org.jetbrains.kotlinx.dataframe.impl.io
22

3-
import com.beust.klaxon.JsonArray
4-
import com.beust.klaxon.JsonObject
3+
import kotlinx.serialization.json.JsonArray
4+
import kotlinx.serialization.json.JsonNull
5+
import kotlinx.serialization.json.JsonObject
6+
import kotlinx.serialization.json.JsonPrimitive
7+
import kotlinx.serialization.json.boolean
8+
import kotlinx.serialization.json.booleanOrNull
9+
import kotlinx.serialization.json.double
10+
import kotlinx.serialization.json.doubleOrNull
11+
import kotlinx.serialization.json.float
12+
import kotlinx.serialization.json.floatOrNull
13+
import kotlinx.serialization.json.int
14+
import kotlinx.serialization.json.intOrNull
15+
import kotlinx.serialization.json.jsonArray
16+
import kotlinx.serialization.json.long
17+
import kotlinx.serialization.json.longOrNull
518
import org.jetbrains.kotlinx.dataframe.AnyCol
619
import org.jetbrains.kotlinx.dataframe.AnyFrame
720
import org.jetbrains.kotlinx.dataframe.DataColumn
@@ -73,8 +86,8 @@ internal fun readJson(
7386
val df: AnyFrame = when (typeClashTactic) {
7487
ARRAY_AND_VALUE_COLUMNS -> {
7588
when (parsed) {
76-
is JsonArray<*> -> fromJsonListArrayAndValueColumns(
77-
records = parsed.value,
89+
is JsonArray -> fromJsonListArrayAndValueColumns(
90+
records = parsed,
7891
header = header,
7992
keyValuePaths = keyValuePaths,
8093
)
@@ -88,8 +101,8 @@ internal fun readJson(
88101

89102
ANY_COLUMNS -> {
90103
when (parsed) {
91-
is JsonArray<*> -> fromJsonListAnyColumns(
92-
records = parsed.value,
104+
is JsonArray -> fromJsonListAnyColumns(
105+
records = parsed,
93106
header = header,
94107
keyValuePaths = keyValuePaths,
95108
)
@@ -126,18 +139,16 @@ internal fun fromJsonListAnyColumns(
126139

127140
// list element type can be JsonObject, JsonArray or primitive
128141
val nameGenerator = ColumnNameGenerator()
129-
records.forEach {
130-
when (it) {
142+
records.forEach { record ->
143+
when (record) {
131144
is JsonObject -> {
132145
hasObject = true
133-
it.entries.forEach {
134-
nameGenerator.addIfAbsent(it.key)
135-
}
146+
record.entries.forEach { nameGenerator.addIfAbsent(it.key) }
136147
}
137148

138-
is JsonArray<*> -> hasArray = true
139-
null -> Unit
140-
else -> hasPrimitive = true
149+
is JsonArray -> hasArray = true
150+
is JsonNull, null -> Unit
151+
is JsonPrimitive -> hasPrimitive = true
141152
}
142153
}
143154

@@ -177,7 +188,7 @@ internal fun fromJsonListAnyColumns(
177188
)
178189
}
179190

180-
is JsonArray<*> -> {
191+
is JsonArray -> {
181192
val parsed = fromJsonListAnyColumns(
182193
records = v,
183194
keyValuePaths = keyValuePaths,
@@ -189,9 +200,22 @@ internal fun fromJsonListAnyColumns(
189200
)
190201
}
191202

192-
"NaN" -> {
193-
nanIndices.add(i)
194-
collector.add(null)
203+
is JsonNull -> collector.add(null)
204+
205+
is JsonPrimitive -> {
206+
when {
207+
v.content == "NaN" -> {
208+
nanIndices.add(i)
209+
collector.add(null)
210+
}
211+
212+
v.isString -> collector.add(v.content)
213+
v.booleanOrNull != null -> collector.add(v.boolean)
214+
v.intOrNull != null -> collector.add(v.int)
215+
v.longOrNull != null -> collector.add(v.long)
216+
v.doubleOrNull != null -> collector.add(v.double)
217+
v.floatOrNull != null -> collector.add(v.float)
218+
}
195219
}
196220

197221
else -> collector.add(v)
@@ -227,8 +251,8 @@ internal fun fromJsonListAnyColumns(
227251
records.forEach {
228252
startIndices.add(values.size)
229253
when (it) {
230-
is JsonArray<*> -> values.addAll(it.value)
231-
null -> Unit
254+
is JsonArray -> values.addAll(it)
255+
is JsonNull, null -> Unit
232256
else -> error("Expected JsonArray, got $it")
233257
}
234258
}
@@ -242,10 +266,10 @@ internal fun fromJsonListAnyColumns(
242266
parsed.isSingleUnnamedColumn() -> {
243267
val col = (parsed.getColumn(0) as UnnamedColumn).col
244268
val elementType = col.type
245-
val values = col.values.asList().splitByIndices(startIndices.asSequence()).toList()
269+
val columnValues = col.values.asList().splitByIndices(startIndices.asSequence()).toList()
246270
DataColumn.createValueColumn(
247271
name = arrayColumnName,
248-
values = values,
272+
values = columnValues,
249273
type = List::class.createType(listOf(KTypeProjection.invariant(elementType))),
250274
)
251275
}
@@ -263,10 +287,10 @@ internal fun fromJsonListAnyColumns(
263287
colType == AnyColType.OBJECTS && isKeyValue -> {
264288
// collect the value types to make sure Value columns with lists and other values aren't all turned into lists
265289
val valueTypes = mutableSetOf<KType>()
266-
val dataFrames = records.map {
267-
when (it) {
290+
val dataFrames = records.map { record ->
291+
when (record) {
268292
is JsonObject -> {
269-
val map = it.map.mapValues { (key, value) ->
293+
val map = record.mapValues { (key, value) ->
270294
val parsed = fromJsonListAnyColumns(
271295
records = listOf(value),
272296
keyValuePaths = keyValuePaths,
@@ -288,8 +312,8 @@ internal fun fromJsonListAnyColumns(
288312
)
289313
}
290314

291-
null -> DataFrame.emptyOf<AnyKeyValueProperty>()
292-
else -> error("Expected JsonObject, got $it")
315+
is JsonNull, null -> DataFrame.emptyOf<AnyKeyValueProperty>()
316+
else -> error("Expected JsonObject, got $record")
293317
}
294318
}
295319

@@ -328,7 +352,7 @@ internal fun fromJsonListAnyColumns(
328352
records.forEach {
329353
when (it) {
330354
is JsonObject -> values.add(it[colName])
331-
null -> values.add(null)
355+
is JsonNull, null -> values.add(null)
332356
else -> error("Expected JsonObject, got $it")
333357
}
334358
}
@@ -401,18 +425,18 @@ internal fun fromJsonListArrayAndValueColumns(
401425
// { "array": [], "value": 123, "a": null, "b": null }
402426

403427
val nameGenerator = ColumnNameGenerator()
404-
records.forEach {
405-
when (it) {
406-
is JsonObject -> it.entries.forEach {
428+
records.forEach { record ->
429+
when (record) {
430+
is JsonObject -> record.entries.forEach {
407431
nameGenerator.addIfAbsent(it.key)
408432
}
409433

410-
is JsonArray<*> -> hasArray = true
411-
null -> Unit
412-
else -> hasPrimitive = true
434+
is JsonArray -> hasArray = true
435+
is JsonNull, null -> Unit
436+
is JsonPrimitive -> hasPrimitive = true
413437
}
414438
}
415-
if (records.all { it == null }) hasPrimitive = true
439+
if (records.all { it == null || it is JsonNull }) hasPrimitive = true
416440

417441
// Add a value column to the collected names if needed
418442
val valueColumn = if (hasPrimitive || records.isEmpty()) {
@@ -433,10 +457,10 @@ internal fun fromJsonListArrayAndValueColumns(
433457
val columns: List<AnyCol> = when {
434458
// instead of using the names, generate a single key/value frame column
435459
isKeyValue -> {
436-
val dataFrames = records.map {
437-
when (it) {
460+
val dataFrames = records.map { record ->
461+
when (record) {
438462
is JsonObject -> {
439-
val map = it.map.mapValues { (key, value) ->
463+
val map = record.mapValues { (key, value) ->
440464
val parsed = fromJsonListArrayAndValueColumns(
441465
records = listOf(value),
442466
keyValuePaths = keyValuePaths,
@@ -459,8 +483,8 @@ internal fun fromJsonListArrayAndValueColumns(
459483
)
460484
}
461485

462-
null -> DataFrame.emptyOf<AnyKeyValueProperty>()
463-
else -> error("Expected JsonObject, got $it")
486+
is JsonNull, null -> DataFrame.emptyOf<AnyKeyValueProperty>()
487+
else -> error("Expected JsonObject, got $record")
464488
}
465489
}
466490

@@ -488,10 +512,22 @@ internal fun fromJsonListArrayAndValueColumns(
488512
records.forEachIndexed { i, v ->
489513
when (v) {
490514
is JsonObject -> collector.add(null)
491-
is JsonArray<*> -> collector.add(null)
492-
"NaN" -> {
493-
nanIndices.add(i)
494-
collector.add(null)
515+
is JsonArray -> collector.add(null)
516+
is JsonNull -> collector.add(null)
517+
is JsonPrimitive -> {
518+
when {
519+
v.content == "NaN" -> {
520+
nanIndices.add(i)
521+
collector.add(null)
522+
}
523+
524+
v.isString -> collector.add(v.content)
525+
v.booleanOrNull != null -> collector.add(v.boolean)
526+
v.intOrNull != null -> collector.add(v.int)
527+
v.longOrNull != null -> collector.add(v.long)
528+
v.doubleOrNull != null -> collector.add(v.double)
529+
v.floatOrNull != null -> collector.add(v.float)
530+
}
495531
}
496532

497533
else -> collector.add(v)
@@ -526,7 +562,7 @@ internal fun fromJsonListArrayAndValueColumns(
526562
val startIndices = ArrayList<Int>()
527563
records.forEach {
528564
startIndices.add(values.size)
529-
if (it is JsonArray<*>) values.addAll(it.value)
565+
if (it is JsonArray) values.addAll(it.jsonArray)
530566
}
531567
val parsed = fromJsonListArrayAndValueColumns(
532568
records = values,
@@ -538,10 +574,11 @@ internal fun fromJsonListArrayAndValueColumns(
538574
parsed.isSingleUnnamedColumn() -> {
539575
val col = (parsed.getColumn(0) as UnnamedColumn).col
540576
val elementType = col.type
541-
val values = col.values.asList().splitByIndices(startIndices.asSequence()).toList()
577+
val columnValues =
578+
col.values.asList().splitByIndices(startIndices.asSequence()).toList()
542579
DataColumn.createValueColumn(
543580
name = colName,
544-
values = values,
581+
values = columnValues,
545582
type = List::class.createType(listOf(KTypeProjection.invariant(elementType))),
546583
)
547584
}

0 commit comments

Comments
 (0)