Skip to content

Commit b385a62

Browse files
JolanrensenJolanrensen
committed
Merge branch 'master' into devcrocod/serialization-patch0
2 parents f3770cf + 54db451 commit b385a62

File tree

27 files changed

+1789
-76
lines changed

27 files changed

+1789
-76
lines changed

‎build.gradle.kts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask
2+
import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension
23
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
34
import org.jetbrains.kotlinx.dataframe.AnyFrame
45
import org.jetbrains.kotlinx.dataframe.DataFrame
@@ -122,6 +123,8 @@ tasks.named<DependencyUpdatesTask>("dependencyUpdates").configure {
122123
}
123124
}
124125

126+
kotlin.jvmToolchain(11)
127+
125128
allprojects {
126129
tasks.withType<KotlinCompile> {
127130
kotlinOptions {
@@ -158,6 +161,9 @@ allprojects {
158161
} catch (_: UnknownDomainObjectException) {
159162
logger.warn("Could not set kotlinter config on :${this.name}")
160163
}
164+
165+
// set the java toolchain version to 11 for all subprojects for CI stability
166+
extensions.findByType<KotlinJvmProjectExtension>()?.jvmToolchain(11)
161167
}
162168
}
163169

‎core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/GroupByImpl.kt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,11 @@ internal fun <T, G, R> aggregateGroupBy(
8383
val result = body(builder, builder)
8484
if (result != Unit && result !is NamedValue && result !is AggregatedPivot<*>) builder.yield(
8585
NamedValue.create(
86-
pathOf(defaultAggregateName), result, null, null, true
86+
path = pathOf(defaultAggregateName),
87+
value = result,
88+
type = null,
89+
defaultValue = null,
90+
guessType = true,
8791
)
8892
)
8993
builder.compute()

‎core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.jetbrains.kotlinx.dataframe.hasNulls
1212
import org.jetbrains.kotlinx.dataframe.impl.columns.guessColumnType
1313
import org.jetbrains.kotlinx.dataframe.impl.commonType
1414
import org.jetbrains.kotlinx.dataframe.impl.getListType
15+
import org.jetbrains.kotlinx.dataframe.impl.guessValueType
1516
import org.jetbrains.kotlinx.dataframe.nrow
1617
import kotlin.reflect.KType
1718
import kotlin.reflect.full.withNullability
@@ -54,7 +55,13 @@ internal fun <T> concatImpl(name: String, columns: List<DataColumn<T>?>, columnS
5455
col.toList()
5556
} else {
5657
val nrow = columnSizes[index]
57-
if (!nulls && nrow > 0 && defaultValue == null) nulls = true
58+
if (!nulls && nrow > 0 && defaultValue == null) {
59+
nulls = true
60+
} else if (defaultValue != null) {
61+
types.add(
62+
guessValueType(sequenceOf(defaultValue))
63+
)
64+
}
5865
List(nrow) { defaultValue }
5966
}
6067
}
@@ -63,7 +70,13 @@ internal fun <T> concatImpl(name: String, columns: List<DataColumn<T>?>, columnS
6370
val baseType = types.commonType()
6471
val tartypeOf = if (guessType || !hasList) baseType.withNullability(nulls)
6572
else getListType(baseType.withNullability(listOfNullable))
66-
return guessColumnType(name, list, tartypeOf, guessType, defaultValue).cast()
73+
return guessColumnType(
74+
name = name,
75+
values = list,
76+
suggestedType = tartypeOf,
77+
suggestedTypeIsUpperBound = guessType,
78+
defaultValue = defaultValue,
79+
).cast()
6780
}
6881
}
6982

‎core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/pivot.kt

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,17 +100,21 @@ internal fun <T, R> aggregatePivot(
100100
val hasResult = result != null && result != Unit
101101

102102
fun NamedValue.apply(path: ColumnPath) =
103-
copy(path = path, value = this.value ?: default ?: globalDefault, default = default ?: globalDefault)
103+
copy(
104+
path = path,
105+
value = this.value ?: default ?: globalDefault,
106+
default = default ?: globalDefault,
107+
)
104108

105109
val values = builder.values
106110
when {
107111
values.size == 1 && values[0].path.isEmpty() -> aggregator.yield(values[0].apply(path))
108112
values.isEmpty() -> aggregator.yield(
109-
path,
110-
if (hasResult) result else globalDefault,
111-
null,
112-
globalDefault,
113-
true
113+
path = path,
114+
value = if (hasResult) result else globalDefault,
115+
type = null,
116+
default = globalDefault,
117+
guessType = true,
114118
)
115119

116120
else -> {

‎core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.jetbrains.kotlinx.dataframe.api
22

33
import io.kotest.matchers.shouldBe
4+
import org.jetbrains.kotlinx.dataframe.impl.commonType
45
import org.junit.Test
56
import kotlin.reflect.typeOf
67

@@ -168,4 +169,20 @@ class PivotTests {
168169
}
169170
df1 shouldBe df.pivot { "category2" then "category3" }.groupBy("category1").count()
170171
}
172+
173+
@Test
174+
fun `pivot with default of other type`() {
175+
val df = dataFrameOf("firstName", "lastName", "age", "city", "weight", "isHappy")(
176+
"Alice", "Cooper", 15, "London", 54, true,
177+
"Bob", "Dylan", 45, "Dubai", 87, true,
178+
"Charlie", "Daniels", 20, "Moscow", null, false,
179+
"Charlie", "Chaplin", 40, "Milan", null, true,
180+
"Bob", "Marley", 30, "Tokyo", 68, true,
181+
"Alice", "Wolf", 20, null, 55, false,
182+
"Charlie", "Byrd", 30, "Moscow", 90, true
183+
).group("firstName", "lastName").into("name")
184+
185+
val pivoted = df.pivot("city").groupBy("name").default(0).min()
186+
pivoted["city"]["London"]["isHappy"].type() shouldBe listOf(typeOf<Int>(), typeOf<Boolean>()).commonType()
187+
}
171188
}

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/GroupByImpl.kt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,11 @@ internal fun <T, G, R> aggregateGroupBy(
8383
val result = body(builder, builder)
8484
if (result != Unit && result !is NamedValue && result !is AggregatedPivot<*>) builder.yield(
8585
NamedValue.create(
86-
pathOf(defaultAggregateName), result, null, null, true
86+
path = pathOf(defaultAggregateName),
87+
value = result,
88+
type = null,
89+
defaultValue = null,
90+
guessType = true,
8791
)
8892
)
8993
builder.compute()

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/concat.kt

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.jetbrains.kotlinx.dataframe.hasNulls
1212
import org.jetbrains.kotlinx.dataframe.impl.columns.guessColumnType
1313
import org.jetbrains.kotlinx.dataframe.impl.commonType
1414
import org.jetbrains.kotlinx.dataframe.impl.getListType
15+
import org.jetbrains.kotlinx.dataframe.impl.guessValueType
1516
import org.jetbrains.kotlinx.dataframe.nrow
1617
import kotlin.reflect.KType
1718
import kotlin.reflect.full.withNullability
@@ -54,7 +55,13 @@ internal fun <T> concatImpl(name: String, columns: List<DataColumn<T>?>, columnS
5455
col.toList()
5556
} else {
5657
val nrow = columnSizes[index]
57-
if (!nulls && nrow > 0 && defaultValue == null) nulls = true
58+
if (!nulls && nrow > 0 && defaultValue == null) {
59+
nulls = true
60+
} else if (defaultValue != null) {
61+
types.add(
62+
guessValueType(sequenceOf(defaultValue))
63+
)
64+
}
5865
List(nrow) { defaultValue }
5966
}
6067
}
@@ -63,7 +70,13 @@ internal fun <T> concatImpl(name: String, columns: List<DataColumn<T>?>, columnS
6370
val baseType = types.commonType()
6471
val tartypeOf = if (guessType || !hasList) baseType.withNullability(nulls)
6572
else getListType(baseType.withNullability(listOfNullable))
66-
return guessColumnType(name, list, tartypeOf, guessType, defaultValue).cast()
73+
return guessColumnType(
74+
name = name,
75+
values = list,
76+
suggestedType = tartypeOf,
77+
suggestedTypeIsUpperBound = guessType,
78+
defaultValue = defaultValue,
79+
).cast()
6780
}
6881
}
6982

‎core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/pivot.kt

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -100,17 +100,21 @@ internal fun <T, R> aggregatePivot(
100100
val hasResult = result != null && result != Unit
101101

102102
fun NamedValue.apply(path: ColumnPath) =
103-
copy(path = path, value = this.value ?: default ?: globalDefault, default = default ?: globalDefault)
103+
copy(
104+
path = path,
105+
value = this.value ?: default ?: globalDefault,
106+
default = default ?: globalDefault,
107+
)
104108

105109
val values = builder.values
106110
when {
107111
values.size == 1 && values[0].path.isEmpty() -> aggregator.yield(values[0].apply(path))
108112
values.isEmpty() -> aggregator.yield(
109-
path,
110-
if (hasResult) result else globalDefault,
111-
null,
112-
globalDefault,
113-
true
113+
path = path,
114+
value = if (hasResult) result else globalDefault,
115+
type = null,
116+
default = globalDefault,
117+
guessType = true,
114118
)
115119

116120
else -> {

‎core/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.jetbrains.kotlinx.dataframe.api
22

33
import io.kotest.matchers.shouldBe
4+
import org.jetbrains.kotlinx.dataframe.impl.commonType
45
import org.junit.Test
56
import kotlin.reflect.typeOf
67

@@ -168,4 +169,20 @@ class PivotTests {
168169
}
169170
df1 shouldBe df.pivot { "category2" then "category3" }.groupBy("category1").count()
170171
}
172+
173+
@Test
174+
fun `pivot with default of other type`() {
175+
val df = dataFrameOf("firstName", "lastName", "age", "city", "weight", "isHappy")(
176+
"Alice", "Cooper", 15, "London", 54, true,
177+
"Bob", "Dylan", 45, "Dubai", 87, true,
178+
"Charlie", "Daniels", 20, "Moscow", null, false,
179+
"Charlie", "Chaplin", 40, "Milan", null, true,
180+
"Bob", "Marley", 30, "Tokyo", 68, true,
181+
"Alice", "Wolf", 20, null, 55, false,
182+
"Charlie", "Byrd", 30, "Moscow", 90, true
183+
).group("firstName", "lastName").into("name")
184+
185+
val pivoted = df.pivot("city").groupBy("name").default(0).min()
186+
pivoted["city"]["London"]["isHappy"].type() shouldBe listOf(typeOf<Int>(), typeOf<Boolean>()).commonType()
187+
}
171188
}

‎dataframe-jdbc/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ dependencies {
2828
testImplementation(libs.mssql)
2929
testImplementation(libs.junit)
3030
testImplementation(libs.sl4j)
31+
testImplementation(libs.jts)
3132
testImplementation(libs.kotestAssertions) {
3233
exclude("org.jetbrains.kotlin", "kotlin-stdlib-jdk8")
3334
}

‎dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/H2.kt

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,33 +10,67 @@ import kotlin.reflect.KType
1010
/**
1111
* Represents the H2 database type.
1212
*
13-
* This class provides methods to convert data from a ResultSet to the appropriate type for H2,
13+
* This class provides methods to convert data from a ResultSet to the appropriate type for H2
1414
* and to generate the corresponding column schema.
1515
*
16-
* NOTE: All date and timestamp related types are converted to String to avoid java.sql.* types.
16+
* NOTE: All date and timestamp-related types are converted to String to avoid java.sql.* types.
1717
*/
18-
public object H2 : DbType("h2") {
18+
public class H2(public val dialect: DbType = MySql) : DbType("h2") {
19+
init {
20+
require(dialect::class != H2::class) { "H2 database could not be specified with H2 dialect!" }
21+
}
22+
23+
/**
24+
* It contains constants related to different database modes.
25+
*
26+
* The mode value is used in the [extractDBTypeFromConnection] function to determine the corresponding `DbType` for the H2 database connection URL.
27+
* For example, if the URL contains the mode value "MySQL", the H2 instance with the MySQL database type is returned.
28+
* Otherwise, the `DbType` is determined based on the URL without the mode value.
29+
*
30+
* @see [extractDBTypeFromConnection]
31+
* @see [createH2Instance]
32+
*/
33+
public companion object {
34+
/** It represents the mode value "MySQL" for the H2 database. */
35+
public const val MODE_MYSQL: String = "MySQL"
36+
37+
/** It represents the mode value "PostgreSQL" for the H2 database. */
38+
public const val MODE_POSTGRESQL: String = "PostgreSQL"
39+
40+
/** It represents the mode value "MSSQLServer" for the H2 database. */
41+
public const val MODE_MSSQLSERVER: String = "MSSQLServer"
42+
43+
/** It represents the mode value "MariaDB" for the H2 database. */
44+
public const val MODE_MARIADB: String = "MariaDB"
45+
}
46+
1947
override val driverClassName: String
2048
get() = "org.h2.Driver"
2149

2250
override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? {
23-
return null
51+
return dialect.convertSqlTypeToColumnSchemaValue(tableColumnMetadata)
2452
}
2553

2654
override fun isSystemTable(tableMetadata: TableMetadata): Boolean {
27-
return tableMetadata.name.lowercase(Locale.getDefault()).contains("sys_") ||
28-
tableMetadata.schemaName?.lowercase(Locale.getDefault())?.contains("information_schema") ?: false
55+
val locale = Locale.getDefault()
56+
fun String?.containsWithLowercase(substr: String) = this?.lowercase(locale)?.contains(substr) == true
57+
val schemaName = tableMetadata.schemaName
58+
59+
// could be extended for other symptoms of the system tables for H2
60+
val isH2SystemTable = schemaName.containsWithLowercase("information_schema")
61+
62+
return isH2SystemTable || dialect.isSystemTable(tableMetadata)
2963
}
3064

3165
override fun buildTableMetadata(tables: ResultSet): TableMetadata {
32-
return TableMetadata(
33-
tables.getString("TABLE_NAME"),
34-
tables.getString("TABLE_SCHEM"),
35-
tables.getString("TABLE_CAT")
36-
)
66+
return dialect.buildTableMetadata(tables)
3767
}
3868

3969
override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? {
40-
return null
70+
return dialect.convertSqlTypeToKType(tableColumnMetadata)
71+
}
72+
73+
public override fun sqlQueryLimit(sqlQuery: String, limit: Int): String {
74+
return dialect.sqlQueryLimit(sqlQuery, limit)
4175
}
4276
}

‎dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MsSql.kt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@ import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata
44
import org.jetbrains.kotlinx.dataframe.io.TableMetadata
55
import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema
66
import java.sql.ResultSet
7-
import java.util.*
7+
import java.util.Locale
88
import kotlin.reflect.KType
9-
import kotlin.reflect.full.createType
109

1110
/**
1211
* Represents the MSSQL database type.

0 commit comments

Comments
 (0)