Skip to content

Commit 08e604a

Browse files
sandwwraithsandwwraith
authoredJun 13, 2024··
Add integration with kotlinx-io library (#2707)
Integration is similar to Okio's one with functions `encodeToSink`, `decodeFromSource`, etc.
·
v1.8.0v1.7.1
1 parent d2dc7d2 commit 08e604a

File tree

20 files changed

+430
-110
lines changed

20 files changed

+430
-110
lines changed
 

‎benchmark/build.gradle.kts

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ tasks.processJmhResources {
3030
tasks.jmhJar {
3131
archiveBaseName.set("benchmarks")
3232
archiveVersion.set("")
33+
archiveClassifier.set("") // benchmarks.jar, not benchmarks-jmh.jar
3334
destinationDirectory.set(file("$rootDir"))
3435
}
3536

@@ -57,8 +58,10 @@ dependencies {
5758
implementation(libs.jackson.databind)
5859
implementation(libs.jackson.module.kotlin)
5960
implementation(libs.okio)
61+
implementation(libs.kotlinx.io)
6062
implementation(project(":kotlinx-serialization-core"))
6163
implementation(project(":kotlinx-serialization-json"))
6264
implementation(project(":kotlinx-serialization-json-okio"))
65+
implementation(project(":kotlinx-serialization-json-io"))
6366
implementation(project(":kotlinx-serialization-protobuf"))
6467
}

‎benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/TwitterFeedStreamBenchmark.kt

+46-4
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@ import com.fasterxml.jackson.databind.ObjectMapper
55
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
66
import kotlinx.benchmarks.model.MacroTwitterFeed
77
import kotlinx.benchmarks.model.MicroTwitterFeed
8+
import kotlinx.io.*
89
import kotlinx.serialization.json.*
10+
import kotlinx.serialization.json.io.*
11+
import kotlinx.serialization.json.okio.*
12+
import okio.*
913
import org.openjdk.jmh.annotations.*
1014
import java.io.*
11-
import java.nio.file.Files
12-
import java.nio.file.Path
1315
import java.util.concurrent.TimeUnit
14-
import kotlin.io.path.deleteIfExists
15-
import kotlin.io.path.outputStream
16+
import kotlin.io.use
17+
import okio.Buffer as OkioBuffer
18+
import okio.Sink as OkioSink
1619

1720
@Warmup(iterations = 7, time = 1)
1821
@Measurement(iterations = 7, time = 1)
@@ -67,6 +70,25 @@ open class TwitterFeedStreamBenchmark {
6770
}
6871
}
6972

73+
@Benchmark
74+
fun encodeTwitterOkioStream(): OkioSink {
75+
val b = OkioBuffer()
76+
Json.encodeToBufferedSink(MacroTwitterFeed.serializer(), twitter, b)
77+
return b
78+
}
79+
80+
@Benchmark
81+
fun encodeTwitterKotlinxIoStream(): Sink {
82+
val b = Buffer()
83+
Json.encodeToSink(MacroTwitterFeed.serializer(), twitter, b)
84+
return b
85+
}
86+
87+
/**
88+
* While encode* benchmarks use MacroTwitterFeed model to output as many bytes as possible,
89+
* decode* benchmarks use MicroTwitterFeed model to also factor for skipping over unnecessary data.
90+
*/
91+
7092
// Difference with TwitterFeedBenchmark.decodeMicroTwitter shows how heavy Java's standard UTF-8 decoding actually is.
7193
@Benchmark
7294
fun decodeMicroTwitterReadText(): MicroTwitterFeed {
@@ -88,4 +110,24 @@ open class TwitterFeedStreamBenchmark {
88110
objectMapper.readValue(it, MicroTwitterFeed::class.java)
89111
}
90112
}
113+
114+
@Benchmark
115+
fun decodeMicroTwitterOkioStream(): MicroTwitterFeed {
116+
// It seems there is no optimal way to reuse `bytes` between benchmark, so we are forced
117+
// to write them to buffer every time.
118+
// Note that it makes comparison with Jackson and InputStream integration much less meaningful.
119+
val b = OkioBuffer()
120+
b.write(bytes)
121+
return jsonIgnoreUnknwn.decodeFromBufferedSource(MicroTwitterFeed.serializer(), b)
122+
}
123+
124+
@Benchmark
125+
fun decodeMicroTwitterKotlinxIoStream(): MicroTwitterFeed {
126+
// It seems there is no way to reuse filled buffer between benchmark iterations, so we are forced
127+
// to write bytes to buffer every time.
128+
// Note that it makes comparison with Jackson and InputStream integration much less meaningful.
129+
val b = Buffer()
130+
b.write(bytes)
131+
return jsonIgnoreUnknwn.decodeFromSource(MicroTwitterFeed.serializer(), b)
132+
}
91133
}

‎build.gradle.kts

+2-1
Original file line numberDiff line numberDiff line change
@@ -184,9 +184,10 @@ val experimentalsInTestEnabled get() = listOf(
184184
val documentedSubprojects get() = setOf("kotlinx-serialization-core",
185185
"kotlinx-serialization-json",
186186
"kotlinx-serialization-json-okio",
187+
"kotlinx-serialization-json-io",
187188
"kotlinx-serialization-cbor",
188189
"kotlinx-serialization-properties",
189190
"kotlinx-serialization-hocon",
190191
"kotlinx-serialization-protobuf")
191192

192-
val uncoveredProjects get() = setOf("kotlinx-serialization-bom", "benchmark", "guide", "kotlinx-serialization-json-okio")
193+
val uncoveredProjects get() = setOf("kotlinx-serialization-bom", "benchmark", "guide", "kotlinx-serialization-json-okio", "kotlinx-serialization-json-io")

‎buildSrc/src/main/kotlin/publishing-conventions.gradle.kts

+11-4
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,16 @@ plugins {
1616
signing
1717
}
1818

19-
val isMultiplatform = name in listOf("kotlinx-serialization-core", "kotlinx-serialization-json", "kotlinx-serialization-json-okio",
20-
"kotlinx-serialization-json-tests", "kotlinx-serialization-protobuf", "kotlinx-serialization-cbor",
21-
"kotlinx-serialization-properties")
19+
val isMultiplatform = name in listOf(
20+
"kotlinx-serialization-core",
21+
"kotlinx-serialization-json",
22+
"kotlinx-serialization-json-okio",
23+
"kotlinx-serialization-json-io",
24+
"kotlinx-serialization-json-tests",
25+
"kotlinx-serialization-protobuf",
26+
"kotlinx-serialization-cbor",
27+
"kotlinx-serialization-properties"
28+
)
2229

2330
val isBom = name == "kotlinx-serialization-bom"
2431

@@ -221,4 +228,4 @@ fun mavenRepositoryUri(): URI {
221228

222229
fun Project.getSensitiveProperty(name: String): String? {
223230
return project.findProperty(name) as? String ?: System.getenv(name)
224-
}
231+
}

‎dokka/moduledoc.md

+7
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ Stable and ready to use JSON format implementation, `JsonElement` API to operate
99
Extensions for kotlinx.serialization.json.Json for integration with the popular [Okio](https://square.github.io/okio/) library.
1010
Currently experimental.
1111

12+
# Module kotlinx-serialization-json-io
13+
Extensions for kotlinx.serialization.json.Json for integration with the [kotlinx-io](https://github.com/Kotlin/kotlinx-io) library.
14+
Currently experimental.
15+
1216
# Module kotlinx-serialization-cbor
1317
Concise Binary Object Representation (CBOR) format implementation, as per [RFC 7049](https://tools.ietf.org/html/rfc7049).
1418

@@ -49,6 +53,9 @@ and JSON-specific serializers.
4953
# Package kotlinx.serialization.json.okio
5054
Extensions for kotlinx.serialization.json.Json for integration with the popular [Okio](https://square.github.io/okio/) library.
5155

56+
# Package kotlinx.serialization.json.io
57+
Extensions for kotlinx.serialization.json.Json for integration with the [kotlinx-io](https://github.com/Kotlin/kotlinx-io) library.
58+
5259
# Package kotlinx.serialization.protobuf
5360
[Protocol buffers](https://protobuf.dev/) serialization format implementation.
5461

‎formats/json-io/api/kotlinx-serialization-json-io.api

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
public final class kotlinx/serialization/json/io/IoStreamsKt {
2+
public static final fun decodeFromSource (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/io/Source;)Ljava/lang/Object;
3+
public static final fun decodeSourceToSequence (Lkotlinx/serialization/json/Json;Lkotlinx/io/Source;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;)Lkotlin/sequences/Sequence;
4+
public static synthetic fun decodeSourceToSequence$default (Lkotlinx/serialization/json/Json;Lkotlinx/io/Source;Lkotlinx/serialization/DeserializationStrategy;Lkotlinx/serialization/json/DecodeSequenceMode;ILjava/lang/Object;)Lkotlin/sequences/Sequence;
5+
public static final fun encodeToSink (Lkotlinx/serialization/json/Json;Lkotlinx/serialization/SerializationStrategy;Ljava/lang/Object;Lkotlinx/io/Sink;)V
6+
}
7+

‎formats/json-io/api/kotlinx-serialization-json-io.klib.api

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Klib ABI Dump
2+
// Targets: [androidNativeArm32, androidNativeArm64, androidNativeX64, androidNativeX86, iosArm64, iosSimulatorArm64, iosX64, js, linuxArm32Hfp, linuxArm64, linuxX64, macosArm64, macosX64, mingwX64, tvosArm64, tvosSimulatorArm64, tvosX64, wasmJs, wasmWasi, watchosArm32, watchosArm64, watchosDeviceArm64, watchosSimulatorArm64, watchosX64]
3+
// Rendering settings:
4+
// - Signature version: 2
5+
// - Show manifest properties: true
6+
// - Show declarations: true
7+
8+
// Library unique name: <org.jetbrains.kotlinx:kotlinx-serialization-json-io>
9+
final fun <#A: kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeFromSource(kotlinx.serialization/DeserializationStrategy<#A>, kotlinx.io/Source): #A // kotlinx.serialization.json.io/decodeFromSource|decodeFromSource@kotlinx.serialization.json.Json(kotlinx.serialization.DeserializationStrategy<0:0>;kotlinx.io.Source){0§<kotlin.Any?>}[0]
10+
final fun <#A: kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeSourceToSequence(kotlinx.io/Source, kotlinx.serialization/DeserializationStrategy<#A>, kotlinx.serialization.json/DecodeSequenceMode = ...): kotlin.sequences/Sequence<#A> // kotlinx.serialization.json.io/decodeSourceToSequence|decodeSourceToSequence@kotlinx.serialization.json.Json(kotlinx.io.Source;kotlinx.serialization.DeserializationStrategy<0:0>;kotlinx.serialization.json.DecodeSequenceMode){0§<kotlin.Any?>}[0]
11+
final fun <#A: kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/encodeToSink(kotlinx.serialization/SerializationStrategy<#A>, #A, kotlinx.io/Sink) // kotlinx.serialization.json.io/encodeToSink|encodeToSink@kotlinx.serialization.json.Json(kotlinx.serialization.SerializationStrategy<0:0>;0:0;kotlinx.io.Sink){0§<kotlin.Any?>}[0]
12+
final inline fun <#A: reified kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeFromSource(kotlinx.io/Source): #A // kotlinx.serialization.json.io/decodeFromSource|decodeFromSource@kotlinx.serialization.json.Json(kotlinx.io.Source){0§<kotlin.Any?>}[0]
13+
final inline fun <#A: reified kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/decodeSourceToSequence(kotlinx.io/Source, kotlinx.serialization.json/DecodeSequenceMode = ...): kotlin.sequences/Sequence<#A> // kotlinx.serialization.json.io/decodeSourceToSequence|decodeSourceToSequence@kotlinx.serialization.json.Json(kotlinx.io.Source;kotlinx.serialization.json.DecodeSequenceMode){0§<kotlin.Any?>}[0]
14+
final inline fun <#A: reified kotlin/Any?> (kotlinx.serialization.json/Json).kotlinx.serialization.json.io/encodeToSink(#A, kotlinx.io/Sink) // kotlinx.serialization.json.io/encodeToSink|encodeToSink@kotlinx.serialization.json.Json(0:0;kotlinx.io.Sink){0§<kotlin.Any?>}[0]

‎formats/json-io/build.gradle.kts

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
import Java9Modularity.configureJava9ModuleInfo
5+
import org.jetbrains.dokka.gradle.*
6+
import java.net.*
7+
8+
plugins {
9+
kotlin("multiplatform")
10+
kotlin("plugin.serialization")
11+
12+
id("native-targets-conventions")
13+
id("source-sets-conventions")
14+
}
15+
16+
kotlin {
17+
sourceSets {
18+
configureEach {
19+
languageSettings {
20+
optIn("kotlinx.serialization.internal.CoreFriendModuleApi")
21+
optIn("kotlinx.serialization.json.internal.JsonFriendModuleApi")
22+
}
23+
}
24+
val commonMain by getting {
25+
dependencies {
26+
api(project(":kotlinx-serialization-core"))
27+
api(project(":kotlinx-serialization-json"))
28+
implementation(libs.kotlinx.io)
29+
}
30+
}
31+
}
32+
}
33+
34+
project.configureJava9ModuleInfo()
35+
36+
tasks.named<DokkaTaskPartial>("dokkaHtmlPartial") {
37+
dokkaSourceSets {
38+
configureEach {
39+
externalDocumentationLink {
40+
url.set(URL("https://kotlin.github.io/kotlinx-io/"))
41+
}
42+
}
43+
}
44+
}

‎formats/json-io/commonMain/src/kotlinx/serialization/json/io/IoStreams.kt

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package kotlinx.serialization.json.io
6+
7+
import kotlinx.serialization.*
8+
import kotlinx.serialization.json.DecodeSequenceMode
9+
import kotlinx.serialization.json.Json
10+
import kotlinx.serialization.json.internal.*
11+
import kotlinx.serialization.json.io.internal.JsonToIoStreamWriter
12+
import kotlinx.serialization.json.internal.decodeToSequenceByReader
13+
import kotlinx.serialization.json.io.internal.IoSerialReader
14+
import kotlinx.io.*
15+
16+
/**
17+
* Serializes the [value] with [serializer] into a [sink] using JSON format and UTF-8 encoding.
18+
*
19+
* @throws [SerializationException] if the given value cannot be serialized to JSON.
20+
* @throws [kotlinx.io.IOException] If an I/O error occurs and sink can't be written to.
21+
*/
22+
@ExperimentalSerializationApi
23+
public fun <T> Json.encodeToSink(
24+
serializer: SerializationStrategy<T>,
25+
value: T,
26+
sink: Sink
27+
) {
28+
val writer = JsonToIoStreamWriter(sink)
29+
try {
30+
encodeByWriter(this, writer, serializer, value)
31+
} finally {
32+
writer.release()
33+
}
34+
}
35+
36+
/**
37+
* Serializes given [value] to a [sink] using UTF-8 encoding and serializer retrieved from the reified type parameter.
38+
*
39+
* @throws [SerializationException] if the given value cannot be serialized to JSON.
40+
* @throws [kotlinx.io.IOException] If an I/O error occurs and sink can't be written to.
41+
*/
42+
@ExperimentalSerializationApi
43+
public inline fun <reified T> Json.encodeToSink(
44+
value: T,
45+
sink: Sink
46+
): Unit = encodeToSink(serializersModule.serializer(), value, sink)
47+
48+
49+
/**
50+
* Deserializes JSON from [source] using UTF-8 encoding to a value of type [T] using [deserializer].
51+
*
52+
* Note that this functions expects that exactly one object would be present in the source
53+
* and throws an exception if there are any dangling bytes after an object.
54+
*
55+
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T].
56+
* @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from.
57+
*/
58+
@ExperimentalSerializationApi
59+
public fun <T> Json.decodeFromSource(
60+
deserializer: DeserializationStrategy<T>,
61+
source: Source
62+
): T {
63+
return decodeByReader(this, deserializer, IoSerialReader(source))
64+
}
65+
66+
/**
67+
* Deserializes the contents of given [source] to the value of type [T] using UTF-8 encoding and
68+
* deserializer retrieved from the reified type parameter.
69+
*
70+
* Note that this functions expects that exactly one object would be present in the stream
71+
* and throws an exception if there are any dangling bytes after an object.
72+
*
73+
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T].
74+
* @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from.
75+
*/
76+
@ExperimentalSerializationApi
77+
public inline fun <reified T> Json.decodeFromSource(source: Source): T =
78+
decodeFromSource(serializersModule.serializer(), source)
79+
80+
81+
/**
82+
* Transforms the given [source] into lazily deserialized sequence of elements of type [T] using UTF-8 encoding and [deserializer].
83+
* Unlike [decodeFromSource], [source] is allowed to have more than one element, separated as [format] declares.
84+
*
85+
* Elements must all be of type [T].
86+
* Elements are parsed lazily when resulting [Sequence] is evaluated.
87+
* Resulting sequence is tied to the stream and can be evaluated only once.
88+
*
89+
* **Resource caution:** this method neither closes the [source] when the parsing is finished nor provides a method to close it manually.
90+
* It is a caller responsibility to hold a reference to a source and close it. Moreover, because source is parsed lazily,
91+
* closing it before returned sequence is evaluated completely will result in [Exception] from decoder.
92+
*
93+
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T].
94+
* @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from.
95+
*/
96+
@ExperimentalSerializationApi
97+
public fun <T> Json.decodeSourceToSequence(
98+
source: Source,
99+
deserializer: DeserializationStrategy<T>,
100+
format: DecodeSequenceMode = DecodeSequenceMode.AUTO_DETECT
101+
): Sequence<T> {
102+
return decodeToSequenceByReader(this, IoSerialReader(source), deserializer, format)
103+
}
104+
105+
/**
106+
* Transforms the given [source] into lazily deserialized sequence of elements of type [T] using UTF-8 encoding and deserializer retrieved from the reified type parameter.
107+
* Unlike [decodeSourceToSequence], [source] is allowed to have more than one element, separated as [format] declares.
108+
*
109+
* Elements must all be of type [T].
110+
* Elements are parsed lazily when resulting [Sequence] is evaluated.
111+
* Resulting sequence is tied to the stream and constrained to be evaluated only once.
112+
*
113+
* **Resource caution:** this method does not close [source] when the parsing is finished neither provides method to close it manually.
114+
* It is a caller responsibility to hold a reference to a source and close it. Moreover, because source is parsed lazily,
115+
* closing it before returned sequence is evaluated fully would result in [Exception] from decoder.
116+
*
117+
* @throws [SerializationException] if the given JSON input cannot be deserialized to the value of type [T].
118+
* @throws [kotlinx.io.IOException] If an I/O error occurs and source can't be read from.
119+
*/
120+
@ExperimentalSerializationApi
121+
public inline fun <reified T> Json.decodeSourceToSequence(
122+
source: Source,
123+
format: DecodeSequenceMode = DecodeSequenceMode.AUTO_DETECT
124+
): Sequence<T> = decodeSourceToSequence(source, serializersModule.serializer(), format)

‎formats/json-io/commonMain/src/kotlinx/serialization/json/io/internal/IoJsonStreams.kt

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
package kotlinx.serialization.json.io.internal
6+
7+
import kotlinx.io.*
8+
import kotlinx.serialization.json.internal.*
9+
10+
private const val QUOTE_CODE = '"'.code
11+
12+
internal class JsonToIoStreamWriter(private val sink: Sink) : InternalJsonWriter {
13+
14+
override fun writeLong(value: Long) {
15+
write(value.toString())
16+
}
17+
18+
override fun writeChar(char: Char) {
19+
sink.writeCodePointValue(char.code)
20+
}
21+
22+
override fun write(text: String) {
23+
sink.writeString(text)
24+
}
25+
26+
override fun writeQuoted(text: String) {
27+
sink.writeCodePointValue(QUOTE_CODE)
28+
InternalJsonWriter.doWriteEscaping(text) { s, start, end -> sink.writeString(s, start, end) }
29+
sink.writeCodePointValue(QUOTE_CODE)
30+
}
31+
32+
override fun release() {
33+
// no-op, see https://github.com/Kotlin/kotlinx.serialization/pull/1982#discussion_r915043700
34+
}
35+
}
36+
37+
internal class IoSerialReader(private val source: Source): InternalJsonReaderCodePointImpl() {
38+
override fun exhausted(): Boolean = source.exhausted()
39+
override fun nextCodePoint(): Int = source.readCodePointValue()
40+
}

‎formats/json-okio/build.gradle.kts

-5
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,6 @@ kotlin {
2929
implementation(libs.okio)
3030
}
3131
}
32-
val commonTest by getting {
33-
dependencies {
34-
implementation(libs.okio)
35-
}
36-
}
3732
}
3833
}
3934

0 commit comments

Comments
 (0)
Please sign in to comment.