Add some documentation for ProtoBufSchemaGenerator in formats.md (#1855)

Author: justinhj

docs/formats.md

@@ -17,6 +17,7 @@ stable, these are currently experimental features of Kotlin Serialization.
   * [Field numbers](#field-numbers)
   * [Integer types](#integer-types)
   * [Lists as repeated fields](#lists-as-repeated-fields)
+  * [ProtoBuf schema generator (experimental)](#protobuf-schema-generator-experimental)
 * [Properties (experimental)](#properties-experimental)
 * [Custom formats (experimental)](#custom-formats-experimental)
   * [Basic encoder](#basic-encoder)
@@ -428,6 +429,62 @@ Field #1: 08 Varint Value = 2, Hex = 02
 Field #1: 08 Varint Value = 3, Hex = 03
 ```
 
+### ProtoBuf schema generator (experimental)
+
+As mentioned above, when working with protocol buffers you usually use a ".proto" file and a code generator for your
+language.  This includes the code to serialize your message to an output stream and deserialize it from an input stream.
+When using Kotlin Serialization this step is not necessary because your `@Serializable` Kotlin data types are used as the
+source for the schema.
+
+This is very convenient for Kotlin-to-Kotlin communication, but makes interoperability between languages complicated.
+Fortunately, you can use the ProtoBuf schema generator to output the ".proto" representation of your messages.  You can
+keep your Kotlin classes as a source of truth and use traditional protoc compilers for other languages at the same time.
+
+As an example, we can display the following data class's ".proto" schema as follows.
+
+<!--- INCLUDE
+import kotlinx.serialization.*
+import kotlinx.serialization.protobuf.*
+import kotlinx.serialization.protobuf.schema.ProtoBufSchemaGenerator
+-->
+
+```kotlin
+@Serializable
+data class SampleData(
+    val amount: Long,
+    val description: String?,
+    val department: String = "QA"
+)
+fun main() {
+  val descriptors = listOf(SampleData.serializer().descriptor)
+  val schemas = ProtoBufSchemaGenerator.generateSchemaText(descriptors)
+  println(schemas)
+}
+```
+> You can get the full code [here](../guide/example/example-formats-08.kt).
+
+Which would output as follows.
+
+```text
+syntax = "proto2";
+
+
+// serial name 'example.exampleFormats08.SampleData'
+message SampleData {
+  required int64 amount = 1;
+  optional string description = 2;
+  // WARNING: a default value decoded when value is missing
+  optional string department = 3;
+}
+
+```
+
+<!--- TEST -->
+
+Note that since default values are not represented in ".proto" files, a warning is generated when one appears in the schema.
+
+See the documentation for [ProtoBufSchemaGenerator] for more information.
+
 ## Properties (experimental)
 
 Kotlin Serialization can serialize a class into a flat map with `String` keys via 
@@ -456,7 +513,7 @@ fun main() {
 }
 ```      
 
-> You can get the full code [here](../guide/example/example-formats-08.kt).
+> You can get the full code [here](../guide/example/example-formats-09.kt).
 
 The resulting map has dot-separated keys representing keys of the nested objects.
 
@@ -536,7 +593,7 @@ fun main() {
 }
 ```                                    
 
-> You can get the full code [here](../guide/example/example-formats-09.kt).
+> You can get the full code [here](../guide/example/example-formats-10.kt).
 
 As a result, we got all the primitive values in our object graph visited and put into a list
 in _serial_ order.
@@ -638,7 +695,7 @@ fun main() {
 }
 ```
 
-> You can get the full code [here](../guide/example/example-formats-10.kt).
+> You can get the full code [here](../guide/example/example-formats-11.kt).
 
 Now we can convert a list of primitives back to an object tree.
 
@@ -729,7 +786,7 @@ fun main() {
 }
 -->
 
-> You can get the full code [here](../guide/example/example-formats-11.kt).
+> You can get the full code [here](../guide/example/example-formats-12.kt).
 
 <!--- TEST 
 [kotlinx.serialization, kotlin, 9000]
@@ -836,7 +893,7 @@ fun main() {
 }
 ```
 
-> You can get the full code [here](../guide/example/example-formats-12.kt).
+> You can get the full code [here](../guide/example/example-formats-13.kt).
 
 We see the size of the list added to the result, letting the decoder know where to stop. 
 
@@ -948,7 +1005,7 @@ fun main() {
 
 ```
 
-> You can get the full code [here](../guide/example/example-formats-13.kt).
+> You can get the full code [here](../guide/example/example-formats-14.kt).
 
 In the output we see how not-null`!!` and `NULL` marks are used.
 
@@ -1076,7 +1133,7 @@ fun main() {
 }
 ```
               
-> You can get the full code [here](../guide/example/example-formats-14.kt).
+> You can get the full code [here](../guide/example/example-formats-15.kt).
 
 As we can see, the result is a dense binary format that only contains the data that is being serialized. 
 It can be easily tweaked for any kind of domain-specific compact encoding.
@@ -1270,7 +1327,7 @@ fun main() {
 }
 ```
               
-> You can get the full code [here](../guide/example/example-formats-15.kt).
+> You can get the full code [here](../guide/example/example-formats-16.kt).
 
 As we can see, our custom byte array format is being used, with the compact encoding of its size in one byte. 
 
@@ -1341,6 +1398,10 @@ This chapter concludes [Kotlin Serialization Guide](serialization-guide.md).
 [ProtoIntegerType.SIGNED]: https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-protobuf/kotlinx.serialization.protobuf/-proto-integer-type/-s-i-g-n-e-d/index.html
 [ProtoIntegerType.FIXED]: https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-protobuf/kotlinx.serialization.protobuf/-proto-integer-type/-f-i-x-e-d/index.html
 
+<!--- INDEX kotlinx-serialization-protobuf/kotlinx.serialization.protobuf.schema -->
+
+[ProtoBufSchemaGenerator]: https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-protobuf/kotlinx.serialization.protobuf.schema/-proto-buf-schema-generator/index.html
+
 <!--- MODULE /kotlinx-serialization-cbor -->
 <!--- INDEX kotlinx-serialization-cbor/kotlinx.serialization.cbor -->
 

docs/serialization-guide.md

@@ -140,6 +140,7 @@ Once the project is set up, we can start serializing some classes.
   * <a name='field-numbers'></a>[Field numbers](formats.md#field-numbers)
   * <a name='integer-types'></a>[Integer types](formats.md#integer-types)
   * <a name='lists-as-repeated-fields'></a>[Lists as repeated fields](formats.md#lists-as-repeated-fields)
+  * <a name='protobuf-schema-generator-experimental'></a>[ProtoBuf schema generator (experimental)](formats.md#protobuf-schema-generator-experimental)
 * <a name='properties-experimental'></a>[Properties (experimental)](formats.md#properties-experimental)
 * <a name='custom-formats-experimental'></a>[Custom formats (experimental)](formats.md#custom-formats-experimental)
   * <a name='basic-encoder'></a>[Basic encoder](formats.md#basic-encoder)

guide/example/example-formats-08.kt

@@ -2,17 +2,17 @@
 package example.exampleFormats08
 
 import kotlinx.serialization.*
-import kotlinx.serialization.properties.Properties // todo: remove when no longer needed
-import kotlinx.serialization.properties.*
+import kotlinx.serialization.protobuf.*
+import kotlinx.serialization.protobuf.schema.ProtoBufSchemaGenerator
 
 @Serializable
-class Project(val name: String, val owner: User)
-
-@Serializable
-class User(val name: String)
-
+data class SampleData(
+    val amount: Long,
+    val description: String?,
+    val department: String = "QA"
+)
 fun main() {
-    val data = Project("kotlinx.serialization",  User("kotlin"))
-    val map = Properties.encodeToMap(data)
-    map.forEach { (k, v) -> println("$k = $v") }
+  val descriptors = listOf(SampleData.serializer().descriptor)
+  val schemas = ProtoBufSchemaGenerator.generateSchemaText(descriptors)
+  println(schemas)
 }

guide/example/example-formats-09.kt

@@ -2,35 +2,17 @@
 package example.exampleFormats09
 
 import kotlinx.serialization.*
-import kotlinx.serialization.descriptors.*
-import kotlinx.serialization.encoding.*
-import kotlinx.serialization.modules.*
-
-class ListEncoder : AbstractEncoder() {
-    val list = mutableListOf<Any>()
-
-    override val serializersModule: SerializersModule = EmptySerializersModule
-
-    override fun encodeValue(value: Any) {
-        list.add(value)
-    }
-}
-
-fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any> {
-    val encoder = ListEncoder()
-    encoder.encodeSerializableValue(serializer, value)
-    return encoder.list
-}
-
-inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
+import kotlinx.serialization.properties.Properties // todo: remove when no longer needed
+import kotlinx.serialization.properties.*
 
 @Serializable
-data class Project(val name: String, val owner: User, val votes: Int)
+class Project(val name: String, val owner: User)
 
 @Serializable
-data class User(val name: String)
+class User(val name: String)
 
 fun main() {
-    val data = Project("kotlinx.serialization",  User("kotlin"), 9000)
-    println(encodeToList(data))
+    val data = Project("kotlinx.serialization",  User("kotlin"))
+    val map = Properties.encodeToMap(data)
+    map.forEach { (k, v) -> println("$k = $v") }
 }

guide/example/example-formats-10.kt

@@ -24,29 +24,6 @@ fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any>
 
 inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
 
-class ListDecoder(val list: ArrayDeque<Any>) : AbstractDecoder() {
-    private var elementIndex = 0
-
-    override val serializersModule: SerializersModule = EmptySerializersModule
-
-    override fun decodeValue(): Any = list.removeFirst()
-    
-    override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
-        if (elementIndex == descriptor.elementsCount) return CompositeDecoder.DECODE_DONE
-        return elementIndex++
-    }
-
-    override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        ListDecoder(list)
-}
-
-fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
-    val decoder = ListDecoder(ArrayDeque(list))
-    return decoder.decodeSerializableValue(deserializer)
-}
-
-inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
-
 @Serializable
 data class Project(val name: String, val owner: User, val votes: Int)
 
@@ -55,8 +32,5 @@ data class User(val name: String)
 
 fun main() {
     val data = Project("kotlinx.serialization",  User("kotlin"), 9000)
-    val list = encodeToList(data)
-    println(list)
-    val obj = decodeFromList<Project>(list)
-    println(obj)
+    println(encodeToList(data))
 }

guide/example/example-formats-11.kt

@@ -37,10 +37,8 @@ class ListDecoder(val list: ArrayDeque<Any>) : AbstractDecoder() {
     }
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        ListDecoder(list) 
-
-    override fun decodeSequentially(): Boolean = true
-}        
+        ListDecoder(list)
+}
 
 fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
     val decoder = ListDecoder(ArrayDeque(list))

guide/example/example-formats-12.kt

@@ -13,12 +13,7 @@ class ListEncoder : AbstractEncoder() {
 
     override fun encodeValue(value: Any) {
         list.add(value)
-    }                               
-
-    override fun beginCollection(descriptor: SerialDescriptor, collectionSize: Int): CompositeEncoder {
-        encodeInt(collectionSize)
-        return this
-    }                                                
+    }
 }
 
 fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any> {
@@ -29,26 +24,23 @@ fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any>
 
 inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value)
 
-class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : AbstractDecoder() {
+class ListDecoder(val list: ArrayDeque<Any>) : AbstractDecoder() {
     private var elementIndex = 0
 
     override val serializersModule: SerializersModule = EmptySerializersModule
 
     override fun decodeValue(): Any = list.removeFirst()
-
+    
     override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
-        if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE
+        if (elementIndex == descriptor.elementsCount) return CompositeDecoder.DECODE_DONE
         return elementIndex++
     }
 
     override fun beginStructure(descriptor: SerialDescriptor): CompositeDecoder =
-        ListDecoder(list, descriptor.elementsCount)
+        ListDecoder(list) 
 
     override fun decodeSequentially(): Boolean = true
-
-    override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
-        decodeInt().also { elementsCount = it }
-}
+}        
 
 fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
     val decoder = ListDecoder(ArrayDeque(list))
@@ -58,13 +50,13 @@ fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>
 inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
 
 @Serializable
-data class Project(val name: String, val owners: List<User>, val votes: Int)
+data class Project(val name: String, val owner: User, val votes: Int)
 
 @Serializable
 data class User(val name: String)
 
 fun main() {
-    val data = Project("kotlinx.serialization",  listOf(User("kotlin"), User("jetbrains")), 9000)
+    val data = Project("kotlinx.serialization",  User("kotlin"), 9000)
     val list = encodeToList(data)
     println(list)
     val obj = decodeFromList<Project>(list)

guide/example/example-formats-13.kt

@@ -19,9 +19,6 @@ class ListEncoder : AbstractEncoder() {
         encodeInt(collectionSize)
         return this
     }                                                
-
-    override fun encodeNull() = encodeValue("NULL")
-    override fun encodeNotNullMark() = encodeValue("!!")
 }
 
 fun <T> encodeToList(serializer: SerializationStrategy<T>, value: T): List<Any> {
@@ -34,7 +31,7 @@ inline fun <reified T> encodeToList(value: T) = encodeToList(serializer(), value
 
 class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : AbstractDecoder() {
     private var elementIndex = 0
-    
+
     override val serializersModule: SerializersModule = EmptySerializersModule
 
     override fun decodeValue(): Any = list.removeFirst()
@@ -51,8 +48,6 @@ class ListDecoder(val list: ArrayDeque<Any>, var elementsCount: Int = 0) : Abstr
 
     override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
         decodeInt().also { elementsCount = it }
-
-    override fun decodeNotNullMark(): Boolean = decodeString() != "NULL"
 }
 
 fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>): T {
@@ -63,16 +58,15 @@ fun <T> decodeFromList(list: List<Any>, deserializer: DeserializationStrategy<T>
 inline fun <reified T> decodeFromList(list: List<Any>): T = decodeFromList(list, serializer())
 
 @Serializable
-data class Project(val name: String, val owner: User?, val votes: Int?)
+data class Project(val name: String, val owners: List<User>, val votes: Int)
 
 @Serializable
 data class User(val name: String)
 
 fun main() {
-    val data = Project("kotlinx.serialization",  User("kotlin") , null)
+    val data = Project("kotlinx.serialization",  listOf(User("kotlin"), User("jetbrains")), 9000)
     val list = encodeToList(data)
     println(list)
     val obj = decodeFromList<Project>(list)
     println(obj)
 }
-