fix!: adhere more closely to the language guide for proto3 default values (#66)

Only write singular fields when they are non-default values (unless in repeated fields).  Always write optional values even when they are non-default values.

Updates test suite to compare generated bytes to protobuf.js to ensure compatibility.

Also adds a README section on differences between protons and protobuf.js.

BREAKING CHANGE: ts definitions will need to be generated from `.proto` files - singular message fields have become optional as message fields are always optional in proto3

fixes #43

Author: achingbrain

packages/protons-runtime/src/codec.ts

@@ -12,6 +12,7 @@ export enum CODEC_TYPES {
 
 export interface EncodeOptions {
   lengthDelimited?: boolean
+  writeDefaults?: boolean
 }
 
 export interface EncodeFunction<T> {

packages/protons-runtime/src/codecs/enum.ts

@@ -20,7 +20,7 @@ export function enumeration <T> (v: any): Codec<T> {
   }
 
   const decode: DecodeFunction<number | string> = function enumDecode (reader) {
-    const val = reader.uint32()
+    const val = reader.int32()
 
     return findValue(val)
   }

packages/protons-runtime/src/decode.ts

@@ -1,25 +1,9 @@
 import type { Uint8ArrayList } from 'uint8arraylist'
 import type { Codec } from './codec.js'
-import pb from 'protobufjs'
-
-const Reader = pb.Reader
-
-// monkey patch the reader to add native bigint support
-const methods = [
-  'uint64', 'int64', 'sint64', 'fixed64', 'sfixed64'
-]
-methods.forEach(method => {
-  // @ts-expect-error
-  const original = Reader.prototype[method]
-  // @ts-expect-error
-  Reader.prototype[method] = function (): bigint {
-    return BigInt(original.call(this).toString())
-  }
-})
+import { reader } from './reader.js'
 
 export function decodeMessage <T> (buf: Uint8Array | Uint8ArrayList, codec: Codec<T>): T {
-  const reader = Reader.create(buf instanceof Uint8Array ? buf : buf.subarray())
+  const r = reader(buf instanceof Uint8Array ? buf : buf.subarray())
 
-  // @ts-expect-error
-  return codec.decode(reader)
+  return codec.decode(r)
 }

packages/protons-runtime/src/encode.ts

@@ -1,25 +1,9 @@
 import type { Codec } from './codec.js'
-import pb from 'protobufjs'
-
-const Writer = pb.Writer
-
-// monkey patch the writer to add native bigint support
-const methods = [
-  'uint64', 'int64', 'sint64', 'fixed64', 'sfixed64'
-]
-methods.forEach(method => {
-  // @ts-expect-error
-  const original = Writer.prototype[method]
-  // @ts-expect-error
-  Writer.prototype[method] = function (val: bigint): pb.Writer {
-    return original.call(this, val.toString())
-  }
-})
+import { writer } from './writer.js'
 
 export function encodeMessage <T> (message: T, codec: Codec<T>): Uint8Array {
-  const w = Writer.create()
+  const w = writer()
 
-  // @ts-expect-error
   codec.encode(message, w, {
     lengthDelimited: false
   })

packages/protons-runtime/src/index.ts

@@ -18,6 +18,8 @@ export {
 
 export { enumeration } from './codecs/enum.js'
 export { message } from './codecs/message.js'
+export { reader } from './reader.js'
+export { writer } from './writer.js'
 export type { Codec, EncodeOptions } from './codec.js'
 
 export interface Writer {

packages/protons-runtime/src/reader.ts

@@ -0,0 +1,22 @@
+import pb from 'protobufjs/minimal.js'
+import type { Reader } from './index.js'
+
+const PBReader = pb.Reader
+
+// monkey patch the reader to add native bigint support
+const methods = [
+  'uint64', 'int64', 'sint64', 'fixed64', 'sfixed64'
+]
+methods.forEach(method => {
+  // @ts-expect-error
+  const original = PBReader.prototype[method]
+  // @ts-expect-error
+  PBReader.prototype[method] = function (): bigint {
+    return BigInt(original.call(this).toString())
+  }
+})
+
+export function reader (buf: Uint8Array): Reader {
+  // @ts-expect-error class is monkey patched
+  return PBReader.create(buf)
+}

packages/protons-runtime/src/writer.ts

@@ -0,0 +1,22 @@
+import pb from 'protobufjs/minimal.js'
+import type { Writer } from './index.js'
+
+const PBWriter = pb.Writer
+
+// monkey patch the writer to add native bigint support
+const methods = [
+  'uint64', 'int64', 'sint64', 'fixed64', 'sfixed64'
+]
+methods.forEach(method => {
+  // @ts-expect-error
+  const original = PBWriter.prototype[method]
+  // @ts-expect-error
+  PBWriter.prototype[method] = function (val: bigint): pb.Writer {
+    return original.call(this, val.toString())
+  }
+})
+
+export function writer (): Writer {
+  // @ts-expect-error class is monkey patched
+  return PBWriter.create()
+}

packages/protons/README.md

@@ -12,6 +12,7 @@
 
 - [Install](#install)
 - [Usage](#usage)
+- [Differences from protobuf.js](#differences-from-protobufjs)
 - [Contribute](#contribute)
 - [License](#license)
 - [Contribute](#contribute-1)
@@ -59,6 +60,21 @@ console.info(decoded.message)
 // 'hello world'
 ```
 
+## Differences from protobuf.js
+
+This module uses the internal reader/writer from `protobuf.js` as it is highly optimised and there's no point reinventing the wheel.
+
+It does have one or two differences:
+
+1. Supports `proto3` semantics only
+2. All 64 bit values are represented as `BigInt`s and not `Long`s (e.g. `int64`, `uint64`, `sint64` etc)
+3. Unset `optional` fields are set on the deserialized object forms as `undefined` instead of the default values
+4. `singular` fields set to default values are not serialized and are set to default values when deserialized if not set - protobuf.js [diverges from the language guide](https://github.com/protobufjs/protobuf.js/issues/1468#issuecomment-745177012) around this feature
+
+## Missing features
+
+Some features are missing `OneOf`, `Map`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.
+
 ## Contribute
 
 Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/protons/issues)!