feat: runtime size limits for arrays and maps (#128)

It's possible to limit the size of arrays and maps at compile time:

```protobuf
message MyMessage {
  repeated uint32 repeatedField = 1 [(protons.options).limit = 10];
  map<string, string> stringMap = 2 [(protons.options).limit = 10];
}
```

This PR adds the ability to do it at runtime too:

```TypeScript
const message = MyMessage.decode(buf, {
  limits: {
    repeatedField: 10,
    stringMap: 10
  }
})
```

Author: achingbrain

packages/protons-runtime/src/codec.ts

@@ -19,8 +19,15 @@ export interface EncodeFunction<T> {
   (value: Partial<T>, writer: Writer, opts?: EncodeOptions): void
 }
 
+export interface DecodeOptions<T> {
+  /**
+   * Runtime-specified limits for lengths of repeated/map fields
+   */
+  limits?: Partial<Record<keyof T, number>>
+}
+
 export interface DecodeFunction<T> {
-  (reader: Reader, length?: number): T
+  (reader: Reader, length?: number, opts?: DecodeOptions<T>): T
 }
 
 export interface Codec<T> {

packages/protons-runtime/src/codecs/message.ts

@@ -1,10 +1,9 @@
-import { createCodec, CODEC_TYPES, type EncodeOptions, type Codec } from '../codec.js'
-import type { Reader, Writer } from '../index.js'
+import { createCodec, CODEC_TYPES, type EncodeFunction, type DecodeFunction, type Codec } from '../codec.js'
 
 export interface Factory<A, T> {
   new (obj: A): T
 }
 
-export function message <T> (encode: (obj: Partial<T>, writer: Writer, opts?: EncodeOptions) => void, decode: (reader: Reader, length?: number) => T): Codec<T> {
+export function message <T> (encode: EncodeFunction<T>, decode: DecodeFunction<T>): Codec<T> {
   return createCodec('message', CODEC_TYPES.LENGTH_DELIMITED, encode, decode)
 }

packages/protons-runtime/src/decode.ts

@@ -1,9 +1,9 @@
 import { createReader } from './utils/reader.js'
-import type { Codec } from './codec.js'
+import type { Codec, DecodeOptions } from './codec.js'
 import type { Uint8ArrayList } from 'uint8arraylist'
 
-export function decodeMessage <T> (buf: Uint8Array | Uint8ArrayList, codec: Codec<T>): T {
+export function decodeMessage <T> (buf: Uint8Array | Uint8ArrayList, codec: Codec<T>, opts?: DecodeOptions<T>): T {
   const reader = createReader(buf)
 
-  return codec.decode(reader)
+  return codec.decode(reader, undefined, opts)
 }

packages/protons-runtime/src/index.ts

@@ -20,7 +20,7 @@ export { enumeration } from './codecs/enum.js'
 export { message } from './codecs/message.js'
 export { createReader as reader } from './utils/reader.js'
 export { createWriter as writer } from './utils/writer.js'
-export type { Codec, EncodeOptions } from './codec.js'
+export type { Codec, EncodeOptions, DecodeOptions } from './codec.js'
 
 export interface Writer {
   /**

packages/protons/README.md

@@ -12,6 +12,9 @@
 - [Install](#install)
 - [Usage](#usage)
 - [Differences from protobuf.js](#differences-from-protobufjs)
+- [Extra features](#extra-features)
+  - [Limiting the size of repeated/map elements](#limiting-the-size-of-repeatedmap-elements)
+  - [Overriding 64 bit types](#overriding-64-bit-types)
 - [Missing features](#missing-features)
 - [API Docs](#api-docs)
 - [License](#license)
@@ -73,6 +76,59 @@ It does have one or two differences:
 5. `map` fields can have keys of any type - protobufs.js [only supports strings](https://github.com/protobufjs/protobuf.js/issues/1203#issuecomment-488637338)
 6. `map` fields are deserialized as ES6 `Map`s - protobuf.js uses `Object`s
 
+## Extra features
+
+### Limiting the size of repeated/map elements
+
+To protect decoders from malicious payloads, it's possible to limit the maximum size of repeated/map elements.
+
+You can either do this at compile time by using the [protons.options](https://github.com/protocolbuffers/protobuf/blob/6f1d88107f268b8ebdad6690d116e74c403e366e/docs/options.md?plain=1#L490-L493) extension:
+
+```protobuf
+message MyMessage {
+  // repeatedField cannot have more than 10 entries
+  repeated uint32 repeatedField = 1 [(protons.options).limit = 10];
+
+  // stringMap cannot have more than 10 keys
+  map<string, string> stringMap = 2 [(protons.options).limit = 10];
+}
+```
+
+Or at runtime by passing objects to the `.decode` function of your message:
+
+```TypeScript
+const message = MyMessage.decode(buf, {
+  limits: {
+    repeatedField: 10,
+    stringMap: 10
+  }
+})
+```
+
+### Overriding 64 bit types
+
+By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s.
+
+Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility.
+
+It's possible to override the JavaScript type 64 bit fields will deserialize to:
+
+```protobuf
+message MyMessage {
+  repeated int64 bigintField = 1;
+  repeated int64 numberField = 2 [jstype = JS_NUMBER];
+  repeated int64 stringField = 3 [jstype = JS_STRING];
+}
+```
+
+```TypeScript
+const message = MyMessage.decode(buf)
+
+console.info(typeof message.bigintField) // bigint
+console.info(typeof message.numberField) // number
+console.info(typeof message.stringField) // string
+```
+
 ## Missing features
 
 Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.

packages/protons/bin/protons.ts

@@ -16,7 +16,6 @@ async function main (): Promise<void> {
   Examples
     $ protons ./path/to/file.proto ./path/to/other/file.proto
 `, {
-    // @ts-expect-error importMeta is missing from the types
     importMeta: import.meta,
     flags: {
       output: {

packages/protons/package.json

@@ -130,7 +130,7 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "meow": "^13.0.0",
+    "meow": "^13.1.0",
     "protobufjs-cli": "^1.0.0"
   },
   "devDependencies": {

packages/protons/src/index.ts

@@ -518,6 +518,7 @@ export namespace ${messageDef.name} {
   moduleDef.addImport('protons-runtime', 'decodeMessage')
   moduleDef.addImport('protons-runtime', 'message')
   moduleDef.addTypeImport('protons-runtime', 'Codec')
+  moduleDef.addTypeImport('protons-runtime', 'DecodeOptions')
   moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList')
 
   const interfaceFields = defineFields(fields, messageDef, moduleDef)
@@ -691,12 +692,16 @@ export interface ${messageDef.name} {
         const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type](jsTypeOverride)}`
 
         if (fieldDef.map) {
-          let limit = ''
+          moduleDef.addImport('protons-runtime', 'CodeError')
 
-          if (fieldDef.lengthLimit != null) {
-            moduleDef.addImport('protons-runtime', 'CodeError')
+          let limit = `
+              if (opts.limits?.${fieldName} != null && obj.${fieldName}.size === opts.limits.${fieldName}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
+              }
+`
 
-            limit = `
+          if (fieldDef.lengthLimit != null) {
+            limit += `
               if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
                 throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
               }
@@ -709,12 +714,16 @@ export interface ${messageDef.name} {
               break
             }`
         } else if (fieldDef.repeated) {
-          let limit = ''
+          moduleDef.addImport('protons-runtime', 'CodeError')
 
-          if (fieldDef.lengthLimit != null) {
-            moduleDef.addImport('protons-runtime', 'CodeError')
+          let limit = `
+              if (opts.limits?.${fieldName} != null && obj.${fieldName}.length === opts.limits.${fieldName}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
+              }
+`
 
-            limit = `
+          if (fieldDef.lengthLimit != null) {
+            limit += `
               if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
                 throw new CodeError('decode error - repeated field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
               }
@@ -750,7 +759,7 @@ ${encodeFields === '' ? '' : `${encodeFields}\n`}
         if (opts.lengthDelimited !== false) {
           w.ldelim()
         }
-      }, (reader, length) => {
+      }, (reader, length, opts = {}) => {
         const obj: any = {${createDefaultObject(fields, messageDef, moduleDef)}}
 
         const end = length == null ? reader.len : reader.pos + length
@@ -777,8 +786,8 @@ ${encodeFields === '' ? '' : `${encodeFields}\n`}
     return encodeMessage(obj, ${messageDef.name}.codec())
   }
 
-  export const decode = (buf: Uint8Array | Uint8ArrayList): ${messageDef.name} => {
-    return decodeMessage(buf, ${messageDef.name}.codec())
+  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<${messageDef.name}>): ${messageDef.name} => {
+    return decodeMessage(buf, ${messageDef.name}.codec(), opts)
   }`
 
   return `

packages/protons/test/fixtures/basic.ts

@@ -4,7 +4,7 @@
 /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
 /* eslint-disable @typescript-eslint/no-empty-interface */
 
-import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime'
+import { type Codec, decodeMessage, type DecodeOptions, encodeMessage, message } from 'protons-runtime'
 import type { Uint8ArrayList } from 'uint8arraylist'
 
 export interface Basic {
@@ -35,7 +35,7 @@ export namespace Basic {
         if (opts.lengthDelimited !== false) {
           w.ldelim()
         }
-      }, (reader, length) => {
+      }, (reader, length, opts = {}) => {
         const obj: any = {
           num: 0
         }
@@ -72,8 +72,8 @@ export namespace Basic {
     return encodeMessage(obj, Basic.codec())
   }
 
-  export const decode = (buf: Uint8Array | Uint8ArrayList): Basic => {
-    return decodeMessage(buf, Basic.codec())
+  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<Basic>): Basic => {
+    return decodeMessage(buf, Basic.codec(), opts)
   }
 }
 
@@ -92,7 +92,7 @@ export namespace Empty {
         if (opts.lengthDelimited !== false) {
           w.ldelim()
         }
-      }, (reader, length) => {
+      }, (reader, length, opts = {}) => {
         const obj: any = {}
 
         const end = length == null ? reader.len : reader.pos + length
@@ -119,7 +119,7 @@ export namespace Empty {
     return encodeMessage(obj, Empty.codec())
   }
 
-  export const decode = (buf: Uint8Array | Uint8ArrayList): Empty => {
-    return decodeMessage(buf, Empty.codec())
+  export const decode = (buf: Uint8Array | Uint8ArrayList, opts?: DecodeOptions<Empty>): Empty => {
+    return decodeMessage(buf, Empty.codec(), opts)
   }
 }