From 3758e63803958b6397d95a48caa2341d353bde18 Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Mon, 23 Oct 2023 14:02:42 +0100
Subject: [PATCH 1/3] feat: add custom protons options for limiting list/map
sizes
Adds the capability to define protons-specific custom options that
control decoding behaviour initially around limiting the sizes of
maps and lists.
```protobuf
// import the options definition - it will work without this but some
// code editor plugins may report an error if the def can't be found
import "protons-rutime/options.proto";
message MessageWithSizeLimitedRepeatedField {
// define the size limit - here more than 10 repeated items will
// cause decoding to fail
repeated string repeatedField = 1 [(protons.limit) = 10];
}
```
The defintion is shipped with the `protons-runtime` module. There is
a [pending PR](https://github.com/protocolbuffers/protobuf/pull/14505)
to reserve the `1186` field ID. This should be merged first and/or
the field ID updated here if it changes due to that PR.
Fixes #113
---
packages/protons-runtime/options.proto | 15 ++
packages/protons-runtime/package.json | 1 +
packages/protons-runtime/src/index.ts | 10 +
packages/protons/src/index.ts | 30 ++-
.../test/fixtures/protons-options.proto | 11 +
.../protons/test/fixtures/protons-options.ts | 213 ++++++++++++++++++
packages/protons/test/protons-options.spec.ts | 28 +++
7 files changed, 306 insertions(+), 2 deletions(-)
create mode 100644 packages/protons-runtime/options.proto
create mode 100644 packages/protons/test/fixtures/protons-options.proto
create mode 100644 packages/protons/test/fixtures/protons-options.ts
create mode 100644 packages/protons/test/protons-options.spec.ts
diff --git a/packages/protons-runtime/options.proto b/packages/protons-runtime/options.proto
new file mode 100644
index 0000000..eafa1b8
--- /dev/null
+++ b/packages/protons-runtime/options.proto
@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+import "google/protobuf/descriptor.proto";
+
+package protons;
+
+message ProtonsOptions {
+ // limit the number of repeated fields or map entries that will be decoded
+ optional int32 limit = 1;
+}
+
+// custom options available for use by protons
+extend google.protobuf.FieldOptions {
+ optional ProtonsOptions protons_options = 1186;
+}
diff --git a/packages/protons-runtime/package.json b/packages/protons-runtime/package.json
index 1e3d8a2..fb3b4a2 100644
--- a/packages/protons-runtime/package.json
+++ b/packages/protons-runtime/package.json
@@ -14,6 +14,7 @@
"type": "module",
"types": "./dist/src/index.d.ts",
"files": [
+ "options.proto",
"src",
"dist",
"!dist/test",
diff --git a/packages/protons-runtime/src/index.ts b/packages/protons-runtime/src/index.ts
index d22fa35..e0e53d5 100644
--- a/packages/protons-runtime/src/index.ts
+++ b/packages/protons-runtime/src/index.ts
@@ -326,3 +326,13 @@ export interface Reader {
*/
sfixed64String(): string
}
+
+export class CodeError extends Error {
+ public code: string
+
+ constructor (message: string, code: string, options?: ErrorOptions) {
+ super(message, options)
+
+ this.code = code
+ }
+}
diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts
index 6476832..843d91c 100644
--- a/packages/protons/src/index.ts
+++ b/packages/protons/src/index.ts
@@ -430,6 +430,7 @@ interface FieldDef {
rule: string
optional: boolean
repeated: boolean
+ lengthLimit?: number
message: boolean
enum: boolean
map: boolean
@@ -685,13 +686,37 @@ export interface ${messageDef.name} {
const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type](jsTypeOverride)}`
if (fieldDef.map) {
- return `case ${fieldDef.id}: {
+ let limit = ''
+
+ if (fieldDef.lengthLimit != null) {
+ moduleDef.imports.add('CodeError')
+
+ limit = `
+ if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
+ throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
+ }
+`
+ }
+
+ return `case ${fieldDef.id}: {${limit}
const entry = ${parseValue}
obj.${fieldName}.set(entry.key, entry.value)
break
}`
} else if (fieldDef.repeated) {
- return `case ${fieldDef.id}: {
+ let limit = ''
+
+ if (fieldDef.lengthLimit != null) {
+ moduleDef.imports.add('CodeError')
+
+ limit = `
+ if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
+ throw new CodeError('decode error - repeated field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
+ }
+`
+ }
+
+ return `case ${fieldDef.id}: {${limit}
obj.${fieldName}.push(${parseValue})
break
}`
@@ -801,6 +826,7 @@ function defineModule (def: ClassDef, flags: Flags): ModuleDef {
fieldDef.repeated = fieldDef.rule === 'repeated'
fieldDef.optional = !fieldDef.repeated && fieldDef.options?.proto3_optional === true
fieldDef.map = fieldDef.keyType != null
+ fieldDef.lengthLimit = fieldDef.options?.['(protons.limit)']
fieldDef.proto2Required = false
if (fieldDef.rule === 'required') {
diff --git a/packages/protons/test/fixtures/protons-options.proto b/packages/protons/test/fixtures/protons-options.proto
new file mode 100644
index 0000000..45dbcf4
--- /dev/null
+++ b/packages/protons/test/fixtures/protons-options.proto
@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+import "protons-rutime/options.proto";
+
+message MessageWithSizeLimitedRepeatedField {
+ repeated string repeatedField = 1 [(protons.limit) = 1];
+}
+
+message MessageWithSizeLimitedMap {
+ map<string, string> mapField = 1 [(protons.limit) = 1];
+}
diff --git a/packages/protons/test/fixtures/protons-options.ts b/packages/protons/test/fixtures/protons-options.ts
new file mode 100644
index 0000000..0ad6bcc
--- /dev/null
+++ b/packages/protons/test/fixtures/protons-options.ts
@@ -0,0 +1,213 @@
+/* eslint-disable import/export */
+/* eslint-disable complexity */
+/* eslint-disable @typescript-eslint/no-namespace */
+/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
+/* eslint-disable @typescript-eslint/no-empty-interface */
+
+import { encodeMessage, decodeMessage, message, CodeError } from 'protons-runtime'
+import type { Codec } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+export interface MessageWithSizeLimitedRepeatedField {
+ repeatedField: string[]
+}
+
+export namespace MessageWithSizeLimitedRepeatedField {
+ let _codec: Codec<MessageWithSizeLimitedRepeatedField>
+
+ export const codec = (): Codec<MessageWithSizeLimitedRepeatedField> => {
+ if (_codec == null) {
+ _codec = message<MessageWithSizeLimitedRepeatedField>((obj, w, opts = {}) => {
+ if (opts.lengthDelimited !== false) {
+ w.fork()
+ }
+
+ if (obj.repeatedField != null) {
+ for (const value of obj.repeatedField) {
+ w.uint32(10)
+ w.string(value)
+ }
+ }
+
+ if (opts.lengthDelimited !== false) {
+ w.ldelim()
+ }
+ }, (reader, length) => {
+ const obj: any = {
+ repeatedField: []
+ }
+
+ const end = length == null ? reader.len : reader.pos + length
+
+ while (reader.pos < end) {
+ const tag = reader.uint32()
+
+ switch (tag >>> 3) {
+ case 1: {
+ if (obj.repeatedField.length === 1) {
+ throw new CodeError('decode error - repeated field "repeatedField" had too many elements', 'ERR_MAX_LENGTH')
+ }
+
+ obj.repeatedField.push(reader.string())
+ break
+ }
+ default: {
+ reader.skipType(tag & 7)
+ break
+ }
+ }
+ }
+
+ return obj
+ })
+ }
+
+ return _codec
+ }
+
+ export const encode = (obj: Partial<MessageWithSizeLimitedRepeatedField>): Uint8Array => {
+ return encodeMessage(obj, MessageWithSizeLimitedRepeatedField.codec())
+ }
+
+ export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedRepeatedField => {
+ return decodeMessage(buf, MessageWithSizeLimitedRepeatedField.codec())
+ }
+}
+
+export interface MessageWithSizeLimitedMap {
+ mapField: Map<string, string>
+}
+
+export namespace MessageWithSizeLimitedMap {
+ export interface MessageWithSizeLimitedMap$mapFieldEntry {
+ key: string
+ value: string
+ }
+
+ export namespace MessageWithSizeLimitedMap$mapFieldEntry {
+ let _codec: Codec<MessageWithSizeLimitedMap$mapFieldEntry>
+
+ export const codec = (): Codec<MessageWithSizeLimitedMap$mapFieldEntry> => {
+ if (_codec == null) {
+ _codec = message<MessageWithSizeLimitedMap$mapFieldEntry>((obj, w, opts = {}) => {
+ if (opts.lengthDelimited !== false) {
+ w.fork()
+ }
+
+ if ((obj.key != null && obj.key !== '')) {
+ w.uint32(10)
+ w.string(obj.key)
+ }
+
+ if ((obj.value != null && obj.value !== '')) {
+ w.uint32(18)
+ w.string(obj.value)
+ }
+
+ if (opts.lengthDelimited !== false) {
+ w.ldelim()
+ }
+ }, (reader, length) => {
+ const obj: any = {
+ key: '',
+ value: ''
+ }
+
+ const end = length == null ? reader.len : reader.pos + length
+
+ while (reader.pos < end) {
+ const tag = reader.uint32()
+
+ switch (tag >>> 3) {
+ case 1: {
+ obj.key = reader.string()
+ break
+ }
+ case 2: {
+ obj.value = reader.string()
+ break
+ }
+ default: {
+ reader.skipType(tag & 7)
+ break
+ }
+ }
+ }
+
+ return obj
+ })
+ }
+
+ return _codec
+ }
+
+ export const encode = (obj: Partial<MessageWithSizeLimitedMap$mapFieldEntry>): Uint8Array => {
+ return encodeMessage(obj, MessageWithSizeLimitedMap$mapFieldEntry.codec())
+ }
+
+ export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedMap$mapFieldEntry => {
+ return decodeMessage(buf, MessageWithSizeLimitedMap$mapFieldEntry.codec())
+ }
+ }
+
+ let _codec: Codec<MessageWithSizeLimitedMap>
+
+ export const codec = (): Codec<MessageWithSizeLimitedMap> => {
+ if (_codec == null) {
+ _codec = message<MessageWithSizeLimitedMap>((obj, w, opts = {}) => {
+ if (opts.lengthDelimited !== false) {
+ w.fork()
+ }
+
+ if (obj.mapField != null && obj.mapField.size !== 0) {
+ for (const [key, value] of obj.mapField.entries()) {
+ w.uint32(10)
+ MessageWithSizeLimitedMap.MessageWithSizeLimitedMap$mapFieldEntry.codec().encode({ key, value }, w)
+ }
+ }
+
+ if (opts.lengthDelimited !== false) {
+ w.ldelim()
+ }
+ }, (reader, length) => {
+ const obj: any = {
+ mapField: new Map<string, string>()
+ }
+
+ const end = length == null ? reader.len : reader.pos + length
+
+ while (reader.pos < end) {
+ const tag = reader.uint32()
+
+ switch (tag >>> 3) {
+ case 1: {
+ if (obj.mapField.size === 1) {
+ throw new CodeError('decode error - map field "mapField" had too many elements', 'ERR_MAX_SIZE')
+ }
+
+ const entry = MessageWithSizeLimitedMap.MessageWithSizeLimitedMap$mapFieldEntry.codec().decode(reader, reader.uint32())
+ obj.mapField.set(entry.key, entry.value)
+ break
+ }
+ default: {
+ reader.skipType(tag & 7)
+ break
+ }
+ }
+ }
+
+ return obj
+ })
+ }
+
+ return _codec
+ }
+
+ export const encode = (obj: Partial<MessageWithSizeLimitedMap>): Uint8Array => {
+ return encodeMessage(obj, MessageWithSizeLimitedMap.codec())
+ }
+
+ export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedMap => {
+ return decodeMessage(buf, MessageWithSizeLimitedMap.codec())
+ }
+}
diff --git a/packages/protons/test/protons-options.spec.ts b/packages/protons/test/protons-options.spec.ts
new file mode 100644
index 0000000..d760bdc
--- /dev/null
+++ b/packages/protons/test/protons-options.spec.ts
@@ -0,0 +1,28 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import { MessageWithSizeLimitedMap, MessageWithSizeLimitedRepeatedField } from './fixtures/protons-options.js'
+
+describe('protons options', () => {
+ it('should not decode message with map that is too big', () => {
+ const obj: MessageWithSizeLimitedMap = {
+ mapField: new Map<string, string>([['one', 'two'], ['three', 'four']])
+ }
+
+ const buf = MessageWithSizeLimitedMap.encode(obj)
+
+ expect(() => MessageWithSizeLimitedMap.decode(buf))
+ .to.throw().with.property('code', 'ERR_MAX_SIZE')
+ })
+
+ it('should not decode message with list that is too big', () => {
+ const obj: MessageWithSizeLimitedRepeatedField = {
+ repeatedField: ['0', '1']
+ }
+
+ const buf = MessageWithSizeLimitedRepeatedField.encode(obj)
+
+ expect(() => MessageWithSizeLimitedRepeatedField.decode(buf))
+ .to.throw().with.property('code', 'ERR_MAX_LENGTH')
+ })
+})
From d0e3513872d658b4a0bc4530e14b2b4cb5df1dcc Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Mon, 23 Oct 2023 17:50:31 +0100
Subject: [PATCH 2/3] chore: update option name
---
packages/protons-runtime/options.proto | 4 ++--
packages/protons/src/index.ts | 2 +-
packages/protons/test/fixtures/protons-options.proto | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/packages/protons-runtime/options.proto b/packages/protons-runtime/options.proto
index eafa1b8..6e16410 100644
--- a/packages/protons-runtime/options.proto
+++ b/packages/protons-runtime/options.proto
@@ -1,4 +1,4 @@
-syntax = "proto3";
+syntax = "proto2";
import "google/protobuf/descriptor.proto";
@@ -11,5 +11,5 @@ message ProtonsOptions {
// custom options available for use by protons
extend google.protobuf.FieldOptions {
- optional ProtonsOptions protons_options = 1186;
+ optional ProtonsOptions options = 1186;
}
diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts
index 843d91c..1357be9 100644
--- a/packages/protons/src/index.ts
+++ b/packages/protons/src/index.ts
@@ -826,7 +826,7 @@ function defineModule (def: ClassDef, flags: Flags): ModuleDef {
fieldDef.repeated = fieldDef.rule === 'repeated'
fieldDef.optional = !fieldDef.repeated && fieldDef.options?.proto3_optional === true
fieldDef.map = fieldDef.keyType != null
- fieldDef.lengthLimit = fieldDef.options?.['(protons.limit)']
+ fieldDef.lengthLimit = fieldDef.options?.['(protons.options).limit']
fieldDef.proto2Required = false
if (fieldDef.rule === 'required') {
diff --git a/packages/protons/test/fixtures/protons-options.proto b/packages/protons/test/fixtures/protons-options.proto
index 45dbcf4..a3b3d93 100644
--- a/packages/protons/test/fixtures/protons-options.proto
+++ b/packages/protons/test/fixtures/protons-options.proto
@@ -3,9 +3,9 @@ syntax = "proto3";
import "protons-rutime/options.proto";
message MessageWithSizeLimitedRepeatedField {
- repeated string repeatedField = 1 [(protons.limit) = 1];
+ repeated string repeatedField = 1 [(protons.options).limit = 1];
}
message MessageWithSizeLimitedMap {
- map<string, string> mapField = 1 [(protons.limit) = 1];
+ map<string, string> mapField = 1 [(protons.options).limit = 1];
}
From c2d122e0a01b5898ac132fbf6051436820dde196 Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Mon, 23 Oct 2023 18:37:38 +0100
Subject: [PATCH 3/3] chore: fix options proto
---
packages/protons-runtime/package.json | 2 +-
.../protons-runtime/{options.proto => protons.proto} | 0
packages/protons/bin/protons.ts | 6 ++++++
packages/protons/src/index.ts | 11 ++++++++++-
packages/protons/test/fixtures/protons-options.proto | 2 +-
5 files changed, 18 insertions(+), 3 deletions(-)
rename packages/protons-runtime/{options.proto => protons.proto} (100%)
diff --git a/packages/protons-runtime/package.json b/packages/protons-runtime/package.json
index fb3b4a2..6e23cae 100644
--- a/packages/protons-runtime/package.json
+++ b/packages/protons-runtime/package.json
@@ -14,7 +14,7 @@
"type": "module",
"types": "./dist/src/index.d.ts",
"files": [
- "options.proto",
+ "protons.proto",
"src",
"dist",
"!dist/test",
diff --git a/packages/protons-runtime/options.proto b/packages/protons-runtime/protons.proto
similarity index 100%
rename from packages/protons-runtime/options.proto
rename to packages/protons-runtime/protons.proto
diff --git a/packages/protons/bin/protons.ts b/packages/protons/bin/protons.ts
index ebb215b..4565ffa 100644
--- a/packages/protons/bin/protons.ts
+++ b/packages/protons/bin/protons.ts
@@ -11,6 +11,7 @@ async function main (): Promise<void> {
Options
--output, -o Path to a directory to write transpiled typescript files into
--strict, -s Causes parsing warnings to become errors
+ --path, -p Adds a directory to the include path
Examples
$ protons ./path/to/file.proto ./path/to/other/file.proto
@@ -25,6 +26,11 @@ async function main (): Promise<void> {
strict: {
type: 'boolean',
shortFlag: 's'
+ },
+ path: {
+ type: 'string',
+ shortFlag: 'p',
+ isMultiple: true
}
}
})
diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts
index 1357be9..fa640e7 100644
--- a/packages/protons/src/index.ts
+++ b/packages/protons/src/index.ts
@@ -902,11 +902,20 @@ interface Flags {
* If true, warnings will be thrown as errors
*/
strict?: boolean
+
+ /**
+ * A list of directories to add to the include path
+ */
+ path?: string[]
}
export async function generate (source: string, flags: Flags): Promise<void> {
// convert .protobuf to .json
- const json = await promisify(pbjs)(['-t', 'json', source])
+ const json = await promisify(pbjs)([
+ '-t', 'json',
+ ...(flags.path ?? []).map(p => ['--path', path.isAbsolute(p) ? p : path.resolve(process.cwd(), p)]).flat(),
+ source
+ ])
if (json == null) {
throw new Error(`Could not convert ${source} to intermediate JSON format`)
diff --git a/packages/protons/test/fixtures/protons-options.proto b/packages/protons/test/fixtures/protons-options.proto
index a3b3d93..21bfbe8 100644
--- a/packages/protons/test/fixtures/protons-options.proto
+++ b/packages/protons/test/fixtures/protons-options.proto
@@ -1,6 +1,6 @@
syntax = "proto3";
-import "protons-rutime/options.proto";
+import "protons.proto";
message MessageWithSizeLimitedRepeatedField {
repeated string repeatedField = 1 [(protons.options).limit = 1];