feat: add custom protons options for limiting list/map sizes (#120)

Adds the capability to define protons-specific custom options that control decoding behaviour initially around limiting the sizes of maps and lists.

```protobuf
// import the options definition - it will work without this but some
// code editor plugins may report an error if the def can't be found
import "protons.proto";

message Message {
  // define the size limit - here more than 10 repeated items will
  // cause decoding to fail
  repeated string repeatedField = 1 [(protons.options).limit = 10];
}
```

The definition is shipped with the `protons-runtime` module. There is a [pending PR](protocolbuffers/protobuf#14505) to reserve the `1186` field ID. This should be merged first and/or the field ID updated here if it changes due to that PR.

Fixes #113

packages/protons-runtime/package.json

@@ -14,6 +14,7 @@
   "type": "module",
   "types": "./dist/src/index.d.ts",
   "files": [
+    "protons.proto",
     "src",
     "dist",
     "!dist/test",

packages/protons-runtime/protons.proto

@@ -0,0 +1,15 @@
+syntax = "proto2";
+
+import "google/protobuf/descriptor.proto";
+
+package protons;
+
+message ProtonsOptions {
+  // limit the number of repeated fields or map entries that will be decoded
+  optional int32 limit = 1;
+}
+
+// custom options available for use by protons
+extend google.protobuf.FieldOptions {
+  optional ProtonsOptions options = 1186;
+}

packages/protons-runtime/src/index.ts

@@ -326,3 +326,13 @@ export interface Reader {
    */
   sfixed64String(): string
 }
+
+export class CodeError extends Error {
+  public code: string
+
+  constructor (message: string, code: string, options?: ErrorOptions) {
+    super(message, options)
+
+    this.code = code
+  }
+}

packages/protons/bin/protons.ts

@@ -11,6 +11,7 @@ async function main (): Promise<void> {
   Options
     --output, -o Path to a directory to write transpiled typescript files into
     --strict, -s Causes parsing warnings to become errors
+    --path, -p Adds a directory to the include path
 
   Examples
     $ protons ./path/to/file.proto ./path/to/other/file.proto
@@ -25,6 +26,11 @@ async function main (): Promise<void> {
       strict: {
         type: 'boolean',
         shortFlag: 's'
+      },
+      path: {
+        type: 'string',
+        shortFlag: 'p',
+        isMultiple: true
       }
     }
   })

packages/protons/src/index.ts

@@ -430,6 +430,7 @@ interface FieldDef {
   rule: string
   optional: boolean
   repeated: boolean
+  lengthLimit?: number
   message: boolean
   enum: boolean
   map: boolean
@@ -685,13 +686,37 @@ export interface ${messageDef.name} {
         const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type](jsTypeOverride)}`
 
         if (fieldDef.map) {
-          return `case ${fieldDef.id}: {
+          let limit = ''
+
+          if (fieldDef.lengthLimit != null) {
+            moduleDef.imports.add('CodeError')
+
+            limit = `
+              if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
+                throw new CodeError('decode error - map field "${fieldName}" had too many elements', 'ERR_MAX_SIZE')
+              }
+`
+          }
+
+          return `case ${fieldDef.id}: {${limit}
               const entry = ${parseValue}
               obj.${fieldName}.set(entry.key, entry.value)
               break
             }`
         } else if (fieldDef.repeated) {
-          return `case ${fieldDef.id}: {
+          let limit = ''
+
+          if (fieldDef.lengthLimit != null) {
+            moduleDef.imports.add('CodeError')
+
+            limit = `
+              if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
+                throw new CodeError('decode error - repeated field "${fieldName}" had too many elements', 'ERR_MAX_LENGTH')
+              }
+`
+          }
+
+          return `case ${fieldDef.id}: {${limit}
               obj.${fieldName}.push(${parseValue})
               break
             }`
@@ -801,6 +826,7 @@ function defineModule (def: ClassDef, flags: Flags): ModuleDef {
           fieldDef.repeated = fieldDef.rule === 'repeated'
           fieldDef.optional = !fieldDef.repeated && fieldDef.options?.proto3_optional === true
           fieldDef.map = fieldDef.keyType != null
+          fieldDef.lengthLimit = fieldDef.options?.['(protons.options).limit']
           fieldDef.proto2Required = false
 
           if (fieldDef.rule === 'required') {
@@ -876,11 +902,20 @@ interface Flags {
    * If true, warnings will be thrown as errors
    */
   strict?: boolean
+
+  /**
+   * A list of directories to add to the include path
+   */
+  path?: string[]
 }
 
 export async function generate (source: string, flags: Flags): Promise<void> {
   // convert .protobuf to .json
-  const json = await promisify(pbjs)(['-t', 'json', source])
+  const json = await promisify(pbjs)([
+    '-t', 'json',
+    ...(flags.path ?? []).map(p => ['--path', path.isAbsolute(p) ? p : path.resolve(process.cwd(), p)]).flat(),
+    source
+  ])
 
   if (json == null) {
     throw new Error(`Could not convert ${source} to intermediate JSON format`)

packages/protons/test/fixtures/protons-options.proto

@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+import "protons.proto";
+
+message MessageWithSizeLimitedRepeatedField {
+  repeated string repeatedField = 1 [(protons.options).limit = 1];
+}
+
+message MessageWithSizeLimitedMap {
+  map<string, string> mapField = 1 [(protons.options).limit = 1];
+}

packages/protons/test/fixtures/protons-options.ts

@@ -0,0 +1,213 @@
+/* eslint-disable import/export */
+/* eslint-disable complexity */
+/* eslint-disable @typescript-eslint/no-namespace */
+/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
+/* eslint-disable @typescript-eslint/no-empty-interface */
+
+import { encodeMessage, decodeMessage, message, CodeError } from 'protons-runtime'
+import type { Codec } from 'protons-runtime'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+export interface MessageWithSizeLimitedRepeatedField {
+  repeatedField: string[]
+}
+
+export namespace MessageWithSizeLimitedRepeatedField {
+  let _codec: Codec<MessageWithSizeLimitedRepeatedField>
+
+  export const codec = (): Codec<MessageWithSizeLimitedRepeatedField> => {
+    if (_codec == null) {
+      _codec = message<MessageWithSizeLimitedRepeatedField>((obj, w, opts = {}) => {
+        if (opts.lengthDelimited !== false) {
+          w.fork()
+        }
+
+        if (obj.repeatedField != null) {
+          for (const value of obj.repeatedField) {
+            w.uint32(10)
+            w.string(value)
+          }
+        }
+
+        if (opts.lengthDelimited !== false) {
+          w.ldelim()
+        }
+      }, (reader, length) => {
+        const obj: any = {
+          repeatedField: []
+        }
+
+        const end = length == null ? reader.len : reader.pos + length
+
+        while (reader.pos < end) {
+          const tag = reader.uint32()
+
+          switch (tag >>> 3) {
+            case 1: {
+              if (obj.repeatedField.length === 1) {
+                throw new CodeError('decode error - repeated field "repeatedField" had too many elements', 'ERR_MAX_LENGTH')
+              }
+
+              obj.repeatedField.push(reader.string())
+              break
+            }
+            default: {
+              reader.skipType(tag & 7)
+              break
+            }
+          }
+        }
+
+        return obj
+      })
+    }
+
+    return _codec
+  }
+
+  export const encode = (obj: Partial<MessageWithSizeLimitedRepeatedField>): Uint8Array => {
+    return encodeMessage(obj, MessageWithSizeLimitedRepeatedField.codec())
+  }
+
+  export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedRepeatedField => {
+    return decodeMessage(buf, MessageWithSizeLimitedRepeatedField.codec())
+  }
+}
+
+export interface MessageWithSizeLimitedMap {
+  mapField: Map<string, string>
+}
+
+export namespace MessageWithSizeLimitedMap {
+  export interface MessageWithSizeLimitedMap$mapFieldEntry {
+    key: string
+    value: string
+  }
+
+  export namespace MessageWithSizeLimitedMap$mapFieldEntry {
+    let _codec: Codec<MessageWithSizeLimitedMap$mapFieldEntry>
+
+    export const codec = (): Codec<MessageWithSizeLimitedMap$mapFieldEntry> => {
+      if (_codec == null) {
+        _codec = message<MessageWithSizeLimitedMap$mapFieldEntry>((obj, w, opts = {}) => {
+          if (opts.lengthDelimited !== false) {
+            w.fork()
+          }
+
+          if ((obj.key != null && obj.key !== '')) {
+            w.uint32(10)
+            w.string(obj.key)
+          }
+
+          if ((obj.value != null && obj.value !== '')) {
+            w.uint32(18)
+            w.string(obj.value)
+          }
+
+          if (opts.lengthDelimited !== false) {
+            w.ldelim()
+          }
+        }, (reader, length) => {
+          const obj: any = {
+            key: '',
+            value: ''
+          }
+
+          const end = length == null ? reader.len : reader.pos + length
+
+          while (reader.pos < end) {
+            const tag = reader.uint32()
+
+            switch (tag >>> 3) {
+              case 1: {
+                obj.key = reader.string()
+                break
+              }
+              case 2: {
+                obj.value = reader.string()
+                break
+              }
+              default: {
+                reader.skipType(tag & 7)
+                break
+              }
+            }
+          }
+
+          return obj
+        })
+      }
+
+      return _codec
+    }
+
+    export const encode = (obj: Partial<MessageWithSizeLimitedMap$mapFieldEntry>): Uint8Array => {
+      return encodeMessage(obj, MessageWithSizeLimitedMap$mapFieldEntry.codec())
+    }
+
+    export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedMap$mapFieldEntry => {
+      return decodeMessage(buf, MessageWithSizeLimitedMap$mapFieldEntry.codec())
+    }
+  }
+
+  let _codec: Codec<MessageWithSizeLimitedMap>
+
+  export const codec = (): Codec<MessageWithSizeLimitedMap> => {
+    if (_codec == null) {
+      _codec = message<MessageWithSizeLimitedMap>((obj, w, opts = {}) => {
+        if (opts.lengthDelimited !== false) {
+          w.fork()
+        }
+
+        if (obj.mapField != null && obj.mapField.size !== 0) {
+          for (const [key, value] of obj.mapField.entries()) {
+            w.uint32(10)
+            MessageWithSizeLimitedMap.MessageWithSizeLimitedMap$mapFieldEntry.codec().encode({ key, value }, w)
+          }
+        }
+
+        if (opts.lengthDelimited !== false) {
+          w.ldelim()
+        }
+      }, (reader, length) => {
+        const obj: any = {
+          mapField: new Map<string, string>()
+        }
+
+        const end = length == null ? reader.len : reader.pos + length
+
+        while (reader.pos < end) {
+          const tag = reader.uint32()
+
+          switch (tag >>> 3) {
+            case 1: {
+              if (obj.mapField.size === 1) {
+                throw new CodeError('decode error - map field "mapField" had too many elements', 'ERR_MAX_SIZE')
+              }
+
+              const entry = MessageWithSizeLimitedMap.MessageWithSizeLimitedMap$mapFieldEntry.codec().decode(reader, reader.uint32())
+              obj.mapField.set(entry.key, entry.value)
+              break
+            }
+            default: {
+              reader.skipType(tag & 7)
+              break
+            }
+          }
+        }
+
+        return obj
+      })
+    }
+
+    return _codec
+  }
+
+  export const encode = (obj: Partial<MessageWithSizeLimitedMap>): Uint8Array => {
+    return encodeMessage(obj, MessageWithSizeLimitedMap.codec())
+  }
+
+  export const decode = (buf: Uint8Array | Uint8ArrayList): MessageWithSizeLimitedMap => {
+    return decodeMessage(buf, MessageWithSizeLimitedMap.codec())
+  }
+}

packages/protons/test/protons-options.spec.ts

@@ -0,0 +1,28 @@
+/* eslint-env mocha */
+
+import { expect } from 'aegir/chai'
+import { MessageWithSizeLimitedMap, MessageWithSizeLimitedRepeatedField } from './fixtures/protons-options.js'
+
+describe('protons options', () => {
+  it('should not decode message with map that is too big', () => {
+    const obj: MessageWithSizeLimitedMap = {
+      mapField: new Map<string, string>([['one', 'two'], ['three', 'four']])
+    }
+
+    const buf = MessageWithSizeLimitedMap.encode(obj)
+
+    expect(() => MessageWithSizeLimitedMap.decode(buf))
+      .to.throw().with.property('code', 'ERR_MAX_SIZE')
+  })
+
+  it('should not decode message with list that is too big', () => {
+    const obj: MessageWithSizeLimitedRepeatedField = {
+      repeatedField: ['0', '1']
+    }
+
+    const buf = MessageWithSizeLimitedRepeatedField.encode(obj)
+
+    expect(() => MessageWithSizeLimitedRepeatedField.decode(buf))
+      .to.throw().with.property('code', 'ERR_MAX_LENGTH')
+  })
+})