Skip to content

Commit

Permalink
fix: use uint8arrays alloc for new buffers (#123)
Browse files Browse the repository at this point in the history 
Will return `Buffer`s under Node.js which have greater performance than `Uint8Array`s.
  • Loading branch information
@achingbrain
achingbrain authored Nov 22, 2023
1 parent 23073eb commit d1bfc94
Show file tree
Hide file tree
Showing 16 changed files with 163 additions and 85 deletions.
3 changes: 2 additions & 1 deletion packages/protons/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@
"pbjs": "^0.0.14",
"protobufjs": "^7.0.0",
"protons-runtime": "^5.0.0",
"uint8arraylist": "^2.4.3"
"uint8arraylist": "^2.4.3",
"uint8arrays": "^4.0.6"
}
}
159 changes: 122 additions & 37 deletions packages/protons/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ const decoderGenerators: Record<string, (jsTypeOverride?: 'number' | 'string') =

const defaultValueGenerators: Record<string, () => string> = {
bool: () => 'false',
bytes: () => 'new Uint8Array(0)',
bytes: () => 'uint8ArrayAlloc(0)',
double: () => '0',
fixed32: () => '0',
fixed64: () => '0n',
Expand Down Expand Up @@ -320,6 +320,10 @@ function createDefaultObject (fields: Record<string, FieldDef>, messageDef: Mess
defaultValueGenerator = defaultValueGeneratorsJsTypeOverrides[jsTypeOverride]
}

if (type === 'bytes') {
moduleDef.addImport('uint8arrays/alloc', 'alloc', 'uint8ArrayAlloc')
}

defaultValue = defaultValueGenerator()
} else {
const def = findDef(fieldDef.type, messageDef, moduleDef)
Expand Down Expand Up @@ -457,7 +461,7 @@ function defineFields (fields: Record<string, FieldDef>, messageDef: MessageDef,

function compileMessage (messageDef: MessageDef, moduleDef: ModuleDef, flags?: Flags): string {
if (isEnumDef(messageDef)) {
moduleDef.imports.add('enumeration')
moduleDef.addImport('protons-runtime', 'enumeration')

// check that the enum def values start from 0
if (Object.values(messageDef.values)[0] !== 0) {
Expand Down Expand Up @@ -510,10 +514,11 @@ export namespace ${messageDef.name} {
const fields = messageDef.fields ?? {}

// import relevant modules
moduleDef.imports.add('encodeMessage')
moduleDef.imports.add('decodeMessage')
moduleDef.imports.add('message')
moduleDef.importedTypes.add('Codec')
moduleDef.addImport('protons-runtime', 'encodeMessage')
moduleDef.addImport('protons-runtime', 'decodeMessage')
moduleDef.addImport('protons-runtime', 'message')
moduleDef.addTypeImport('protons-runtime', 'Codec')
moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList')

const interfaceFields = defineFields(fields, messageDef, moduleDef)
.join('\n ')
Expand Down Expand Up @@ -544,10 +549,10 @@ export interface ${messageDef.name} {

if (codec == null) {
if (fieldDef.enum) {
moduleDef.imports.add('enumeration')
moduleDef.addImport('protons-runtime', 'enumeration')
type = 'enum'
} else {
moduleDef.imports.add('message')
moduleDef.addImport('protons-runtime', 'message')
type = 'message'
}

Expand Down Expand Up @@ -669,10 +674,10 @@ export interface ${messageDef.name} {

if (codec == null) {
if (fieldDef.enum) {
moduleDef.imports.add('enumeration')
moduleDef.addImport('protons-runtime', 'enumeration')
type = 'enum'
} else {
moduleDef.imports.add('message')
moduleDef.addImport('protons-runtime', 'message')
type = 'message'
}

Expand All @@ -689,7 +694,7 @@ export interface ${messageDef.name} {
let limit = ''

if (fieldDef.lengthLimit != null) {
moduleDef.imports.add('CodeError')
moduleDef.addImport('protons-runtime', 'CodeError')

limit = `
if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
Expand All @@ -707,7 +712,7 @@ export interface ${messageDef.name} {
let limit = ''

if (fieldDef.lengthLimit != null) {
moduleDef.imports.add('CodeError')
moduleDef.addImport('protons-runtime', 'CodeError')

limit = `
if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
Expand Down Expand Up @@ -785,23 +790,83 @@ export namespace ${messageDef.name} {
`.trimStart()
}

interface ModuleDef {
imports: Set<string>
importedTypes: Set<string>
interface Import {
symbol: string
alias?: string
type: boolean
}

class ModuleDef {
imports: Map<string, Import[]>
types: Set<string>
compiled: string[]
globals: Record<string, ClassDef>
}

function defineModule (def: ClassDef, flags: Flags): ModuleDef {
const moduleDef: ModuleDef = {
imports: new Set(),
importedTypes: new Set(),
types: new Set(),
compiled: [],
globals: {}
constructor () {
this.imports = new Map()
this.types = new Set()
this.compiled = []
this.globals = {}
}

addImport (module: string, symbol: string, alias?: string): void {
const defs = this._findDefs(module)

for (const def of defs) {
// check if we already have a definition for this symbol
if (def.symbol === symbol) {
if (alias !== def.alias) {
throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`)
}

// if it was a type before it's not now
def.type = false
return
}
}

defs.push({
symbol,
alias,
type: false
})
}

addTypeImport (module: string, symbol: string, alias?: string): void {
const defs = this._findDefs(module)

for (const def of defs) {
// check if we already have a definition for this symbol
if (def.symbol === symbol) {
if (alias !== def.alias) {
throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`)
}

return
}
}

defs.push({
symbol,
alias,
type: true
})
}

_findDefs (module: string): Import[] {
let defs = this.imports.get(module)

if (defs == null) {
defs = []
this.imports.set(module, defs)
}

return defs
}
}

function defineModule (def: ClassDef, flags: Flags): ModuleDef {
const moduleDef = new ModuleDef()
const defs = def.nested

if (defs == null) {
Expand Down Expand Up @@ -963,28 +1028,48 @@ export async function generate (source: string, flags: Flags): Promise<void> {
]

const imports = []
const importedModules = Array.from([...moduleDef.imports.entries()])
.sort((a, b) => {
return a[0].localeCompare(b[0])
})
.sort((a, b) => {
const aAllTypes = a[1].reduce((acc, curr) => {
return acc && curr.type
}, true)

if (moduleDef.imports.size > 0) {
imports.push(`import { ${Array.from(moduleDef.imports).join(', ')} } from 'protons-runtime'`)
}
const bAllTypes = b[1].reduce((acc, curr) => {
return acc && curr.type
}, true)

if (moduleDef.imports.has('encodeMessage')) {
imports.push("import type { Uint8ArrayList } from 'uint8arraylist'")
}
if (aAllTypes && !bAllTypes) {
return 1
}

if (!aAllTypes && bAllTypes) {
return -1
}

if (moduleDef.importedTypes.size > 0) {
imports.push(`import type { ${Array.from(moduleDef.importedTypes).join(', ')} } from 'protons-runtime'`)
return 0
})

for (const imp of importedModules) {
const allTypes = imp[1].reduce((acc, curr) => {
return acc && curr.type
}, true)

const symbols = imp[1].sort((a, b) => {
return a.symbol.localeCompare(b.symbol)
}).map(imp => {
return `${!allTypes && imp.type ? 'type ' : ''}${imp.symbol}${imp.alias != null ? ` as ${imp.alias}` : ''}`
}).join(', ')

imports.push(`import ${allTypes ? 'type ' : ''}{ ${symbols} } from '${imp[0]}'`)
}

const lines = [
...ignores,
'',
...imports.sort((a, b) => {
const aModule = a.split("from '")[1].toString()
const bModule = b.split("from '")[1].toString()

return aModule.localeCompare(bModule)
}),
...imports,
'',
...moduleDef.compiled
]
Expand Down
3 changes: 1 addition & 2 deletions packages/protons/test/fixtures/basic.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
/* eslint-disable @typescript-eslint/no-empty-interface */

import { encodeMessage, decodeMessage, message } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'

export interface Basic {
Expand Down
12 changes: 6 additions & 6 deletions packages/protons/test/fixtures/bitswap.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
/* eslint-disable @typescript-eslint/no-empty-interface */

import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime'
import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
import type { Uint8ArrayList } from 'uint8arraylist'

export interface Message {
Expand Down Expand Up @@ -87,7 +87,7 @@ export namespace Message {
}
}, (reader, length) => {
const obj: any = {
block: new Uint8Array(0),
block: uint8ArrayAlloc(0),
priority: 0,
wantType: WantType.Block,
sendDontHave: false
Expand Down Expand Up @@ -239,8 +239,8 @@ export namespace Message {
}
}, (reader, length) => {
const obj: any = {
prefix: new Uint8Array(0),
data: new Uint8Array(0)
prefix: uint8ArrayAlloc(0),
data: uint8ArrayAlloc(0)
}

const end = length == null ? reader.len : reader.pos + length
Expand Down Expand Up @@ -326,7 +326,7 @@ export namespace Message {
}
}, (reader, length) => {
const obj: any = {
cid: new Uint8Array(0),
cid: uint8ArrayAlloc(0),
type: BlockPresenceType.Have
}

Expand Down
6 changes: 3 additions & 3 deletions packages/protons/test/fixtures/circuit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
/* eslint-disable @typescript-eslint/no-empty-interface */

import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime'
import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
import type { Uint8ArrayList } from 'uint8arraylist'

export interface CircuitRelay {
Expand Down Expand Up @@ -112,7 +112,7 @@ export namespace CircuitRelay {
}
}, (reader, length) => {
const obj: any = {
id: new Uint8Array(0),
id: uint8ArrayAlloc(0),
addrs: []
}

Expand Down
3 changes: 1 addition & 2 deletions packages/protons/test/fixtures/custom-option-jstype.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
/* eslint-disable @typescript-eslint/no-empty-interface */

import { encodeMessage, decodeMessage, message } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'

export interface CustomOptionNumber {
Expand Down
Loading

0 comments on commit d1bfc94

Please sign in to comment.