From d1bfc948940dbff3e5fe25ab09d7df9d8367d191 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 22 Nov 2023 15:48:04 +0000 Subject: [PATCH] fix: use uint8arrays alloc for new buffers (#123) Will return `Buffer`s under Node.js which have greater performance than `Uint8Array`s. --- packages/protons/package.json | 3 +- packages/protons/src/index.ts | 159 ++++++++++++++---- packages/protons/test/fixtures/basic.ts | 3 +- packages/protons/test/fixtures/bitswap.ts | 12 +- packages/protons/test/fixtures/circuit.ts | 6 +- .../test/fixtures/custom-option-jstype.ts | 3 +- packages/protons/test/fixtures/daemon.ts | 20 +-- packages/protons/test/fixtures/dht.ts | 3 +- packages/protons/test/fixtures/maps.ts | 3 +- packages/protons/test/fixtures/noise.ts | 10 +- packages/protons/test/fixtures/optional.ts | 3 +- packages/protons/test/fixtures/peer.ts | 8 +- packages/protons/test/fixtures/proto2.ts | 3 +- .../protons/test/fixtures/protons-options.ts | 3 +- packages/protons/test/fixtures/singular.ts | 6 +- packages/protons/test/fixtures/test.ts | 3 +- 16 files changed, 163 insertions(+), 85 deletions(-) diff --git a/packages/protons/package.json b/packages/protons/package.json index 4dafce0..beb05f7 100644 --- a/packages/protons/package.json +++ b/packages/protons/package.json @@ -139,6 +139,7 @@ "pbjs": "^0.0.14", "protobufjs": "^7.0.0", "protons-runtime": "^5.0.0", - "uint8arraylist": "^2.4.3" + "uint8arraylist": "^2.4.3", + "uint8arrays": "^4.0.6" } } diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts index fa640e7..aff1816 100644 --- a/packages/protons/src/index.ts +++ b/packages/protons/src/index.ts @@ -190,7 +190,7 @@ const decoderGenerators: Record string> = { bool: () => 'false', - bytes: () => 'new Uint8Array(0)', + bytes: () => 'uint8ArrayAlloc(0)', double: () => '0', fixed32: () => '0', fixed64: () => '0n', @@ -320,6 +320,10 @@ function createDefaultObject (fields: Record, messageDef: Mess defaultValueGenerator = defaultValueGeneratorsJsTypeOverrides[jsTypeOverride] } + if (type === 'bytes') { + moduleDef.addImport('uint8arrays/alloc', 'alloc', 'uint8ArrayAlloc') + } + defaultValue = defaultValueGenerator() } else { const def = findDef(fieldDef.type, messageDef, moduleDef) @@ -457,7 +461,7 @@ function defineFields (fields: Record, messageDef: MessageDef, function compileMessage (messageDef: MessageDef, moduleDef: ModuleDef, flags?: Flags): string { if (isEnumDef(messageDef)) { - moduleDef.imports.add('enumeration') + moduleDef.addImport('protons-runtime', 'enumeration') // check that the enum def values start from 0 if (Object.values(messageDef.values)[0] !== 0) { @@ -510,10 +514,11 @@ export namespace ${messageDef.name} { const fields = messageDef.fields ?? {} // import relevant modules - moduleDef.imports.add('encodeMessage') - moduleDef.imports.add('decodeMessage') - moduleDef.imports.add('message') - moduleDef.importedTypes.add('Codec') + moduleDef.addImport('protons-runtime', 'encodeMessage') + moduleDef.addImport('protons-runtime', 'decodeMessage') + moduleDef.addImport('protons-runtime', 'message') + moduleDef.addTypeImport('protons-runtime', 'Codec') + moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList') const interfaceFields = defineFields(fields, messageDef, moduleDef) .join('\n ') @@ -544,10 +549,10 @@ export interface ${messageDef.name} { if (codec == null) { if (fieldDef.enum) { - moduleDef.imports.add('enumeration') + moduleDef.addImport('protons-runtime', 'enumeration') type = 'enum' } else { - moduleDef.imports.add('message') + moduleDef.addImport('protons-runtime', 'message') type = 'message' } @@ -669,10 +674,10 @@ export interface ${messageDef.name} { if (codec == null) { if (fieldDef.enum) { - moduleDef.imports.add('enumeration') + moduleDef.addImport('protons-runtime', 'enumeration') type = 'enum' } else { - moduleDef.imports.add('message') + moduleDef.addImport('protons-runtime', 'message') type = 'message' } @@ -689,7 +694,7 @@ export interface ${messageDef.name} { let limit = '' if (fieldDef.lengthLimit != null) { - moduleDef.imports.add('CodeError') + moduleDef.addImport('protons-runtime', 'CodeError') limit = ` if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) { @@ -707,7 +712,7 @@ export interface ${messageDef.name} { let limit = '' if (fieldDef.lengthLimit != null) { - moduleDef.imports.add('CodeError') + moduleDef.addImport('protons-runtime', 'CodeError') limit = ` if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) { @@ -785,23 +790,83 @@ export namespace ${messageDef.name} { `.trimStart() } -interface ModuleDef { - imports: Set - importedTypes: Set +interface Import { + symbol: string + alias?: string + type: boolean +} + +class ModuleDef { + imports: Map types: Set compiled: string[] globals: Record -} -function defineModule (def: ClassDef, flags: Flags): ModuleDef { - const moduleDef: ModuleDef = { - imports: new Set(), - importedTypes: new Set(), - types: new Set(), - compiled: [], - globals: {} + constructor () { + this.imports = new Map() + this.types = new Set() + this.compiled = [] + this.globals = {} + } + + addImport (module: string, symbol: string, alias?: string): void { + const defs = this._findDefs(module) + + for (const def of defs) { + // check if we already have a definition for this symbol + if (def.symbol === symbol) { + if (alias !== def.alias) { + throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`) + } + + // if it was a type before it's not now + def.type = false + return + } + } + + defs.push({ + symbol, + alias, + type: false + }) + } + + addTypeImport (module: string, symbol: string, alias?: string): void { + const defs = this._findDefs(module) + + for (const def of defs) { + // check if we already have a definition for this symbol + if (def.symbol === symbol) { + if (alias !== def.alias) { + throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`) + } + + return + } + } + + defs.push({ + symbol, + alias, + type: true + }) } + _findDefs (module: string): Import[] { + let defs = this.imports.get(module) + + if (defs == null) { + defs = [] + this.imports.set(module, defs) + } + + return defs + } +} + +function defineModule (def: ClassDef, flags: Flags): ModuleDef { + const moduleDef = new ModuleDef() const defs = def.nested if (defs == null) { @@ -963,28 +1028,48 @@ export async function generate (source: string, flags: Flags): Promise { ] const imports = [] + const importedModules = Array.from([...moduleDef.imports.entries()]) + .sort((a, b) => { + return a[0].localeCompare(b[0]) + }) + .sort((a, b) => { + const aAllTypes = a[1].reduce((acc, curr) => { + return acc && curr.type + }, true) - if (moduleDef.imports.size > 0) { - imports.push(`import { ${Array.from(moduleDef.imports).join(', ')} } from 'protons-runtime'`) - } + const bAllTypes = b[1].reduce((acc, curr) => { + return acc && curr.type + }, true) - if (moduleDef.imports.has('encodeMessage')) { - imports.push("import type { Uint8ArrayList } from 'uint8arraylist'") - } + if (aAllTypes && !bAllTypes) { + return 1 + } + + if (!aAllTypes && bAllTypes) { + return -1 + } - if (moduleDef.importedTypes.size > 0) { - imports.push(`import type { ${Array.from(moduleDef.importedTypes).join(', ')} } from 'protons-runtime'`) + return 0 + }) + + for (const imp of importedModules) { + const allTypes = imp[1].reduce((acc, curr) => { + return acc && curr.type + }, true) + + const symbols = imp[1].sort((a, b) => { + return a.symbol.localeCompare(b.symbol) + }).map(imp => { + return `${!allTypes && imp.type ? 'type ' : ''}${imp.symbol}${imp.alias != null ? ` as ${imp.alias}` : ''}` + }).join(', ') + + imports.push(`import ${allTypes ? 'type ' : ''}{ ${symbols} } from '${imp[0]}'`) } const lines = [ ...ignores, '', - ...imports.sort((a, b) => { - const aModule = a.split("from '")[1].toString() - const bModule = b.split("from '")[1].toString() - - return aModule.localeCompare(bModule) - }), + ...imports, '', ...moduleDef.compiled ] diff --git a/packages/protons/test/fixtures/basic.ts b/packages/protons/test/fixtures/basic.ts index 7a902bf..53a7731 100644 --- a/packages/protons/test/fixtures/basic.ts +++ b/packages/protons/test/fixtures/basic.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export interface Basic { diff --git a/packages/protons/test/fixtures/bitswap.ts b/packages/protons/test/fixtures/bitswap.ts index e8687bb..60f4bfe 100644 --- a/packages/protons/test/fixtures/bitswap.ts +++ b/packages/protons/test/fixtures/bitswap.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export interface Message { @@ -87,7 +87,7 @@ export namespace Message { } }, (reader, length) => { const obj: any = { - block: new Uint8Array(0), + block: uint8ArrayAlloc(0), priority: 0, wantType: WantType.Block, sendDontHave: false @@ -239,8 +239,8 @@ export namespace Message { } }, (reader, length) => { const obj: any = { - prefix: new Uint8Array(0), - data: new Uint8Array(0) + prefix: uint8ArrayAlloc(0), + data: uint8ArrayAlloc(0) } const end = length == null ? reader.len : reader.pos + length @@ -326,7 +326,7 @@ export namespace Message { } }, (reader, length) => { const obj: any = { - cid: new Uint8Array(0), + cid: uint8ArrayAlloc(0), type: BlockPresenceType.Have } diff --git a/packages/protons/test/fixtures/circuit.ts b/packages/protons/test/fixtures/circuit.ts index 046c06f..3bce53f 100644 --- a/packages/protons/test/fixtures/circuit.ts +++ b/packages/protons/test/fixtures/circuit.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export interface CircuitRelay { @@ -112,7 +112,7 @@ export namespace CircuitRelay { } }, (reader, length) => { const obj: any = { - id: new Uint8Array(0), + id: uint8ArrayAlloc(0), addrs: [] } diff --git a/packages/protons/test/fixtures/custom-option-jstype.ts b/packages/protons/test/fixtures/custom-option-jstype.ts index c05483b..1370674 100644 --- a/packages/protons/test/fixtures/custom-option-jstype.ts +++ b/packages/protons/test/fixtures/custom-option-jstype.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export interface CustomOptionNumber { diff --git a/packages/protons/test/fixtures/daemon.ts b/packages/protons/test/fixtures/daemon.ts index a8a2108..37c9d26 100644 --- a/packages/protons/test/fixtures/daemon.ts +++ b/packages/protons/test/fixtures/daemon.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export interface Request { @@ -361,7 +361,7 @@ export namespace IdentifyResponse { } }, (reader, length) => { const obj: any = { - id: new Uint8Array(0), + id: uint8ArrayAlloc(0), addrs: [] } @@ -440,7 +440,7 @@ export namespace ConnectRequest { } }, (reader, length) => { const obj: any = { - peer: new Uint8Array(0), + peer: uint8ArrayAlloc(0), addrs: [] } @@ -523,7 +523,7 @@ export namespace StreamOpenRequest { } }, (reader, length) => { const obj: any = { - peer: new Uint8Array(0), + peer: uint8ArrayAlloc(0), proto: [] } @@ -600,7 +600,7 @@ export namespace StreamHandlerRequest { } }, (reader, length) => { const obj: any = { - addr: new Uint8Array(0), + addr: uint8ArrayAlloc(0), proto: [] } @@ -737,8 +737,8 @@ export namespace StreamInfo { } }, (reader, length) => { const obj: any = { - peer: new Uint8Array(0), - addr: new Uint8Array(0), + peer: uint8ArrayAlloc(0), + addr: uint8ArrayAlloc(0), proto: '' } @@ -1063,7 +1063,7 @@ export namespace PeerInfo { } }, (reader, length) => { const obj: any = { - id: new Uint8Array(0), + id: uint8ArrayAlloc(0), addrs: [] } @@ -1236,7 +1236,7 @@ export namespace DisconnectRequest { } }, (reader, length) => { const obj: any = { - peer: new Uint8Array(0) + peer: uint8ArrayAlloc(0) } const end = length == null ? reader.len : reader.pos + length diff --git a/packages/protons/test/fixtures/dht.ts b/packages/protons/test/fixtures/dht.ts index abc18f9..e12e068 100644 --- a/packages/protons/test/fixtures/dht.ts +++ b/packages/protons/test/fixtures/dht.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message, enumeration } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export interface Record { diff --git a/packages/protons/test/fixtures/maps.ts b/packages/protons/test/fixtures/maps.ts index 8f8bb31..f65d46f 100644 --- a/packages/protons/test/fixtures/maps.ts +++ b/packages/protons/test/fixtures/maps.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export interface SubMessage { diff --git a/packages/protons/test/fixtures/noise.ts b/packages/protons/test/fixtures/noise.ts index a9ab7c1..252b03a 100644 --- a/packages/protons/test/fixtures/noise.ts +++ b/packages/protons/test/fixtures/noise.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export interface pb {} @@ -47,9 +47,9 @@ export namespace pb { } }, (reader, length) => { const obj: any = { - identityKey: new Uint8Array(0), - identitySig: new Uint8Array(0), - data: new Uint8Array(0) + identityKey: uint8ArrayAlloc(0), + identitySig: uint8ArrayAlloc(0), + data: uint8ArrayAlloc(0) } const end = length == null ? reader.len : reader.pos + length diff --git a/packages/protons/test/fixtures/optional.ts b/packages/protons/test/fixtures/optional.ts index 646b3bf..aca5239 100644 --- a/packages/protons/test/fixtures/optional.ts +++ b/packages/protons/test/fixtures/optional.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export enum OptionalEnum { diff --git a/packages/protons/test/fixtures/peer.ts b/packages/protons/test/fixtures/peer.ts index 2954b2b..ad27fa4 100644 --- a/packages/protons/test/fixtures/peer.ts +++ b/packages/protons/test/fixtures/peer.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export interface Peer { @@ -146,7 +146,7 @@ export namespace Address { } }, (reader, length) => { const obj: any = { - multiaddr: new Uint8Array(0) + multiaddr: uint8ArrayAlloc(0) } const end = length == null ? reader.len : reader.pos + length @@ -217,7 +217,7 @@ export namespace Metadata { }, (reader, length) => { const obj: any = { key: '', - value: new Uint8Array(0) + value: uint8ArrayAlloc(0) } const end = length == null ? reader.len : reader.pos + length diff --git a/packages/protons/test/fixtures/proto2.ts b/packages/protons/test/fixtures/proto2.ts index 6a6f242..f3e44b7 100644 --- a/packages/protons/test/fixtures/proto2.ts +++ b/packages/protons/test/fixtures/proto2.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export interface MessageWithRequired { diff --git a/packages/protons/test/fixtures/protons-options.ts b/packages/protons/test/fixtures/protons-options.ts index 0ad6bcc..bd580f7 100644 --- a/packages/protons/test/fixtures/protons-options.ts +++ b/packages/protons/test/fixtures/protons-options.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { encodeMessage, decodeMessage, message, CodeError } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, CodeError, decodeMessage, encodeMessage, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export interface MessageWithSizeLimitedRepeatedField { diff --git a/packages/protons/test/fixtures/singular.ts b/packages/protons/test/fixtures/singular.ts index bf4920a..406c03e 100644 --- a/packages/protons/test/fixtures/singular.ts +++ b/packages/protons/test/fixtures/singular.ts @@ -4,8 +4,8 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' +import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc' import type { Uint8ArrayList } from 'uint8arraylist' export enum SingularEnum { @@ -230,7 +230,7 @@ export namespace Singular { sfixed64: 0n, bool: false, string: '', - bytes: new Uint8Array(0), + bytes: uint8ArrayAlloc(0), enum: SingularEnum.NO_VALUE } diff --git a/packages/protons/test/fixtures/test.ts b/packages/protons/test/fixtures/test.ts index 0ca31c4..c91c6c3 100644 --- a/packages/protons/test/fixtures/test.ts +++ b/packages/protons/test/fixtures/test.ts @@ -4,8 +4,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ /* eslint-disable @typescript-eslint/no-empty-interface */ -import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' -import type { Codec } from 'protons-runtime' +import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' export enum AnEnum {