From 96adddbcd1cfb3631a9c9ae7a301c9d8ffeed6a4 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Wed, 21 Sep 2022 18:39:37 -0700 Subject: [PATCH 1/3] Remove lodash dependency from @mcap/core Fixes #588 --- typescript/core/src/pre0/parse.ts | 15 ++++++++-- typescript/core/src/v0/ChunkCursor.ts | 28 ++++++++++++++++-- typescript/core/src/v0/Mcap0StreamReader.ts | 32 +++++++++++++++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/typescript/core/src/pre0/parse.ts b/typescript/core/src/pre0/parse.ts index 58c666151d..ee6230a838 100644 --- a/typescript/core/src/pre0/parse.ts +++ b/typescript/core/src/pre0/parse.ts @@ -1,5 +1,3 @@ -import { isEqual } from "lodash"; - import { getBigUint64 } from "../common/getBigUint64"; import { MCAP_MAGIC, RecordType } from "./constants"; import { McapMagic, McapRecord, ChannelInfo } from "./types"; @@ -122,7 +120,7 @@ export function parseRecord( channelInfosSeenInThisChunk.add(id); const existingInfo = channelInfosById.get(id); if (existingInfo) { - if (!isEqual(existingInfo, record)) { + if (!isChannelInfoEqual(existingInfo, record)) { throw new Error(`differing channel infos for ${record.id}`); } return { @@ -191,3 +189,14 @@ export function parseRecord( throw new Error("Not yet implemented"); } } + +function isChannelInfoEqual(a: ChannelInfo, b: ChannelInfo): boolean { + return ( + a.data.byteLength === b.data.byteLength && + a.encoding === b.encoding && + a.id === b.id && + a.schema === b.schema && + a.schemaName === b.schemaName && + a.topic === b.topic + ); +} diff --git a/typescript/core/src/v0/ChunkCursor.ts b/typescript/core/src/v0/ChunkCursor.ts index 7d050e7560..63dd30c664 100644 --- a/typescript/core/src/v0/ChunkCursor.ts +++ b/typescript/core/src/v0/ChunkCursor.ts @@ -1,5 +1,4 @@ import Heap from "heap-js"; -import { sortedIndexBy } from "lodash"; import { parseRecord } from "./parse"; import { IReadable, TypedMcapRecords } from "./types"; @@ -245,7 +244,7 @@ export class ChunkCursor { startIndex = sortedIndexBy( result.record.records, [this.startTime], - ([logTime]) => logTime, + ([logTime]) => logTime!, ); } } @@ -292,3 +291,28 @@ export class ChunkCursor { return cursor.records[cursor.index]![0]; } } + +// Binary search based on lodash's sortedIndexBy() +function sortedIndexBy(array: T[], value: T, iteratee: (value: T) => number | bigint): number { + let low = 0; + let high = array == null ? 0 : array.length; + if (high === 0) { + return 0; + } + + const valueNumber = iteratee(value); + const valIsNaN = valueNumber !== valueNumber; + + while (low < high) { + const mid = Math.floor((low + high) / 2); + const computed = iteratee(array[mid]!); + const othIsReflexive = computed === computed; + + if (valIsNaN ? othIsReflexive : computed < valueNumber) { + low = mid + 1; + } else { + high = mid; + } + } + return high; +} diff --git a/typescript/core/src/v0/Mcap0StreamReader.ts b/typescript/core/src/v0/Mcap0StreamReader.ts index 27c308af2d..bbdedeac5f 100644 --- a/typescript/core/src/v0/Mcap0StreamReader.ts +++ b/typescript/core/src/v0/Mcap0StreamReader.ts @@ -1,10 +1,15 @@ import { crc32 } from "@foxglove/crc"; -import { isEqual } from "lodash"; import StreamBuffer from "../common/StreamBuffer"; import { MCAP0_MAGIC } from "./constants"; import { parseMagic, parseRecord } from "./parse"; -import { DecompressHandlers, McapStreamReader, TypedMcapRecord, TypedMcapRecords } from "./types"; +import { + Channel, + DecompressHandlers, + McapStreamReader, + TypedMcapRecord, + TypedMcapRecords, +} from "./types"; type McapReaderOptions = { /** @@ -99,7 +104,7 @@ export default class Mcap0StreamReader implements McapStreamReader { if (result.value?.type === "Channel") { const existing = this.channelsById.get(result.value.id); this.channelsById.set(result.value.id, result.value); - if (existing && !isEqual(existing, result.value)) { + if (existing && !isChannelEqual(existing, result.value)) { throw new Error( `Channel record for id ${result.value.id} (topic: ${result.value.topic}) differs from previous channel record of the same id.`, ); @@ -258,3 +263,24 @@ export default class Mcap0StreamReader implements McapStreamReader { } } } + +function isChannelEqual(a: Channel, b: Channel): boolean { + if ( + !( + a.id === b.id && + a.messageEncoding === b.messageEncoding && + a.schemaId === b.schemaId && + a.topic === b.topic && + a.metadata.size === b.metadata.size + ) + ) { + return false; + } + for (const [keyA, valueA] of a.metadata.entries()) { + const valueB = b.metadata.get(keyA); + if (valueA !== valueB) { + return false; + } + } + return true; +} From 6589c771d66a1f4685e3c2793caff4f735c4af02 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Thu, 22 Sep 2022 11:42:40 -0700 Subject: [PATCH 2/3] Deep compare ChannelInfo#data --- typescript/core/src/pre0/parse.ts | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/typescript/core/src/pre0/parse.ts b/typescript/core/src/pre0/parse.ts index ee6230a838..02f20062c2 100644 --- a/typescript/core/src/pre0/parse.ts +++ b/typescript/core/src/pre0/parse.ts @@ -191,12 +191,24 @@ export function parseRecord( } function isChannelInfoEqual(a: ChannelInfo, b: ChannelInfo): boolean { - return ( - a.data.byteLength === b.data.byteLength && - a.encoding === b.encoding && - a.id === b.id && - a.schema === b.schema && - a.schemaName === b.schemaName && - a.topic === b.topic - ); + if ( + a.data.byteLength !== b.data.byteLength || + a.encoding !== b.encoding || + a.id !== b.id || + a.schema !== b.schema || + a.schemaName !== b.schemaName || + a.topic !== b.topic + ) { + return false; + } + + const aData = new Uint8Array(a.data); + const bData = new Uint8Array(b.data); + for (let i = 0; i < aData.length; i++) { + if (aData[i] !== bData[i]) { + return false; + } + } + + return true; } From 0aefd913fcee04ee983bbb361f1d6a1bc2d2aaf2 Mon Sep 17 00:00:00 2001 From: John Hurliman Date: Thu, 22 Sep 2022 11:43:08 -0700 Subject: [PATCH 3/3] Specialize sortedIndexBy, tests --- typescript/core/src/v0/ChunkCursor.ts | 38 ++------------ typescript/core/src/v0/sortedIndexBy.test.ts | 53 ++++++++++++++++++++ typescript/core/src/v0/sortedIndexBy.ts | 29 +++++++++++ 3 files changed, 85 insertions(+), 35 deletions(-) create mode 100644 typescript/core/src/v0/sortedIndexBy.test.ts create mode 100644 typescript/core/src/v0/sortedIndexBy.ts diff --git a/typescript/core/src/v0/ChunkCursor.ts b/typescript/core/src/v0/ChunkCursor.ts index 63dd30c664..091b42452a 100644 --- a/typescript/core/src/v0/ChunkCursor.ts +++ b/typescript/core/src/v0/ChunkCursor.ts @@ -1,6 +1,7 @@ import Heap from "heap-js"; import { parseRecord } from "./parse"; +import { sortedIndexBy } from "./sortedIndexBy"; import { IReadable, TypedMcapRecords } from "./types"; type ChunkCursorParams = { @@ -233,19 +234,11 @@ export class ChunkCursor { let startIndex = 0; if (reverse) { if (this.endTime != undefined) { - startIndex = sortedIndexBy( - result.record.records, - [this.endTime], - ([logTime]) => -logTime!, - ); + startIndex = sortedIndexBy(result.record.records, this.endTime, (logTime) => -logTime); } } else { if (this.startTime != undefined) { - startIndex = sortedIndexBy( - result.record.records, - [this.startTime], - ([logTime]) => logTime!, - ); + startIndex = sortedIndexBy(result.record.records, this.startTime, (logTime) => logTime); } } @@ -291,28 +284,3 @@ export class ChunkCursor { return cursor.records[cursor.index]![0]; } } - -// Binary search based on lodash's sortedIndexBy() -function sortedIndexBy(array: T[], value: T, iteratee: (value: T) => number | bigint): number { - let low = 0; - let high = array == null ? 0 : array.length; - if (high === 0) { - return 0; - } - - const valueNumber = iteratee(value); - const valIsNaN = valueNumber !== valueNumber; - - while (low < high) { - const mid = Math.floor((low + high) / 2); - const computed = iteratee(array[mid]!); - const othIsReflexive = computed === computed; - - if (valIsNaN ? othIsReflexive : computed < valueNumber) { - low = mid + 1; - } else { - high = mid; - } - } - return high; -} diff --git a/typescript/core/src/v0/sortedIndexBy.test.ts b/typescript/core/src/v0/sortedIndexBy.test.ts new file mode 100644 index 0000000000..d89e0a552f --- /dev/null +++ b/typescript/core/src/v0/sortedIndexBy.test.ts @@ -0,0 +1,53 @@ +import { sortedIndexBy } from "./sortedIndexBy"; + +describe("sortedIndexBy", () => { + it("handles an empty array", () => { + const array: [bigint, bigint][] = []; + + expect(sortedIndexBy(array, 0n, (x) => x)).toEqual(0); + expect(sortedIndexBy(array, 42n, (x) => x)).toEqual(0); + }); + + it("handles a contiguous array", () => { + const array: [bigint, bigint][] = [ + [1n, 42n], + [2n, 42n], + [3n, 42n], + ]; + + expect(sortedIndexBy(array, 0n, (x) => x)).toEqual(0); + expect(sortedIndexBy(array, 1n, (x) => x)).toEqual(0); + expect(sortedIndexBy(array, 2n, (x) => x)).toEqual(1); + expect(sortedIndexBy(array, 3n, (x) => x)).toEqual(2); + expect(sortedIndexBy(array, 4n, (x) => x)).toEqual(3); + }); + + it("handles a sparse array", () => { + const array: [bigint, bigint][] = [ + [1n, 42n], + [3n, 42n], + ]; + + expect(sortedIndexBy(array, 0n, (x) => x)).toEqual(0); + expect(sortedIndexBy(array, 1n, (x) => x)).toEqual(0); + expect(sortedIndexBy(array, 2n, (x) => x)).toEqual(1); + expect(sortedIndexBy(array, 3n, (x) => x)).toEqual(1); + expect(sortedIndexBy(array, 4n, (x) => x)).toEqual(2); + }); + + it("handles negation", () => { + const array: [bigint, bigint][] = [ + [1n, 42n], + [2n, 42n], + [3n, 42n], + [4n, 42n], + ]; + + expect(sortedIndexBy(array, 0n, (x) => -x)).toEqual(4); + expect(sortedIndexBy(array, 1n, (x) => -x)).toEqual(4); + expect(sortedIndexBy(array, 2n, (x) => -x)).toEqual(4); + expect(sortedIndexBy(array, 3n, (x) => -x)).toEqual(0); + expect(sortedIndexBy(array, 4n, (x) => -x)).toEqual(0); + expect(sortedIndexBy(array, 5n, (x) => -x)).toEqual(0); + }); +}); diff --git a/typescript/core/src/v0/sortedIndexBy.ts b/typescript/core/src/v0/sortedIndexBy.ts new file mode 100644 index 0000000000..6ec31f15f1 --- /dev/null +++ b/typescript/core/src/v0/sortedIndexBy.ts @@ -0,0 +1,29 @@ +/** + * Return the lowest index of `array` where an element can be inserted and maintain its sorted + * order. This is a specialization of lodash's sortedIndexBy(). + */ +export function sortedIndexBy( + array: [bigint, bigint][], + value: bigint, + iteratee: (value: bigint) => bigint, +): number { + let low = 0; + let high = array.length; + if (high === 0) { + return 0; + } + + const computedValue = iteratee(value); + + while (low < high) { + const mid = (low + high) >>> 1; + const curComputedValue = iteratee(array[mid]![0]); + + if (curComputedValue < computedValue) { + low = mid + 1; + } else { + high = mid; + } + } + return high; +}