From a98fa644777c086b1fc82b95ad829281ffde7da1 Mon Sep 17 00:00:00 2001 From: Loris Leiva Date: Mon, 11 Mar 2024 11:14:40 +0000 Subject: [PATCH] refactor(experimental): add resizeCodec helper to @solana/codecs-core --- packages/codecs-core/README.md | 67 +++++++++++- .../src/__tests__/resize-codec-test.ts | 43 ++++++++ .../__typetests__/resize-codec-typetest.ts | 77 +++++++++++++ packages/codecs-core/src/index.ts | 1 + packages/codecs-core/src/resize-codec.ts | 103 ++++++++++++++++++ packages/codecs/README.md | 1 + packages/errors/src/codes.ts | 2 + packages/errors/src/context.ts | 5 + packages/errors/src/messages.ts | 3 + 9 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 packages/codecs-core/src/__tests__/resize-codec-test.ts create mode 100644 packages/codecs-core/src/__typetests__/resize-codec-typetest.ts create mode 100644 packages/codecs-core/src/resize-codec.ts diff --git a/packages/codecs-core/README.md b/packages/codecs-core/README.md index 194a45889666..52b3769ff3a0 100644 --- a/packages/codecs-core/README.md +++ b/packages/codecs-core/README.md @@ -362,6 +362,71 @@ const get32BytesBase58Decoder = () => fixDecoder(getBase58Decoder(), 32); const get32BytesBase58Codec = () => combineCodec(get32BytesBase58Encoder(), get32BytesBase58Codec()); ``` +## Adjusting the size of codecs + +The `resizeCodec` helper re-defines the size of a given codec by accepting a function that takes the current size of the codec and returns a new size. This works for both fixed-size and variable-size codecs. + +```ts +// Fixed-size codec. +const getBiggerU32Codec = () => resizeCodec(getU32Codec(), size => size + 4); +getBiggerU32Codec().encode(42); +// 0x2a00000000000000 +// | └-- Empty buffer space caused by the resizeCodec function. +// └-- Our encoded u32 number. + +// Variable-size codec. +const getBiggerStringCodec = () => resizeCodec(getStringCodec(), size => size + 4); +getBiggerStringCodec().encode('ABC'); +// 0x0300000041424300000000 +// | └-- Empty buffer space caused by the resizeCodec function. +// └-- Our encoded string with a 4-byte size prefix. +``` + +Note that the `resizeCodec` function doesn't change any encoded or decoded bytes, it merely tells the `encode` and `decode` functions how big the `Uint8Array` should be before delegating to their respective `write` and `read` functions. In fact, this is completely bypassed when using the `write` and `read` functions directly. For instance: + +```ts +const getBiggerU32Codec = () => resizeCodec(getU32Codec(), size => size + 4); + +// Using the encode function. +getBiggerU32Codec().encode(42); +// 0x2a00000000000000 + +// Using the lower-level write function. +const myCustomBytes = new Uint8Array(4); +getBiggerU32Codec().write(42, myCustomBytes, 0); +// 0x2a000000 +``` + +So when would it make sense to use the `resizeCodec` function? This function is particularly useful when combined with the `offsetCodec` function described below. Whilst the `offsetCodec` may help us push the offset forward — e.g. to skip some padding — it won't change the size of the encoded data which means the last bytes will be truncated by how much we pushed the offset forward. The `resizeCodec` function can be used to fix that. For instance, here's how we can use the `resizeCodec` and the `offsetCodec` functions together to create a struct codec that includes some padding. + +```ts +const personCodec = getStructCodec([ + ['name', getStringCodec({ size: 8 })], + // There is a 4-byte padding between name and age. + [ + 'age', + offsetCodec( + resizeCodec(getU32Codec(), size => size + 4), + ({ preOffset }) => preOffset + 4, + ), + ], +]); + +personCodec.encode({ name: 'Alice', age: 42 }); +// 0x416c696365000000000000002a000000 +// | | └-- Our encoded u32 (42). +// | └-- The 4-bytes of padding we are skipping. +// └-- Our 8-byte encoded string ("Alice"). +``` + +As usual, the `resizeEncoder` and `resizeDecoder` functions can also be used to achieve that. + +```ts +const getBiggerU32Encoder = () => resizeEncoder(getU32Codec(), size => size + 4); +const getBiggerU32Decoder = () => resizeDecoder(getU32Codec(), size => size + 4); +const getBiggerU32Codec = () => combineCodec(getBiggerU32Encoder(), getBiggerU32Decoder()); +``` + ## Reversing codecs The `reverseCodec` helper reverses the bytes of the provided `FixedSizeCodec`. @@ -376,7 +441,7 @@ Note that number codecs can already do that for you via their `endian` option. const getBigEndianU64Codec = () => getU64Codec({ endian: Endian.BIG }); ``` -As usual, the `reverseEncoder` and `reverseDecoder` can also be used to achieve that. +As usual, the `reverseEncoder` and `reverseDecoder` functions can also be used to achieve that. ```ts const getBigEndianU64Encoder = () => reverseEncoder(getU64Encoder()); diff --git a/packages/codecs-core/src/__tests__/resize-codec-test.ts b/packages/codecs-core/src/__tests__/resize-codec-test.ts new file mode 100644 index 000000000000..f73440389388 --- /dev/null +++ b/packages/codecs-core/src/__tests__/resize-codec-test.ts @@ -0,0 +1,43 @@ +import { SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, SolanaError } from '@solana/errors'; + +import { FixedSizeCodec } from '../codec'; +import { resizeCodec } from '../resize-codec'; +import { getMockCodec } from './__setup__'; + +describe('resizeCodec', () => { + it('resizes fixed-size codecs', () => { + const mockCodec = getMockCodec({ size: 42 }) as FixedSizeCodec; + expect(resizeCodec(mockCodec, size => size + 1).fixedSize).toBe(43); + expect(resizeCodec(mockCodec, size => size * 2).fixedSize).toBe(84); + expect(resizeCodec(mockCodec, () => 0).fixedSize).toBe(0); + }); + + it('resizes variable-size codecs', () => { + const mockCodec = getMockCodec(); + mockCodec.getSizeFromValue.mockReturnValue(42); + expect(resizeCodec(mockCodec, size => size + 1).getSizeFromValue(null)).toBe(43); + expect(resizeCodec(mockCodec, size => size * 2).getSizeFromValue(null)).toBe(84); + expect(resizeCodec(mockCodec, () => 0).getSizeFromValue(null)).toBe(0); + }); + + it('throws when fixed-size codecs have negative sizes', () => { + const mockCodec = getMockCodec({ size: 42 }) as FixedSizeCodec; + expect(() => resizeCodec(mockCodec, size => size - 100).fixedSize).toThrow( + new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, { + bytesLength: -58, + codecDescription: 'resizeEncoder', + }), + ); + }); + + it('throws when variable-size codecs have negative sizes', () => { + const mockCodec = getMockCodec(); + mockCodec.getSizeFromValue.mockReturnValue(42); + expect(() => resizeCodec(mockCodec, size => size - 100).getSizeFromValue(null)).toThrow( + new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, { + bytesLength: -58, + codecDescription: 'resizeEncoder', + }), + ); + }); +}); diff --git a/packages/codecs-core/src/__typetests__/resize-codec-typetest.ts b/packages/codecs-core/src/__typetests__/resize-codec-typetest.ts new file mode 100644 index 000000000000..e5cc4c59a726 --- /dev/null +++ b/packages/codecs-core/src/__typetests__/resize-codec-typetest.ts @@ -0,0 +1,77 @@ +import { + Codec, + Decoder, + Encoder, + FixedSizeCodec, + FixedSizeDecoder, + FixedSizeEncoder, + VariableSizeCodec, + VariableSizeDecoder, + VariableSizeEncoder, +} from '../codec'; +import { resizeCodec, resizeDecoder, resizeEncoder } from '../resize-codec'; + +type NumberToArray = T['length'] extends N + ? T + : NumberToArray; +type Increment = [...NumberToArray, unknown]['length']; + +{ + // [resizeEncoder]: It returns the same encoder type as the one provided for non-fixed size encoders. + type BrandedEncoder = Encoder<42> & { readonly __brand: unique symbol }; + const resize = (size: number) => size * 2; + resizeEncoder({} as BrandedEncoder, resize) satisfies BrandedEncoder; + resizeEncoder({} as VariableSizeEncoder, resize) satisfies VariableSizeEncoder; + resizeEncoder({} as Encoder, resize) satisfies Encoder; +} + +{ + // [resizeEncoder]: It uses the resize ReturnType as size for fixed-size encoders. + const doubleResize = (size: number): number => size * 2; + const encoder = {} as FixedSizeEncoder; + resizeEncoder(encoder, doubleResize) satisfies FixedSizeEncoder; + // @ts-expect-error We no longer know if the fixed size is 42. + resizeEncoder(encoder, doubleResize) satisfies FixedSizeEncoder; + const incrementResize = (size: TSize) => (size + 1) as Increment; + resizeEncoder(encoder, incrementResize) satisfies FixedSizeEncoder; +} + +{ + // [resizeDecoder]: It returns the same decoder type as the one provided for non-fixed size decoders. + type BrandedDecoder = Decoder<42> & { readonly __brand: unique symbol }; + const resize = (size: number) => size * 2; + resizeDecoder({} as BrandedDecoder, resize) satisfies BrandedDecoder; + resizeDecoder({} as VariableSizeDecoder, resize) satisfies VariableSizeDecoder; + resizeDecoder({} as Decoder, resize) satisfies Decoder; +} + +{ + // [resizeDecoder]: It uses the resize ReturnType as size for fixed-size decoders. + const doubleResize = (size: number): number => size * 2; + const decoder = {} as FixedSizeDecoder; + resizeDecoder(decoder, doubleResize) satisfies FixedSizeDecoder; + // @ts-expect-error We no longer know if the fixed size is 42. + resizeDecoder(decoder, doubleResize) satisfies FixedSizeDecoder; + const incrementResize = (size: TSize) => (size + 1) as Increment; + resizeDecoder(decoder, incrementResize) satisfies FixedSizeDecoder; +} + +{ + // [resizeCodec]: It returns the same codec type as the one provided for non-fixed size codecs. + type BrandedCodec = Codec<42> & { readonly __brand: unique symbol }; + const resize = (size: number) => size * 2; + resizeCodec({} as BrandedCodec, resize) satisfies BrandedCodec; + resizeCodec({} as VariableSizeCodec, resize) satisfies VariableSizeCodec; + resizeCodec({} as Codec, resize) satisfies Codec; +} + +{ + // [resizeCodec]: It uses the resize ReturnType as size for fixed-size codecs. + const doubleResize = (size: number): number => size * 2; + const codec = {} as FixedSizeCodec; + resizeCodec(codec, doubleResize) satisfies FixedSizeCodec; + // @ts-expect-error We no longer know if the fixed size is 42. + resizeCodec(codec, doubleResize) satisfies FixedSizeCodec; + const incrementResize = (size: TSize) => (size + 1) as Increment; + resizeCodec(codec, incrementResize) satisfies FixedSizeCodec; +} diff --git a/packages/codecs-core/src/index.ts b/packages/codecs-core/src/index.ts index f6b2b56dbacc..5df8f73dd3d6 100644 --- a/packages/codecs-core/src/index.ts +++ b/packages/codecs-core/src/index.ts @@ -4,4 +4,5 @@ export * from './codec'; export * from './combine-codec'; export * from './fix-codec'; export * from './map-codec'; +export * from './resize-codec'; export * from './reverse-codec'; diff --git a/packages/codecs-core/src/resize-codec.ts b/packages/codecs-core/src/resize-codec.ts new file mode 100644 index 000000000000..6bf56a0099a9 --- /dev/null +++ b/packages/codecs-core/src/resize-codec.ts @@ -0,0 +1,103 @@ +import { SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, SolanaError } from '@solana/errors'; + +import { + Codec, + createDecoder, + createEncoder, + Decoder, + Encoder, + FixedSizeCodec, + FixedSizeDecoder, + FixedSizeEncoder, + isFixedSize, +} from './codec'; +import { combineCodec } from './combine-codec'; + +/** + * Updates the size of a given encoder. + */ +export function resizeEncoder( + encoder: FixedSizeEncoder, + resize: (size: TSize) => TNewSize, +): FixedSizeEncoder; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeEncoder>( + encoder: TEncoder, + resize: (size: number) => number, +): TEncoder; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeEncoder>( + encoder: TEncoder, + resize: (size: number) => number, +): TEncoder { + if (isFixedSize(encoder)) { + const fixedSize = resize(encoder.fixedSize); + if (fixedSize < 0) { + throw new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, { + bytesLength: fixedSize, + codecDescription: 'resizeEncoder', + }); + } + return createEncoder({ ...encoder, fixedSize }) as TEncoder; + } + return createEncoder({ + ...encoder, + getSizeFromValue: value => { + const newSize = resize(encoder.getSizeFromValue(value)); + if (newSize < 0) { + throw new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, { + bytesLength: newSize, + codecDescription: 'resizeEncoder', + }); + } + return newSize; + }, + }) as TEncoder; +} + +/** + * Updates the size of a given decoder. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any + +export function resizeDecoder( + decoder: FixedSizeDecoder, + resize: (size: TSize) => TNewSize, +): FixedSizeDecoder; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeDecoder>( + decoder: TDecoder, + resize: (size: number) => number, +): TDecoder; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeDecoder>( + decoder: TDecoder, + resize: (size: number) => number, +): TDecoder { + if (isFixedSize(decoder)) { + const fixedSize = resize(decoder.fixedSize); + if (fixedSize < 0) { + throw new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, { + bytesLength: fixedSize, + codecDescription: 'resizeDecoder', + }); + } + return createDecoder({ ...decoder, fixedSize }) as TDecoder; + } + return decoder; +} + +/** + * Updates the size of a given codec. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeCodec( + codec: FixedSizeCodec, + resize: (size: TSize) => TNewSize, +): FixedSizeCodec; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeCodec>(codec: TCodec, resize: (size: number) => number): TCodec; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function resizeCodec>(codec: TCodec, resize: (size: number) => number): TCodec { + return combineCodec(resizeEncoder(codec, resize), resizeDecoder(codec, resize)) as TCodec; +} diff --git a/packages/codecs/README.md b/packages/codecs/README.md index 65635cc41060..6b3c84ff391a 100644 --- a/packages/codecs/README.md +++ b/packages/codecs/README.md @@ -57,6 +57,7 @@ The `@solana/codecs` package is composed of several smaller packages, each with - [Creating custom codecs](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-core#creating-custom-codecs). - [Mapping codecs](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-core#mapping-codecs). - [Fixing the size of codecs](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-core#fixing-the-size-of-codecs). + - [Adjusting the size of codecs](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-core#adjusting-the-size-of-codecs). - [Reversing codecs](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-core#reversing-codecs). - [Byte helpers](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-core#byte-helpers). - [`@solana/codecs-numbers`](https://github.com/solana-labs/solana-web3.js/tree/master/packages/codecs-numbers) This package offers codecs for numbers of various sizes and characteristics. diff --git a/packages/errors/src/codes.ts b/packages/errors/src/codes.ts index 5e4e9b514797..e06bd7d273d4 100644 --- a/packages/errors/src/codes.ts +++ b/packages/errors/src/codes.ts @@ -227,6 +227,7 @@ export const SOLANA_ERROR__CODECS__INVALID_DATA_ENUM_VARIANT = 8078009 as const; export const SOLANA_ERROR__CODECS__INVALID_SCALAR_ENUM_VARIANT = 8078010 as const; export const SOLANA_ERROR__CODECS__NUMBER_OUT_OF_RANGE = 8078011 as const; export const SOLANA_ERROR__CODECS__INVALID_STRING_FOR_BASE = 8078012 as const; +export const SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH = 8078013 as const; // RPC-related errors. // Reserve error codes in the range [8100000-8100999]. @@ -292,6 +293,7 @@ export type SolanaErrorCode = | typeof SOLANA_ERROR__CODECS__ENCODER_DECODER_SIZE_COMPATIBILITY_MISMATCH | typeof SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE | typeof SOLANA_ERROR__CODECS__EXPECTED_FIXED_LENGTH + | typeof SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH | typeof SOLANA_ERROR__CODECS__EXPECTED_VARIABLE_LENGTH | typeof SOLANA_ERROR__CODECS__INVALID_BYTE_LENGTH | typeof SOLANA_ERROR__CODECS__INVALID_DATA_ENUM_VARIANT diff --git a/packages/errors/src/context.ts b/packages/errors/src/context.ts index d58430339d3b..9cd24bed5e79 100644 --- a/packages/errors/src/context.ts +++ b/packages/errors/src/context.ts @@ -16,6 +16,7 @@ import { SOLANA_ERROR__CODECS__ENCODER_DECODER_FIXED_SIZE_MISMATCH, SOLANA_ERROR__CODECS__ENCODER_DECODER_MAX_SIZE_MISMATCH, SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE, + SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, SOLANA_ERROR__CODECS__INVALID_BYTE_LENGTH, SOLANA_ERROR__CODECS__INVALID_DATA_ENUM_VARIANT, SOLANA_ERROR__CODECS__INVALID_NUMBER_OF_ITEMS, @@ -251,6 +252,10 @@ export type SolanaErrorContext = DefaultUnspecifiedErrorContextToUndefined< maxRange: number; minRange: number; }; + [SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH]: { + bytesLength: number; + codecDescription: string; + }; [SOLANA_ERROR__CODECS__INVALID_BYTE_LENGTH]: { bytesLength: number; codecDescription: string; diff --git a/packages/errors/src/messages.ts b/packages/errors/src/messages.ts index 424f423e05e7..6fc1c7d9e111 100644 --- a/packages/errors/src/messages.ts +++ b/packages/errors/src/messages.ts @@ -23,6 +23,7 @@ import { SOLANA_ERROR__CODECS__ENCODER_DECODER_SIZE_COMPATIBILITY_MISMATCH, SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE, SOLANA_ERROR__CODECS__EXPECTED_FIXED_LENGTH, + SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH, SOLANA_ERROR__CODECS__EXPECTED_VARIABLE_LENGTH, SOLANA_ERROR__CODECS__INVALID_BYTE_LENGTH, SOLANA_ERROR__CODECS__INVALID_DATA_ENUM_VARIANT, @@ -235,6 +236,8 @@ export const SolanaErrorMessages: Readonly<{ [SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE]: 'Enum discriminator out of range. Expected a number between $minRange and $maxRange, got $discriminator.', [SOLANA_ERROR__CODECS__EXPECTED_FIXED_LENGTH]: 'Expected a fixed-size codec, got a variable-size one.', + [SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH]: + 'Codec [$codecDescription] expected a positive byte length, got $bytesLength.', [SOLANA_ERROR__CODECS__EXPECTED_VARIABLE_LENGTH]: 'Expected a variable-size codec, got a fixed-size one.', [SOLANA_ERROR__CODECS__INVALID_BYTE_LENGTH]: 'Codec [$codecDescription] expected $expected bytes, got $bytesLength.',