Add a function to create a decoder that checks the size of the input bytes (#944)

#### Problem

When an app has access to many codecs, for example when using generated clients, it is easy to write code that accidentally uses the wrong type of `Decoder` for a given byte array. This is difficult to debug as you will often be able to use the wrong decoder without any errors, but will be getting invalid data.

#### Summary of Changes

This PR adds a new helper function `createDecoderThatUsesExactByteArray` that can wrap any decoder. The transformed decoder throws if the new offset after decoding a byte array, is not at the end of the input bytes.

This means that if you pass a byte array that is longer or shorter than the wrapped decoder expects, the decode will fail.

As noted in the issue this is not a strong guarantee that you're using the correct decoder, many decoders are the same size or variable size. But it should catch a lot of difficult to debug cases in practice, such as decoding account data using the wrong decoder.

Note that to enable this, the signature of `transformDecoder` is modified to additionally provide the `newOffset` to the callback. We use both the original offset and the new offset in the error context.  

Fixes #755

Author: mcintyre94

.changeset/quick-crabs-cut.md

@@ -0,0 +1,6 @@
+---
+'@solana/codecs-core': patch
+'@solana/errors': patch
+---
+
+Add a function to create a decoder that checks the size of the input bytes

packages/codecs-core/src/__tests__/decoder-entire-byte-array-test.ts

@@ -0,0 +1,111 @@
+import { SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, SolanaError } from '@solana/errors';
+
+import { createDecoder, Decoder } from '../codec';
+import { createDecoderThatConsumesEntireByteArray } from '../decoder-entire-byte-array';
+
+describe('createDecoderThatConsumesEntireByteArray', () => {
+    // Given: a decoder that consumes exactly 4 bytes
+    const outputNumber = 1234;
+    const innerDecoder: Decoder<number> = createDecoder({
+        fixedSize: 4,
+        read: (_bytes, offset) => [outputNumber, offset + 4],
+    });
+
+    describe('decode function', () => {
+        describe('with no offset', () => {
+            it('returns the same value as the inner decoder when the entire byte array is consumed', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(4);
+                const value = decoder.decode(bytes);
+                expect(value).toBe(outputNumber);
+            });
+
+            it('fatals when the inner decoder does not consume the entire byte array', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(5);
+                expect(() => decoder.decode(bytes)).toThrow(
+                    new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, {
+                        expectedLength: 4,
+                        numExcessBytes: 1,
+                    }),
+                );
+            });
+        });
+
+        describe('with an offset', () => {
+            it('returns the same value as the inner decoder when the entire byte array is consumed', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(6);
+                const value = decoder.decode(bytes, 2);
+                expect(value).toBe(outputNumber);
+            });
+
+            it('fatals when the inner decoder does not consume the entire byte array', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(7);
+                expect(() => decoder.decode(bytes, 2)).toThrow(
+                    new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, {
+                        expectedLength: 6,
+                        numExcessBytes: 1,
+                    }),
+                );
+            });
+        });
+    });
+
+    describe('read function', () => {
+        describe('with no offset', () => {
+            it('returns the same value as the inner decoder when the entire byte array is consumed', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(4);
+                const [value] = decoder.read(bytes, 0);
+                expect(value).toBe(outputNumber);
+            });
+
+            it('returns the same offset as the inner decoder when the entire byte array is consumed', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(4);
+                const [, offset] = decoder.read(bytes, 0);
+                expect(offset).toBe(4);
+            });
+
+            it('fatals when the inner decoder does not consume the entire byte array', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(5);
+                expect(() => decoder.read(bytes, 0)).toThrow(
+                    new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, {
+                        expectedLength: 4,
+                        numExcessBytes: 1,
+                    }),
+                );
+            });
+        });
+
+        describe('with an offset', () => {
+            it('returns the same value as the inner decoder when the entire byte array is consumed', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(6);
+                const [value] = decoder.read(bytes, 2);
+                expect(value).toBe(outputNumber);
+            });
+
+            it('returns the same offset as the inner decoder when the entire byte array is consumed', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(6);
+                const [, offset] = decoder.read(bytes, 2);
+                expect(offset).toBe(6);
+            });
+
+            it('fatals when the inner decoder does not consume the entire byte array', () => {
+                const decoder = createDecoderThatConsumesEntireByteArray(innerDecoder);
+                const bytes = new Uint8Array(7);
+                expect(() => decoder.read(bytes, 2)).toThrow(
+                    new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, {
+                        expectedLength: 6,
+                        numExcessBytes: 1,
+                    }),
+                );
+            });
+        });
+    });
+});

packages/codecs-core/src/decoder-entire-byte-array.ts

@@ -0,0 +1,45 @@
+import { SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, SolanaError } from '@solana/errors';
+
+import { createDecoder, Decoder } from './codec';
+
+/**
+ * Create a {@link Decoder} that asserts that the bytes provided to `decode` or `read` are fully consumed by the inner decoder
+ * @param decoder A decoder to wrap
+ * @returns A new decoder that will throw if provided with a byte array that it does not fully consume
+ *
+ * @typeParam T - The type of the decoder
+ *
+ * @remarks
+ * Note that this compares the offset after encoding to the length of the input byte array
+ *
+ * The `offset` parameter to `decode` and `read` is still considered, and will affect the new offset that is compared to the byte array length
+ *
+ * The error that is thrown by the returned decoder is a {@link SolanaError} with the code `SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY`
+ *
+ * @example
+ * Create a decoder that decodes a `u32` (4 bytes) and ensures the entire byte array is consumed
+ * ```ts
+ * const decoder = createDecoderThatUsesExactByteArray(getU32Decoder());
+ * decoder.decode(new Uint8Array([0, 0, 0, 0])); // 0
+ * decoder.decode(new Uint8Array([0, 0, 0, 0, 0])); // throws
+ *
+ * // with an offset
+ * decoder.decode(new Uint8Array([0, 0, 0, 0, 0]), 1); // 0
+ * decoder.decode(new Uint8Array([0, 0, 0, 0, 0, 0]), 1); // throws
+ * ```
+ */
+export function createDecoderThatConsumesEntireByteArray<T>(decoder: Decoder<T>): Decoder<T> {
+    return createDecoder({
+        ...decoder,
+        read(bytes, offset) {
+            const [value, newOffset] = decoder.read(bytes, offset);
+            if (bytes.length > newOffset) {
+                throw new SolanaError(SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY, {
+                    expectedLength: newOffset,
+                    numExcessBytes: bytes.length - newOffset,
+                });
+            }
+            return [value, newOffset];
+        },
+    });
+}

packages/codecs-core/src/index.ts

@@ -657,6 +657,7 @@ export * from './assertions';
 export * from './bytes';
 export * from './codec';
 export * from './combine-codec';
+export * from './decoder-entire-byte-array';
 export * from './fix-codec-size';
 export * from './offset-codec';
 export * from './pad-codec';

packages/errors/src/codes.ts

@@ -290,6 +290,7 @@ export const SOLANA_ERROR__CODECS__EXPECTED_ZERO_VALUE_TO_MATCH_ITEM_FIXED_SIZE
 export const SOLANA_ERROR__CODECS__ENCODED_BYTES_MUST_NOT_INCLUDE_SENTINEL = 8078020;
 export const SOLANA_ERROR__CODECS__SENTINEL_MISSING_IN_DECODED_BYTES = 8078021;
 export const SOLANA_ERROR__CODECS__CANNOT_USE_LEXICAL_VALUES_AS_ENUM_DISCRIMINATORS = 8078022;
+export const SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY = 8078023;
 
 // RPC-related errors.
 // Reserve error codes in the range [8100000-8100999].
@@ -361,6 +362,7 @@ export type SolanaErrorCode =
     | typeof SOLANA_ERROR__CODECS__ENCODER_DECODER_MAX_SIZE_MISMATCH
     | typeof SOLANA_ERROR__CODECS__ENCODER_DECODER_SIZE_COMPATIBILITY_MISMATCH
     | typeof SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE
+    | typeof SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY
     | typeof SOLANA_ERROR__CODECS__EXPECTED_FIXED_LENGTH
     | typeof SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH
     | typeof SOLANA_ERROR__CODECS__EXPECTED_VARIABLE_LENGTH

packages/errors/src/context.ts

@@ -26,6 +26,7 @@ import {
     SOLANA_ERROR__CODECS__ENCODER_DECODER_FIXED_SIZE_MISMATCH,
     SOLANA_ERROR__CODECS__ENCODER_DECODER_MAX_SIZE_MISMATCH,
     SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE,
+    SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY,
     SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH,
     SOLANA_ERROR__CODECS__EXPECTED_ZERO_VALUE_TO_MATCH_ITEM_FIXED_SIZE,
     SOLANA_ERROR__CODECS__INVALID_BYTE_LENGTH,
@@ -316,6 +317,10 @@ export type SolanaErrorContext = DefaultUnspecifiedErrorContextToUndefined<
             formattedValidDiscriminators: string;
             validDiscriminators: number[];
         };
+        [SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY]: {
+            expectedLength: number;
+            numExcessBytes: number;
+        };
         [SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH]: {
             bytesLength: number;
             codecDescription: string;

packages/errors/src/messages.ts

@@ -33,6 +33,7 @@ import {
     SOLANA_ERROR__CODECS__ENCODER_DECODER_MAX_SIZE_MISMATCH,
     SOLANA_ERROR__CODECS__ENCODER_DECODER_SIZE_COMPATIBILITY_MISMATCH,
     SOLANA_ERROR__CODECS__ENUM_DISCRIMINATOR_OUT_OF_RANGE,
+    SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY,
     SOLANA_ERROR__CODECS__EXPECTED_FIXED_LENGTH,
     SOLANA_ERROR__CODECS__EXPECTED_POSITIVE_BYTE_LENGTH,
     SOLANA_ERROR__CODECS__EXPECTED_VARIABLE_LENGTH,
@@ -330,6 +331,8 @@ export const SolanaErrorMessages: Readonly<{
         'Expected sentinel [$hexSentinel] to be present in decoded bytes [$hexDecodedBytes].',
     [SOLANA_ERROR__CODECS__UNION_VARIANT_OUT_OF_RANGE]:
         'Union variant out of range. Expected an index between $minRange and $maxRange, got $variant.',
+    [SOLANA_ERROR__CODECS__EXPECTED_DECODER_TO_CONSUME_ENTIRE_BYTE_ARRAY]:
+        'This decoder expected a byte array of exactly $expectedLength bytes, but $numExcessBytes unexpected excess bytes remained after decoding. Are you sure that you have chosen the correct decoder for this data?',
     [SOLANA_ERROR__CRYPTO__RANDOM_VALUES_FUNCTION_UNIMPLEMENTED]: 'No random values implementation could be found.',
     [SOLANA_ERROR__INSTRUCTION_ERROR__ACCOUNT_ALREADY_INITIALIZED]: 'instruction requires an uninitialized account',
     [SOLANA_ERROR__INSTRUCTION_ERROR__ACCOUNT_BORROW_FAILED]: