From 0966a2938e0f2a7a1e6061abcf835ac3f5cf7517 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 26 Aug 2025 13:32:43 +0200 Subject: [PATCH 01/14] Parse custom postgres types --- modules/module-postgres/src/index.ts | 2 - .../src/replication/PgManager.ts | 4 + .../src/replication/WalStream.ts | 9 +- modules/module-postgres/src/types/custom.ts | 142 ++++++++ modules/module-postgres/src/types/registry.ts | 316 ++++++++++++++++++ .../module-postgres/src/utils/pgwire_utils.ts | 48 --- .../module-postgres/test/src/pg_test.test.ts | 73 +++- .../test/src/types/registry.test.ts | 114 +++++++ modules/module-postgres/test/src/util.ts | 16 + packages/jpgwire/package.json | 7 +- packages/jpgwire/src/index.ts | 1 + packages/jpgwire/src/pgwire_types.ts | 83 +++-- packages/jpgwire/src/sequence_tokenizer.ts | 207 ++++++++++++ .../jpgwire/test/sequence_tokenizer.test.ts | 123 +++++++ packages/sync-rules/src/compatibility.ts | 9 +- pnpm-lock.yaml | 3 + 16 files changed, 1061 insertions(+), 96 deletions(-) create mode 100644 modules/module-postgres/src/types/custom.ts create mode 100644 modules/module-postgres/src/types/registry.ts delete mode 100644 modules/module-postgres/src/utils/pgwire_utils.ts create mode 100644 modules/module-postgres/test/src/types/registry.test.ts create mode 100644 packages/jpgwire/src/sequence_tokenizer.ts create mode 100644 packages/jpgwire/test/sequence_tokenizer.test.ts diff --git a/modules/module-postgres/src/index.ts b/modules/module-postgres/src/index.ts index ec110750f..3b0d87195 100644 --- a/modules/module-postgres/src/index.ts +++ b/modules/module-postgres/src/index.ts @@ -1,3 +1 @@ export * from './module/PostgresModule.js'; - -export * as pg_utils from './utils/pgwire_utils.js'; diff --git a/modules/module-postgres/src/replication/PgManager.ts b/modules/module-postgres/src/replication/PgManager.ts index 53d062cd6..caa54b23f 100644 --- a/modules/module-postgres/src/replication/PgManager.ts +++ b/modules/module-postgres/src/replication/PgManager.ts @@ -2,6 +2,7 @@ import * as pgwire from '@powersync/service-jpgwire'; import semver from 'semver'; import { NormalizedPostgresConnectionConfig } from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; +import { PostgresTypeCache } from '../types/custom.js'; /** * Shorter timeout for snapshot connections than for replication connections. @@ -14,6 +15,8 @@ export class PgManager { */ public readonly pool: pgwire.PgClient; + public readonly types: PostgresTypeCache; + private connectionPromises: Promise[] = []; constructor( @@ -22,6 +25,7 @@ export class PgManager { ) { // The pool is lazy - no connections are opened until a query is performed. this.pool = pgwire.connectPgWirePool(this.options, poolOptions); + this.types = new PostgresTypeCache(this.pool); } public get connectionTag() { diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index b5e26db4f..1afe74fe3 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -29,7 +29,6 @@ import { TablePattern, toSyncRulesRow } from '@powersync/service-sync-rules'; -import * as pg_utils from '../utils/pgwire_utils.js'; import { PgManager } from './PgManager.js'; import { getPgOutputRelation, getRelId } from './PgRelation.js'; @@ -789,7 +788,7 @@ WHERE oid = $1::regclass`, if (msg.tag == 'insert') { this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); - const baseRecord = pg_utils.constructAfterRecord(msg); + const baseRecord = this.connections.types.constructAfterRecord(msg); return await batch.save({ tag: storage.SaveOperationTag.INSERT, sourceTable: table, @@ -802,8 +801,8 @@ WHERE oid = $1::regclass`, this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. - const before = pg_utils.constructBeforeRecord(msg); - const after = pg_utils.constructAfterRecord(msg); + const before = this.connections.types.constructBeforeRecord(msg); + const after = this.connections.types.constructAfterRecord(msg); return await batch.save({ tag: storage.SaveOperationTag.UPDATE, sourceTable: table, @@ -814,7 +813,7 @@ WHERE oid = $1::regclass`, }); } else if (msg.tag == 'delete') { this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); - const before = pg_utils.constructBeforeRecord(msg)!; + const before = this.connections.types.constructBeforeRecord(msg)!; return await batch.save({ tag: storage.SaveOperationTag.DELETE, diff --git a/modules/module-postgres/src/types/custom.ts b/modules/module-postgres/src/types/custom.ts new file mode 100644 index 000000000..773575063 --- /dev/null +++ b/modules/module-postgres/src/types/custom.ts @@ -0,0 +1,142 @@ +import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules'; +import * as pgwire from '@powersync/service-jpgwire'; +import { CustomTypeRegistry } from './registry.js'; + +export class PostgresTypeCache { + readonly registry: CustomTypeRegistry; + + constructor(private readonly pool: pgwire.PgClient) { + this.registry = new CustomTypeRegistry(); + } + + public async fetchTypes(oids: number[]) { + let pending = oids.filter((id) => !(id in Object.values(pgwire.PgTypeOid))); + // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html + const statement = ` +SELECT oid, pg_type.typtype, + CASE pg_type.typtype + WHEN 'd' THEN json_build_object('type', pg_type.typbasetype) + WHEN 'c' THEN json_build_object( + 'elements', + (SELECT json_agg(json_build_object('name', a.attname, 'type', a.atttypid)) + FROM pg_attribute a + WHERE a.attrelid = pg_type.typrelid) + ) + ELSE NULL + END AS desc +FROM pg_type +WHERE pg_type.oid = ANY($1) +`; + + while (pending.length != 0) { + const query = await this.pool.query({ statement, params: [{ type: 1016, value: pending }] }); + const stillPending: number[] = []; + + const requireType = (oid: number) => { + if (!this.registry.knows(oid) && !pending.includes(oid) && !stillPending.includes(oid)) { + stillPending.push(oid); + } + }; + + for (const row of pgwire.pgwireRows(query)) { + const oid = Number(row.oid); + const desc = JSON.parse(row.desc); + + switch (row.typtype) { + case 'd': + // For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync + // the inner type (pg_type.typbasetype). + const inner = Number(desc.type); + this.registry.setDomainType(oid, inner); + requireType(inner); + break; + case 'c': + // For composite types, we sync the JSON representation. + const elements: { name: string; typeId: number }[] = []; + for (const { name, type } of desc.elements) { + const typeId = Number(type); + elements.push({ name, typeId }); + requireType(typeId); + } + + this.registry.set(oid, { + type: 'composite', + members: elements, + sqliteType: () => 'text' // Since it's JSON + }); + break; + } + } + + pending = stillPending; + } + } + + /** + * Used for testing - fetches all custom types referenced by any column in the schema. + */ + public async fetchTypesForSchema(schema: string = 'public') { + const sql = ` +SELECT DISTINCT a.atttypid AS type_oid +FROM pg_attribute a +JOIN pg_class c ON c.oid = a.attrelid +JOIN pg_namespace cn ON cn.oid = c.relnamespace +JOIN pg_type t ON t.oid = a.atttypid +JOIN pg_namespace tn ON tn.oid = t.typnamespace +WHERE a.attnum > 0 + AND NOT a.attisdropped + AND cn.nspname = $1 + AND tn.nspname NOT IN ('pg_catalog', 'information_schema'); + `; + + const query = await this.pool.query({ statement: sql, params: [{ type: 'varchar', value: schema }] }); + let ids: number[] = []; + for (const row of pgwire.pgwireRows(query)) { + ids.push(Number(row.type_oid)); + } + + await this.fetchTypes(ids); + } + + /** + * pgwire message -> SQLite row. + * @param message + */ + constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteInputRow { + const rawData = (message as any).afterRaw; + + const record = this.decodeTuple(message.relation, rawData); + return toSyncRulesRow(record); + } + + /** + * pgwire message -> SQLite row. + * @param message + */ + constructBeforeRecord(message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate): SqliteInputRow | undefined { + const rawData = (message as any).beforeRaw; + if (rawData == null) { + return undefined; + } + const record = this.decodeTuple(message.relation, rawData); + return toSyncRulesRow(record); + } + + /** + * We need a high level of control over how values are decoded, to make sure there is no loss + * of precision in the process. + */ + decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record): DatabaseInputRow { + let result: Record = {}; + for (let columnName in tupleRaw) { + const rawval = tupleRaw[columnName]; + const typeOid = (relation as any)._tupleDecoder._typeOids.get(columnName); + if (typeof rawval == 'string' && typeOid) { + result[columnName] = this.registry.decodeDatabaseValue(rawval, typeOid); + } else { + result[columnName] = rawval; + } + } + return result; + } +} diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts new file mode 100644 index 000000000..ed01bcfb6 --- /dev/null +++ b/modules/module-postgres/src/types/registry.ts @@ -0,0 +1,316 @@ +import { + applyValueContext, + CompatibilityContext, + CompatibilityOption, + CustomSqliteValue, + DatabaseInputValue, + SqliteValue, + SqliteValueType, + toSyncRulesValue +} from '@powersync/service-sync-rules'; +import * as pgwire from '@powersync/service-jpgwire'; + +interface BaseType { + sqliteType: () => SqliteValueType; +} + +interface BuiltinType extends BaseType { + type: 'builtin'; + oid: number; +} + +interface ArrayType extends BaseType { + type: 'array'; + innerId: number; + separatorCharCode: number; +} + +interface DomainType extends BaseType { + type: 'domain'; + innerId: number; +} + +interface CompositeType extends BaseType { + type: 'composite'; + members: { name: string; typeId: number }[]; +} + +type KnownType = BuiltinType | ArrayType | DomainType | DomainType | CompositeType; + +interface UnknownType extends BaseType { + type: 'unknown'; +} + +type MaybeKnownType = KnownType | UnknownType; + +const UNKNOWN_TYPE: UnknownType = { + type: 'unknown', + sqliteType: () => 'text' +}; + +class CustomTypeValue extends CustomSqliteValue { + constructor( + readonly oid: number, + readonly cache: CustomTypeRegistry, + readonly rawValue: string + ) { + super(); + } + + private lookup(): KnownType | UnknownType { + return this.cache.lookupType(this.oid); + } + + toSqliteValue(context: CompatibilityContext): SqliteValue { + if (context.isEnabled(CompatibilityOption.customTypes)) { + try { + const value = toSyncRulesValue(this.cache.decodeWithCustomTypes(this.rawValue, this.oid)); + return applyValueContext(value, context); + } catch (_e) { + return this.rawValue; + } + } + + return this.rawValue; + } + + get sqliteType(): SqliteValueType { + return this.lookup().sqliteType(); + } +} + +export class CustomTypeRegistry { + private readonly byOid: Map; + + constructor() { + this.byOid = new Map(); + + for (const builtin of Object.values(pgwire.PgTypeOid)) { + if (typeof builtin == 'number') { + let sqliteType: SqliteValueType; + switch (builtin) { + case pgwire.PgTypeOid.TEXT: + case pgwire.PgTypeOid.UUID: + case pgwire.PgTypeOid.VARCHAR: + case pgwire.PgTypeOid.DATE: + case pgwire.PgTypeOid.TIMESTAMP: + case pgwire.PgTypeOid.TIMESTAMPTZ: + case pgwire.PgTypeOid.TIME: + case pgwire.PgTypeOid.JSON: + case pgwire.PgTypeOid.JSONB: + case pgwire.PgTypeOid.PG_LSN: + sqliteType = 'text'; + break; + case pgwire.PgTypeOid.BYTEA: + sqliteType = 'blob'; + break; + case pgwire.PgTypeOid.BOOL: + case pgwire.PgTypeOid.INT2: + case pgwire.PgTypeOid.INT4: + case pgwire.PgTypeOid.OID: + case pgwire.PgTypeOid.INT8: + sqliteType = 'integer'; + break; + case pgwire.PgTypeOid.FLOAT4: + case pgwire.PgTypeOid.FLOAT8: + sqliteType = 'real'; + break; + default: + sqliteType = 'text'; + } + + this.byOid.set(builtin, { + type: 'builtin', + oid: builtin, + sqliteType: () => sqliteType + }); + + const arrayVariant = pgwire.PgType.getArrayType(builtin); + if (arrayVariant != null) { + // NOTE: We could use builtin for this, since PgType.decode can decode arrays. Especially in the presence of + // nested arrays (or arrays in compounds) though, we prefer to keep a common decoder state across everything + // (since it's otherwise hard to decode inner separators properly). So, this ships its own array decoder. + this.byOid.set(arrayVariant, { + type: 'array', + innerId: builtin, + sqliteType: () => sqliteType, + // We assume builtin arrays use commas as a separator (the default) + separatorCharCode: pgwire.CHAR_CODE_COMMA + }); + } + } + } + } + + knows(oid: number): boolean { + return this.byOid.has(oid); + } + + set(oid: number, value: KnownType) { + this.byOid.set(oid, value); + } + + setDomainType(oid: number, inner: number) { + this.set(oid, { + type: 'domain', + innerId: inner, + sqliteType: () => this.lookupType(inner).sqliteType() + }); + } + + decodeWithCustomTypes(raw: string, oid: number): DatabaseInputValue { + const resolved = this.lookupType(oid); + switch (resolved.type) { + case 'builtin': + case 'unknown': + return pgwire.PgType.decode(raw, oid); + case 'domain': + return this.decodeWithCustomTypes(raw, resolved.innerId); + } + + type StructureState = (ArrayType & { parsed: any[] }) | (CompositeType & { parsed: [string, any][] }); + const stateStack: StructureState[] = []; + let pendingNestedStructure: ArrayType | CompositeType | null = resolved; + + const pushParsedValue = (value: any) => { + const top = stateStack[stateStack.length - 1]; + if (top.type == 'array') { + top.parsed.push(value); + } else { + const nextMember = top.members[top.parsed.length]; + if (nextMember) { + top.parsed.push([nextMember.name, value]); + } + } + }; + + const resolveCurrentStructureTypeId = () => { + const top = stateStack[stateStack.length - 1]; + if (top.type == 'array') { + return top.innerId; + } else { + const nextMember = top.members[top.parsed.length]; + if (nextMember) { + return nextMember.typeId; + } else { + return -1; + } + } + }; + + let result: any; + pgwire.decodeSequence({ + source: raw, + delimiters: this.delimitersFor(resolved), + listener: { + onStructureStart: () => { + stateStack.push({ + ...pendingNestedStructure!, + parsed: [] + }); + pendingNestedStructure = null; + }, + onValue: (raw) => { + // this isn't an array or a composite (because in that case we'd make maybeParseSubStructure return nested + // delimiters and this wouldn't get called). + pushParsedValue(raw == null ? null : this.decodeWithCustomTypes(raw, resolveCurrentStructureTypeId())); + }, + onStructureEnd: () => { + const top = stateStack.pop()!; + // For arrays, pop the parsed array. For compounds, create an object from the key-value entries. + const parsedValue = top.type == 'array' ? top.parsed : Object.fromEntries(top.parsed); + + if (stateStack.length == 0) { + // We have exited the outermost structure, parsedValue is the result. + result = parsedValue; + } else { + // Add the result of parsing a nested structure to the current outer structure. + pushParsedValue(parsedValue); + } + }, + maybeParseSubStructure: (firstChar: number) => { + if (firstChar != pgwire.CHAR_CODE_LEFT_BRACE && firstChar != pgwire.CHAR_CODE_LEFT_PAREN) { + // Fast path - definitely not a sub-structure. + return null; + } + + const top = stateStack[stateStack.length - 1]; + if (top.type == 'array' && firstChar == pgwire.CHAR_CODE_LEFT_BRACE) { + // Postgres arrays are natively multidimensional - so if we're in an array, we can always parse sub-arrays + // of the same type. + pendingNestedStructure = top; + return this.delimitersFor(top); + } + + const current = this.lookupType(resolveCurrentStructureTypeId()); + const structure = this.resolveStructure(current); + if (structure != null) { + const [nestedType, delimiters] = structure; + if (delimiters.openingCharCode == firstChar) { + pendingNestedStructure = nestedType; + return delimiters; + } + } + + return null; + } + } + }); + + return result; + } + + private resolveStructure(type: MaybeKnownType): [ArrayType | CompositeType, pgwire.Delimiters] | null { + switch (type.type) { + case 'builtin': + case 'unknown': + return null; + case 'domain': + return this.resolveStructure(this.lookupType(type.innerId)); + case 'array': + case 'composite': + return [type, this.delimitersFor(type)]; + } + } + + private delimitersFor(type: ArrayType | CompositeType): pgwire.Delimiters { + if (type.type == 'array') { + return { + openingCharCode: pgwire.CHAR_CODE_LEFT_BRACE, + closingCharCode: pgwire.CHAR_CODE_RIGHT_BRACE, + delimiterCharCode: type.separatorCharCode + }; + } else { + return { + openingCharCode: pgwire.CHAR_CODE_LEFT_PAREN, + closingCharCode: pgwire.CHAR_CODE_RIGHT_PAREN, + delimiterCharCode: pgwire.CHAR_CODE_COMMA + }; + } + } + + lookupType(type: number): KnownType | UnknownType { + return this.byOid.get(type) ?? UNKNOWN_TYPE; + } + + private isParsedWithoutCustomTypesSupport(type: MaybeKnownType): boolean { + switch (type.type) { + case 'builtin': + case 'unknown': + return true; + case 'array': + return this.isParsedWithoutCustomTypesSupport(this.lookupType(type.innerId)); + default: + return false; + } + } + + decodeDatabaseValue(value: string, oid: number): DatabaseInputValue { + const resolved = this.lookupType(oid); + if (this.isParsedWithoutCustomTypesSupport(resolved)) { + return pgwire.PgType.decode(value, oid); + } else { + return new CustomTypeValue(oid, this, value); + } + } +} diff --git a/modules/module-postgres/src/utils/pgwire_utils.ts b/modules/module-postgres/src/utils/pgwire_utils.ts deleted file mode 100644 index 40c60b216..000000000 --- a/modules/module-postgres/src/utils/pgwire_utils.ts +++ /dev/null @@ -1,48 +0,0 @@ -// Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218 - -import * as pgwire from '@powersync/service-jpgwire'; -import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules'; - -/** - * pgwire message -> SQLite row. - * @param message - */ -export function constructAfterRecord(message: pgwire.PgoutputInsert | pgwire.PgoutputUpdate): SqliteInputRow { - const rawData = (message as any).afterRaw; - - const record = decodeTuple(message.relation, rawData); - return toSyncRulesRow(record); -} - -/** - * pgwire message -> SQLite row. - * @param message - */ -export function constructBeforeRecord( - message: pgwire.PgoutputDelete | pgwire.PgoutputUpdate -): SqliteInputRow | undefined { - const rawData = (message as any).beforeRaw; - if (rawData == null) { - return undefined; - } - const record = decodeTuple(message.relation, rawData); - return toSyncRulesRow(record); -} - -/** - * We need a high level of control over how values are decoded, to make sure there is no loss - * of precision in the process. - */ -export function decodeTuple(relation: pgwire.PgoutputRelation, tupleRaw: Record): DatabaseInputRow { - let result: Record = {}; - for (let columnName in tupleRaw) { - const rawval = tupleRaw[columnName]; - const typeOid = (relation as any)._tupleDecoder._typeOids.get(columnName); - if (typeof rawval == 'string' && typeOid) { - result[columnName] = pgwire.PgType.decode(rawval, typeOid); - } else { - result[columnName] = rawval; - } - } - return result; -} diff --git a/modules/module-postgres/test/src/pg_test.test.ts b/modules/module-postgres/test/src/pg_test.test.ts index 116d95cbc..bd31fa729 100644 --- a/modules/module-postgres/test/src/pg_test.test.ts +++ b/modules/module-postgres/test/src/pg_test.test.ts @@ -1,4 +1,3 @@ -import { constructAfterRecord } from '@module/utils/pgwire_utils.js'; import * as pgwire from '@powersync/service-jpgwire'; import { applyRowContext, @@ -11,6 +10,7 @@ import { import { describe, expect, test } from 'vitest'; import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js'; import { WalStream } from '@module/replication/WalStream.js'; +import { PostgresTypeCache } from '@module/types/custom.js'; describe('pg data types', () => { async function setupTable(db: pgwire.PgClient) { @@ -382,7 +382,7 @@ VALUES(10, ARRAY['null']::TEXT[]); } }); - const transformed = await getReplicationTx(replicationStream); + const transformed = await getReplicationTx(db, replicationStream); await pg.end(); checkResults(transformed); @@ -419,7 +419,7 @@ VALUES(10, ARRAY['null']::TEXT[]); } }); - const transformed = await getReplicationTx(replicationStream); + const transformed = await getReplicationTx(db, replicationStream); await pg.end(); checkResultArrays(transformed.map((e) => applyRowContext(e, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY))); @@ -470,17 +470,80 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' await db.end(); } }); + + test('test replication - custom types', async () => { + const db = await connectPgPool(); + try { + await clearTestDb(db); + await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`); + await db.query(`CREATE TYPE composite AS (foo rating_value, bar TEXT);`); + + await db.query(`CREATE TABLE test_custom( + id serial primary key, + rating rating_value, + composite composite + );`); + + const slotName = 'test_slot'; + + await db.query({ + statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1', + params: [{ type: 'varchar', value: slotName }] + }); + + await db.query({ + statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`, + params: [{ type: 'varchar', value: slotName }] + }); + + await db.query(` + INSERT INTO test_custom + (rating, composite) + VALUES ( + 1, + (2, 'bar') + ); + `); + + const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI); + const replicationStream = await pg.logicalReplication({ + slot: slotName, + options: { + proto_version: '1', + publication_names: 'powersync' + } + }); + + const [transformed] = await getReplicationTx(db, replicationStream); + await pg.end(); + + const oldFormat = applyRowContext(transformed, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY); + expect(oldFormat).toMatchObject({ + rating: '1' + }); + + const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)); + expect(newFormat).toMatchObject({ + rating: 1 + }); + } finally { + await db.end(); + } + }); }); /** * Return all the inserts from the first transaction in the replication stream. */ -async function getReplicationTx(replicationStream: pgwire.ReplicationStream) { +async function getReplicationTx(db: pgwire.PgClient, replicationStream: pgwire.ReplicationStream) { + const typeCache = new PostgresTypeCache(db); + await typeCache.fetchTypesForSchema(); + let transformed: SqliteInputRow[] = []; for await (const batch of replicationStream.pgoutputDecode()) { for (const msg of batch.messages) { if (msg.tag == 'insert') { - transformed.push(constructAfterRecord(msg)); + transformed.push(typeCache.constructAfterRecord(msg)); } else if (msg.tag == 'commit') { return transformed; } diff --git a/modules/module-postgres/test/src/types/registry.test.ts b/modules/module-postgres/test/src/types/registry.test.ts new file mode 100644 index 000000000..4be36d042 --- /dev/null +++ b/modules/module-postgres/test/src/types/registry.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test, beforeEach } from 'vitest'; +import { CustomTypeRegistry } from '@module/types/registry.js'; +import { CHAR_CODE_COMMA, PgTypeOid } from '@powersync/service-jpgwire'; +import { + applyValueContext, + CompatibilityContext, + CompatibilityEdition, + toSyncRulesValue +} from '@powersync/service-sync-rules'; + +describe('custom type registry', () => { + let registry: CustomTypeRegistry; + + beforeEach(() => { + registry = new CustomTypeRegistry(); + }); + + function checkResult(raw: string, type: number, old: any, fixed: any) { + const input = registry.decodeDatabaseValue(raw, type); + const syncRulesValue = toSyncRulesValue(input); + + expect(applyValueContext(syncRulesValue, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY)).toStrictEqual(old); + expect( + applyValueContext(syncRulesValue, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)) + ).toStrictEqual(fixed); + } + + test('domain types', () => { + registry.setDomainType(1337, PgTypeOid.INT4); // create domain wrapping integer + checkResult('12', 1337, '12', 12n); // Should be raw text value without fix, parsed as inner type if enabled + }); + + test('array of domain types', () => { + registry.setDomainType(1337, PgTypeOid.INT4); + registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' }); + + checkResult('{1,2,3}', 1338, '{1,2,3}', '[1,2,3]'); + }); + + test('nested array through domain type', () => { + registry.setDomainType(1337, PgTypeOid.INT4); + registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' }); + registry.setDomainType(1339, 1338); + + checkResult('{1,2,3}', 1339, '{1,2,3}', '[1,2,3]'); + }); + + test('structure', () => { + registry.set(1337, { + type: 'composite', + sqliteType: () => 'text', + members: [ + { name: 'a', typeId: PgTypeOid.BOOL }, + { name: 'b', typeId: PgTypeOid.INT4 }, + { name: 'c', typeId: 1009 } // text array + ] + }); + + checkResult('(t,123,{foo,bar})', 1337, '(t,123,{foo,bar})', '{"a":1,"b":123,"c":["foo","bar"]}'); + }); + + test('array of structure', () => { + registry.set(1337, { + type: 'composite', + sqliteType: () => 'text', + members: [ + { name: 'a', typeId: PgTypeOid.BOOL }, + { name: 'b', typeId: PgTypeOid.INT4 }, + { name: 'c', typeId: 1009 } // text array + ] + }); + registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' }); + + checkResult( + '{(t,123,{foo,bar}),(f,0,{})}', + 1338, + '{(t,123,{foo,bar}),(f,0,{})}', + '[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":0,"c":[]}]' + ); + }); + + test('domain type of structure', () => { + registry.set(1337, { + type: 'composite', + sqliteType: () => 'text', + members: [ + { name: 'a', typeId: PgTypeOid.BOOL }, + { name: 'b', typeId: PgTypeOid.INT4 } + ] + }); + registry.setDomainType(1338, 1337); + + checkResult('(t,123)', 1337, '(t,123)', '{"a":1,"b":123}'); + }); + + test('structure of another structure', () => { + registry.set(1337, { + type: 'composite', + sqliteType: () => 'text', + members: [ + { name: 'a', typeId: PgTypeOid.BOOL }, + { name: 'b', typeId: PgTypeOid.INT4 } + ] + }); + registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' }); + registry.set(1339, { + type: 'composite', + sqliteType: () => 'text', + members: [{ name: 'c', typeId: 1338 }] + }); + + checkResult('({(f,2)})', 1339, '({(f,2)})', '{"c":[{"a":0,"b":2}]}'); + }); +}); diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 130b70fe9..410dd50e2 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -59,6 +59,22 @@ export async function clearTestDb(db: pgwire.PgClient) { await db.query(`DROP TABLE public.${lib_postgres.escapeIdentifier(name)}`); } } + + const domainRows = pgwire.pgwireRows( + await db.query(` + SELECT typname,typtype + FROM pg_type t + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace + WHERE n.nspname = 'public' AND typarray != 0 + `) + ); + for (let row of domainRows) { + if (row.typtype == 'd') { + await db.query(`DROP DOMAIN public.${lib_postgres.escapeIdentifier(row.typname)} CASCADE`); + } else { + await db.query(`DROP TYPE public.${lib_postgres.escapeIdentifier(row.typname)} CASCADE`); + } + } } export async function connectPgWire(type?: 'replication' | 'standard') { diff --git a/packages/jpgwire/package.json b/packages/jpgwire/package.json index 0218ead66..245e11780 100644 --- a/packages/jpgwire/package.json +++ b/packages/jpgwire/package.json @@ -15,12 +15,15 @@ "type": "module", "scripts": { "clean": "rm -r ./dist && tsc -b --clean", - "build": "tsc -b" + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "test": "vitest" }, "dependencies": { "@powersync/service-jsonbig": "workspace:^", "@powersync/service-sync-rules": "workspace:^", "date-fns": "^4.1.0", - "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87" + "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", + "vitest": "^3.0.5" } } diff --git a/packages/jpgwire/src/index.ts b/packages/jpgwire/src/index.ts index 0caa6b349..d7c5e461c 100644 --- a/packages/jpgwire/src/index.ts +++ b/packages/jpgwire/src/index.ts @@ -3,3 +3,4 @@ export * from './certs.js'; export * from './util.js'; export * from './metrics.js'; export * from './pgwire_types.js'; +export * from './sequence_tokenizer.js'; diff --git a/packages/jpgwire/src/pgwire_types.ts b/packages/jpgwire/src/pgwire_types.ts index d71a14d34..ee2fed5e7 100644 --- a/packages/jpgwire/src/pgwire_types.ts +++ b/packages/jpgwire/src/pgwire_types.ts @@ -3,6 +3,14 @@ import { JsonContainer } from '@powersync/service-jsonbig'; import { TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules'; import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js'; +import { + CHAR_CODE_COMMA, + CHAR_CODE_LEFT_BRACE, + CHAR_CODE_RIGHT_BRACE, + decodeSequence, + Delimiters, + SequenceListener +} from './sequence_tokenizer.js'; export enum PgTypeOid { TEXT = 25, @@ -141,52 +149,61 @@ export class PgType { case PgTypeOid.PG_LSN: return lsnMakeComparable(text); } - const elemTypeid = this._elemTypeOid(typeOid); + const elemTypeid = this.elemTypeOid(typeOid); if (elemTypeid != null) { return this._decodeArray(text, elemTypeid); } return text; // unknown type } - static _elemTypeOid(arrayTypeOid: number): number | undefined { + static elemTypeOid(arrayTypeOid: number): number | undefined { // select 'case ' || typarray || ': return ' || oid || '; // ' || typname from pg_catalog.pg_type WHERE typarray != 0; return ARRAY_TO_ELEM_OID.get(arrayTypeOid); } - static _decodeArray(text: string, elemTypeOid: number): any { + static _decodeArray(text: string, elemTypeOid: number): DatabaseInputValue[] { text = text.replace(/^\[.+=/, ''); // skip dimensions - let result: any; - for (let i = 0, inQuotes = false, elStart = 0, stack: any[] = []; i < text.length; i++) { - const ch = text.charCodeAt(i); - if (ch == 0x5c /*\*/) { - i++; // escape - } else if (ch == 0x22 /*"*/) { - inQuotes = !inQuotes; - } else if (inQuotes) { - } else if (ch == 0x7b /*{*/) { - // continue - stack.unshift([]), (elStart = i + 1); - } else if (ch == 0x7d /*}*/ || ch == 0x2c /*,*/) { - // TODO configurable delimiter - // TODO ensure .slice is cheap enough to do it unconditionally - const escaped = text.slice(elStart, i); // TODO trim ' \t\n\r\v\f' - if (result) { - stack[0].push(result); - } else if (/^NULL$/i.test(escaped)) { - stack[0].push(null); - } else if (escaped.length) { - const unescaped = escaped.replace(/^"|"$|(? { + // We're parsing a new array + stack.push([]); + }, + onValue: function (value: string | null): void { + // Atomic (non-array) value, add to current array. + stack[stack.length - 1].push(value && PgType.decode(value, elemTypeOid)); + }, + onStructureEnd: () => { + // We're done parsing an array. + const subarray = stack.pop()!; + if (stack.length == 0) { + // We are done with the outermost array, set results. + results = subarray; + } else { + // We were busy parsing a nested array, continue outer array. + stack[stack.length - 1].push(subarray); } - result = ch == 0x7d /*}*/ && stack.shift(); - elStart = i + 1; // TODO dry } - } - return result; + }; + + decodeSequence({ + source: text, + listener, + delimiters + }); + + return results!; } static _decodeBytea(text: string): Uint8Array { diff --git a/packages/jpgwire/src/sequence_tokenizer.ts b/packages/jpgwire/src/sequence_tokenizer.ts new file mode 100644 index 000000000..62e2e5c5c --- /dev/null +++ b/packages/jpgwire/src/sequence_tokenizer.ts @@ -0,0 +1,207 @@ +export interface SequenceListener { + /** + * Using the context of the listener, determine whether the given character starts a sub-sequence. If so, return the + * {@link Delimiters} for that structure. + * + * For nested arrays, the inner delimiters would always match the outer delimiters. But for other structures (e.g. + * a compound type where one element is an array, that's not the case). That's also why this information is part + * of the listener, as it is inherently stateful! If a compount type has another compound type as a field and an array + * as another, the behavior of this callback depends on the index in the outer compound. + */ + maybeParseSubStructure(firstChar: number): Delimiters | null; + /** + * Invoked whenever the tokenizer has begun decoding a structure (that is, once in the beginning and then for + * every sub-structure). + */ + onStructureStart: () => void; + /** + * Invoked whenever the tokenizer has finished parsing a value that isn't a nested structure. + * + * @param value the raw value, with escape characters related to the outer structure being removed. `null` for the + * literal text `NULL`. + */ + onValue: (value: string | null) => void; + /** + * Invoked whenever a tokenizer has completed a structure (meaning that it's closing brace has been consumed). + */ + onStructureEnd: () => void; +} + +export interface Delimiters { + openingCharCode: number; + closingCharCode: number; + delimiterCharCode: number; +} + +export interface DecodeSequenceOptions { + /** The original text to parse */ + source: string; + /** Delimiters for the outermost structure */ + delimiters: Delimiters; + /** Callbacks to control how values are interpreted and how substructures should be parsed. */ + listener: SequenceListener; +} + +/** + * Decodes a sequence of values, such as arrays or composite types represented as text. + * + * It supports nested arrays, composite types with nested array types, and so on. However, it does not know how to + * parse + */ +export function decodeSequence(options: DecodeSequenceOptions) { + let { source, delimiters, listener } = options; + + const olderStateStack: SequenceDecoderState[] = []; + const olderDelimiterStack: Delimiters[] = []; + let currentState: SequenceDecoderState = SequenceDecoderState.BEFORE_SEQUENCE as SequenceDecoderState; + + consumeChar: for (let i = 0; i < source.length; i++) { + function error(msg: string): never { + throw new Error(`Error decoding Postgres sequence at position ${i}: ${msg}`); + } + + const charCode = source.charCodeAt(i); + + function check(expected: number) { + if (charCode != expected) { + error(`Expected ${String.fromCharCode(expected)}, got ${String.fromCharCode(charCode)}`); + } + } + + function peek(): number { + if (i == source.length - 1) { + error('Unexpected end of input'); + } + + return source.charCodeAt(i + 1); + } + + function advance(): number { + const value = peek(); + i++; + return value; + } + + function quotedString(): string { + const charCodes: number[] = []; + let previousWasBackslash = false; + + while (true) { + const next = advance(); + if (previousWasBackslash) { + if (next != CHAR_CODE_DOUBLE_QUOTE && next != CHAR_CODE_BACKSLASH) { + error('Expected escaped double quote or escaped backslash'); + } + charCodes.push(next); + previousWasBackslash = false; + } else if (next == CHAR_CODE_DOUBLE_QUOTE) { + break; // End of string. + } else if (next == CHAR_CODE_BACKSLASH) { + previousWasBackslash = true; + } else { + charCodes.push(next); + } + } + + return String.fromCharCode(...charCodes); + } + + function unquotedString(): string { + const start = i; + let next = peek(); + while (next != delimiters.delimiterCharCode && next != delimiters.closingCharCode) { + if (next == delimiters.openingCharCode || next == CHAR_CODE_DOUBLE_QUOTE) { + error('illegal char, should require escaping'); + } + + i++; + next = peek(); + } + + return source.substring(start, i + 1); + } + + function endStructure() { + currentState = SequenceDecoderState.AFTER_SEQUENCE; + listener.onStructureEnd(); + if (olderStateStack.length > 0) { + currentState = olderStateStack.pop()!; + delimiters = olderDelimiterStack.pop()!; + } + } + + switch (currentState) { + case SequenceDecoderState.BEFORE_SEQUENCE: + check(delimiters.openingCharCode); + currentState = SequenceDecoderState.BEFORE_ELEMENT_OR_END; + listener.onStructureStart(); + break; + case SequenceDecoderState.BEFORE_ELEMENT_OR_END: + if (charCode == delimiters.closingCharCode) { + endStructure(); + continue consumeChar; + } + // No break between these, end has been handled. + case SequenceDecoderState.BEFORE_ELEMENT: + // What follows is either NULL, a non-empty string value that does not contain delimiters, or an escaped string + // value. + if (charCode == CHAR_CODE_DOUBLE_QUOTE) { + const value = quotedString(); + listener.onValue(value); + } else { + const behavior = listener.maybeParseSubStructure(charCode); + if (behavior == null) { + // Parse the current cell as one value + const value = unquotedString(); + listener.onValue(value == 'NULL' ? null : value); + } else { + currentState = SequenceDecoderState.AFTER_ELEMENT; + listener.onStructureStart(); + olderDelimiterStack.push(delimiters); + olderStateStack.push(currentState); + + delimiters = behavior; + // We've consumed the opening delimiter already, so the inner state can either parse an element or + // immediately close. + currentState = SequenceDecoderState.BEFORE_ELEMENT_OR_END; + continue consumeChar; + } + } + currentState = SequenceDecoderState.AFTER_ELEMENT; + break; + case SequenceDecoderState.AFTER_ELEMENT: + // There can be another element here, or a closing brace + if (charCode == delimiters.closingCharCode) { + endStructure(); + } else { + check(delimiters.delimiterCharCode); + currentState = SequenceDecoderState.BEFORE_ELEMENT; + } + break; + case SequenceDecoderState.AFTER_SEQUENCE: + error('Unexpected trailing text'); + default: + error('Internal error: Unknown state'); + } + } + + if (currentState != SequenceDecoderState.AFTER_SEQUENCE) { + throw Error('Unexpected end of input'); + } +} + +const CHAR_CODE_DOUBLE_QUOTE = 0x22; +const CHAR_CODE_BACKSLASH = 0x5c; +export const CHAR_CODE_COMMA = 0x2c; +export const CHAR_CODE_LEFT_BRACE = 0x7b; +export const CHAR_CODE_RIGHT_BRACE = 0x7d; +export const CHAR_CODE_LEFT_PAREN = 0x28; +export const CHAR_CODE_RIGHT_PAREN = 0x29; + +enum SequenceDecoderState { + BEFORE_SEQUENCE = 1, + BEFORE_ELEMENT_OR_END = 2, + BEFORE_ELEMENT = 3, + AFTER_ELEMENT = 4, + AFTER_SEQUENCE = 5 +} diff --git a/packages/jpgwire/test/sequence_tokenizer.test.ts b/packages/jpgwire/test/sequence_tokenizer.test.ts new file mode 100644 index 000000000..d21af80a5 --- /dev/null +++ b/packages/jpgwire/test/sequence_tokenizer.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, test } from 'vitest'; +import { + SequenceListener, + Delimiters, + CHAR_CODE_COMMA, + CHAR_CODE_RIGHT_BRACE, + CHAR_CODE_LEFT_BRACE, + decodeSequence, + CHAR_CODE_LEFT_PAREN, + CHAR_CODE_RIGHT_PAREN +} from '../src/index'; + +test('empty array', () => { + expect(recordParseEvents('{}', arrayDelimiters)).toStrictEqual(['structureStart', 'structureEnd']); +}); + +test('regular array', () => { + expect(recordParseEvents('{foo,bar}', arrayDelimiters)).toStrictEqual([ + 'structureStart', + 'foo', + 'bar', + 'structureEnd' + ]); +}); + +test('null', () => { + expect(recordParseEvents('{NULL}', arrayDelimiters)).toStrictEqual(['structureStart', null, 'structureEnd']); + expect(recordParseEvents('{null}', arrayDelimiters)).toStrictEqual(['structureStart', 'null', 'structureEnd']); +}); + +test('escaped', () => { + expect(recordParseEvents('{""}', arrayDelimiters)).toStrictEqual(['structureStart', '', 'structureEnd']); + expect(recordParseEvents('{"foo"}', arrayDelimiters)).toStrictEqual(['structureStart', 'foo', 'structureEnd']); + expect(recordParseEvents('{"fo\\"o,"}', arrayDelimiters)).toStrictEqual(['structureStart', 'fo"o,', 'structureEnd']); + expect(recordParseEvents('{"fo\\\\o,"}', arrayDelimiters)).toStrictEqual([ + 'structureStart', + 'fo\\o,', + 'structureEnd' + ]); +}); + +test('nested array', () => { + expect( + recordParseEvents('{0,{0,{}}}', arrayDelimiters, (c) => (c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters : null)) + ).toStrictEqual([ + 'structureStart', + '0', + 'structureStart', + '0', + 'structureStart', + 'structureEnd', + 'structureEnd', + 'structureEnd' + ]); +}); + +test('other structures', () => { + const outerDelimiters: Delimiters = { + openingCharCode: CHAR_CODE_LEFT_PAREN, + closingCharCode: CHAR_CODE_RIGHT_PAREN, + delimiterCharCode: CHAR_CODE_COMMA + }; + + expect(recordParseEvents('()', outerDelimiters)).toStrictEqual(['structureStart', 'structureEnd']); + expect( + recordParseEvents('(foo,bar,{baz})', outerDelimiters, (c) => (c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters : null)) + ).toStrictEqual(['structureStart', 'foo', 'bar', 'structureStart', 'baz', 'structureEnd', 'structureEnd']); +}); + +describe('errors', () => { + test('unclosed array', () => { + expect(() => recordParseEvents('{', arrayDelimiters)).toThrow(/Unexpected end of input/); + }); + + test('trailing data', () => { + expect(() => recordParseEvents('{foo,bar}baz', arrayDelimiters)).toThrow(/Unexpected trailing text/); + }); + + test('improper escaped string', () => { + expect(() => recordParseEvents('{foo,"bar}', arrayDelimiters)).toThrow(/Unexpected end of input/); + }); + + test('illegal escape sequence', () => { + expect(() => recordParseEvents('{foo,"b\\ar"}', arrayDelimiters)).toThrow( + /Expected escaped double quote or escaped backslash/ + ); + }); + + test('illegal delimiter in value', () => { + expect(() => recordParseEvents('{foo{}', arrayDelimiters)).toThrow(/illegal char, should require escaping/); + }); + + test('illegal quote in value', () => { + expect(() => recordParseEvents('{foo"}', arrayDelimiters)).toThrow(/illegal char, should require escaping/); + }); +}); + +const arrayDelimiters: Delimiters = { + openingCharCode: CHAR_CODE_LEFT_BRACE, + closingCharCode: CHAR_CODE_RIGHT_BRACE, + delimiterCharCode: CHAR_CODE_COMMA +}; + +function recordParseEvents( + source: string, + delimiters: Delimiters, + maybeParseSubStructure?: (firstChar: number) => Delimiters | null +) { + maybeParseSubStructure ??= (_) => null; + + const events: any[] = []; + const listener: SequenceListener = { + maybeParseSubStructure, + onStructureStart: () => events.push('structureStart'), + onValue: (value) => { + events.push(value); + }, + onStructureEnd: () => events.push('structureEnd') + }; + + decodeSequence({ source, delimiters, listener }); + return events; +} diff --git a/packages/sync-rules/src/compatibility.ts b/packages/sync-rules/src/compatibility.ts index 091365b55..630256a8b 100644 --- a/packages/sync-rules/src/compatibility.ts +++ b/packages/sync-rules/src/compatibility.ts @@ -34,10 +34,17 @@ export class CompatibilityOption { CompatibilityEdition.SYNC_STREAMS ); + static customTypes = new CompatibilityOption( + 'custom_types', + 'Map custom types into appropriate structures.', + CompatibilityEdition.SYNC_STREAMS + ); + static byName: Record = Object.freeze({ timestamps_iso8601: this.timestampsIso8601, versioned_bucket_ids: this.versionedBucketIds, - fixed_json_extract: this.fixedJsonExtract + fixed_json_extract: this.fixedJsonExtract, + custom_types: this.customTypes }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d24c9a0e9..3541c5456 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -428,6 +428,9 @@ importers: pgwire: specifier: github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87 version: https://codeload.github.com/kagis/pgwire/tar.gz/f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87 + vitest: + specifier: ^3.0.5 + version: 3.0.5(@types/node@22.16.2)(yaml@2.5.0) packages/jsonbig: dependencies: From 91f2c14227b71097bde6ab0857890b84c9059b69 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 26 Aug 2025 15:14:31 +0200 Subject: [PATCH 02/14] Fix encoding --- modules/module-postgres/src/types/custom.ts | 38 ++++++--- modules/module-postgres/src/types/registry.ts | 33 ++------ .../module-postgres/test/src/pg_test.test.ts | 19 +++-- .../test/src/types/registry.test.ts | 17 ++-- packages/jpgwire/src/pgwire_types.ts | 7 +- packages/jpgwire/src/sequence_tokenizer.ts | 50 ++++++++++- .../jpgwire/test/sequence_tokenizer.test.ts | 84 +++++++++++-------- 7 files changed, 158 insertions(+), 90 deletions(-) diff --git a/modules/module-postgres/src/types/custom.ts b/modules/module-postgres/src/types/custom.ts index 773575063..d536ed57d 100644 --- a/modules/module-postgres/src/types/custom.ts +++ b/modules/module-postgres/src/types/custom.ts @@ -13,19 +13,20 @@ export class PostgresTypeCache { let pending = oids.filter((id) => !(id in Object.values(pgwire.PgTypeOid))); // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html const statement = ` -SELECT oid, pg_type.typtype, - CASE pg_type.typtype - WHEN 'd' THEN json_build_object('type', pg_type.typbasetype) +SELECT oid, t.typtype, + CASE t.typtype + WHEN 'b' THEN json_build_object('element_type', t.typelem, 'delim', (SELECT typdelim FROM pg_type i WHERE i.oid = t.typelem)) + WHEN 'd' THEN json_build_object('type', t.typbasetype) WHEN 'c' THEN json_build_object( 'elements', (SELECT json_agg(json_build_object('name', a.attname, 'type', a.atttypid)) FROM pg_attribute a - WHERE a.attrelid = pg_type.typrelid) + WHERE a.attrelid = t.typrelid) ) ELSE NULL END AS desc -FROM pg_type -WHERE pg_type.oid = ANY($1) +FROM pg_type t +WHERE t.oid = ANY($1) `; while (pending.length != 0) { @@ -43,12 +44,18 @@ WHERE pg_type.oid = ANY($1) const desc = JSON.parse(row.desc); switch (row.typtype) { - case 'd': - // For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync - // the inner type (pg_type.typbasetype). - const inner = Number(desc.type); - this.registry.setDomainType(oid, inner); - requireType(inner); + case 'b': + const { element_type, delim } = desc; + + if (!this.registry.knows(oid)) { + // This type is an array of another custom type. + this.registry.set(oid, { + type: 'array', + innerId: Number(element_type), + separatorCharCode: (delim as string).charCodeAt(0), + sqliteType: () => 'text' // Since it's JSON + }); + } break; case 'c': // For composite types, we sync the JSON representation. @@ -65,6 +72,13 @@ WHERE pg_type.oid = ANY($1) sqliteType: () => 'text' // Since it's JSON }); break; + case 'd': + // For domain values like CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5), we sync + // the inner type (pg_type.typbasetype). + const inner = Number(desc.type); + this.registry.setDomainType(oid, inner); + requireType(inner); + break; } } diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index ed01bcfb6..10bbf1299 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -64,7 +64,8 @@ class CustomTypeValue extends CustomSqliteValue { toSqliteValue(context: CompatibilityContext): SqliteValue { if (context.isEnabled(CompatibilityOption.customTypes)) { try { - const value = toSyncRulesValue(this.cache.decodeWithCustomTypes(this.rawValue, this.oid)); + const rawValue = this.cache.decodeWithCustomTypes(this.rawValue, this.oid); + const value = toSyncRulesValue(rawValue); return applyValueContext(value, context); } catch (_e) { return this.rawValue; @@ -211,8 +212,6 @@ export class CustomTypeRegistry { pendingNestedStructure = null; }, onValue: (raw) => { - // this isn't an array or a composite (because in that case we'd make maybeParseSubStructure return nested - // delimiters and this wouldn't get called). pushParsedValue(raw == null ? null : this.decodeWithCustomTypes(raw, resolveCurrentStructureTypeId())); }, onStructureEnd: () => { @@ -229,11 +228,6 @@ export class CustomTypeRegistry { } }, maybeParseSubStructure: (firstChar: number) => { - if (firstChar != pgwire.CHAR_CODE_LEFT_BRACE && firstChar != pgwire.CHAR_CODE_LEFT_PAREN) { - // Fast path - definitely not a sub-structure. - return null; - } - const top = stateStack[stateStack.length - 1]; if (top.type == 'array' && firstChar == pgwire.CHAR_CODE_LEFT_BRACE) { // Postgres arrays are natively multidimensional - so if we're in an array, we can always parse sub-arrays @@ -242,16 +236,7 @@ export class CustomTypeRegistry { return this.delimitersFor(top); } - const current = this.lookupType(resolveCurrentStructureTypeId()); - const structure = this.resolveStructure(current); - if (structure != null) { - const [nestedType, delimiters] = structure; - if (delimiters.openingCharCode == firstChar) { - pendingNestedStructure = nestedType; - return delimiters; - } - } - + // If we're in a compound type, nested compound values or arrays are encoded as strings. return null; } } @@ -275,17 +260,9 @@ export class CustomTypeRegistry { private delimitersFor(type: ArrayType | CompositeType): pgwire.Delimiters { if (type.type == 'array') { - return { - openingCharCode: pgwire.CHAR_CODE_LEFT_BRACE, - closingCharCode: pgwire.CHAR_CODE_RIGHT_BRACE, - delimiterCharCode: type.separatorCharCode - }; + return pgwire.arrayDelimiters(type.separatorCharCode); } else { - return { - openingCharCode: pgwire.CHAR_CODE_LEFT_PAREN, - closingCharCode: pgwire.CHAR_CODE_RIGHT_PAREN, - delimiterCharCode: pgwire.CHAR_CODE_COMMA - }; + return pgwire.COMPOSITE_DELIMITERS; } } diff --git a/modules/module-postgres/test/src/pg_test.test.ts b/modules/module-postgres/test/src/pg_test.test.ts index bd31fa729..63abccb9c 100644 --- a/modules/module-postgres/test/src/pg_test.test.ts +++ b/modules/module-postgres/test/src/pg_test.test.ts @@ -476,12 +476,14 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' try { await clearTestDb(db); await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`); - await db.query(`CREATE TYPE composite AS (foo rating_value, bar TEXT);`); + await db.query(`CREATE TYPE composite AS (foo rating_value[], bar TEXT);`); + await db.query(`CREATE TYPE nested_composite AS (a BOOLEAN, b composite);`); await db.query(`CREATE TABLE test_custom( id serial primary key, rating rating_value, - composite composite + composite composite, + nested_composite nested_composite );`); const slotName = 'test_slot'; @@ -498,10 +500,11 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' await db.query(` INSERT INTO test_custom - (rating, composite) + (rating, composite, nested_composite) VALUES ( 1, - (2, 'bar') + (ARRAY[2,3], 'bar'), + (TRUE, (ARRAY[2,3], 'bar')) ); `); @@ -519,12 +522,16 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' const oldFormat = applyRowContext(transformed, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY); expect(oldFormat).toMatchObject({ - rating: '1' + rating: '1', + composite: '("{2,3}",bar)', + nested_composite: '(t,"(""{2,3}"",bar)")' }); const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)); expect(newFormat).toMatchObject({ - rating: 1 + rating: 1, + composite: '{"foo":[2.0,3.0],"bar":"bar"}', + nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}' }); } finally { await db.end(); diff --git a/modules/module-postgres/test/src/types/registry.test.ts b/modules/module-postgres/test/src/types/registry.test.ts index 4be36d042..e2891547f 100644 --- a/modules/module-postgres/test/src/types/registry.test.ts +++ b/modules/module-postgres/test/src/types/registry.test.ts @@ -46,6 +46,7 @@ describe('custom type registry', () => { }); test('structure', () => { + // create type c1 AS (a bool, b integer, c text[]); registry.set(1337, { type: 'composite', sqliteType: () => 'text', @@ -56,10 +57,12 @@ describe('custom type registry', () => { ] }); - checkResult('(t,123,{foo,bar})', 1337, '(t,123,{foo,bar})', '{"a":1,"b":123,"c":["foo","bar"]}'); + // SELECT (TRUE, 123, ARRAY['foo', 'bar'])::c1; + checkResult('(t,123,"{foo,bar}")', 1337, '(t,123,"{foo,bar}")', '{"a":1,"b":123,"c":["foo","bar"]}'); }); test('array of structure', () => { + // create type c1 AS (a bool, b integer, c text[]); registry.set(1337, { type: 'composite', sqliteType: () => 'text', @@ -71,11 +74,12 @@ describe('custom type registry', () => { }); registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' }); + // SELECT ARRAY[(TRUE, 123, ARRAY['foo', 'bar']),(FALSE, NULL, ARRAY[]::text[])]::c1[]; checkResult( - '{(t,123,{foo,bar}),(f,0,{})}', + '{"(t,123,\\"{foo,bar}\\")","(f,,{})"}', 1338, - '{(t,123,{foo,bar}),(f,0,{})}', - '[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":0,"c":[]}]' + '{"(t,123,\\"{foo,bar}\\")","(f,,{})"}', + '[{"a":1,"b":123,"c":["foo","bar"]},{"a":0,"b":null,"c":[]}]' ); }); @@ -94,6 +98,7 @@ describe('custom type registry', () => { }); test('structure of another structure', () => { + // CREATE TYPE c2 AS (a BOOLEAN, b INTEGER); registry.set(1337, { type: 'composite', sqliteType: () => 'text', @@ -103,12 +108,14 @@ describe('custom type registry', () => { ] }); registry.set(1338, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1337, sqliteType: () => 'text' }); + // CREATE TYPE c3 (c c2[]); registry.set(1339, { type: 'composite', sqliteType: () => 'text', members: [{ name: 'c', typeId: 1338 }] }); - checkResult('({(f,2)})', 1339, '({(f,2)})', '{"c":[{"a":0,"b":2}]}'); + // SELECT ROW(ARRAY[(FALSE,2)]::c2[])::c3; + checkResult('("{""(f,2)""}")', 1339, '("{""(f,2)""}")', '{"c":[{"a":0,"b":2}]}'); }); }); diff --git a/packages/jpgwire/src/pgwire_types.ts b/packages/jpgwire/src/pgwire_types.ts index ee2fed5e7..3908bb019 100644 --- a/packages/jpgwire/src/pgwire_types.ts +++ b/packages/jpgwire/src/pgwire_types.ts @@ -4,6 +4,7 @@ import { JsonContainer } from '@powersync/service-jsonbig'; import { TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules'; import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js'; import { + arrayDelimiters, CHAR_CODE_COMMA, CHAR_CODE_LEFT_BRACE, CHAR_CODE_RIGHT_BRACE, @@ -166,11 +167,7 @@ export class PgType { let results: DatabaseInputValue[]; const stack: DatabaseInputValue[][] = []; - const delimiters: Delimiters = { - openingCharCode: CHAR_CODE_LEFT_BRACE, - closingCharCode: CHAR_CODE_RIGHT_BRACE, - delimiterCharCode: CHAR_CODE_COMMA - }; + const delimiters = arrayDelimiters(); const listener: SequenceListener = { maybeParseSubStructure: function (firstChar: number): Delimiters | null { diff --git a/packages/jpgwire/src/sequence_tokenizer.ts b/packages/jpgwire/src/sequence_tokenizer.ts index 62e2e5c5c..80331e7ae 100644 --- a/packages/jpgwire/src/sequence_tokenizer.ts +++ b/packages/jpgwire/src/sequence_tokenizer.ts @@ -31,6 +31,9 @@ export interface Delimiters { openingCharCode: number; closingCharCode: number; delimiterCharCode: number; + allowEscapingWithDoubleDoubleQuote: boolean; + allowEmpty: boolean; + nullLiteral: string; } export interface DecodeSequenceOptions { @@ -83,6 +86,7 @@ export function decodeSequence(options: DecodeSequenceOptions) { } function quotedString(): string { + const start = i; const charCodes: number[] = []; let previousWasBackslash = false; @@ -95,6 +99,15 @@ export function decodeSequence(options: DecodeSequenceOptions) { charCodes.push(next); previousWasBackslash = false; } else if (next == CHAR_CODE_DOUBLE_QUOTE) { + if (i != start && delimiters.allowEscapingWithDoubleDoubleQuote) { + // If the next character is also a double quote, that escapes a single double quote + if (i < source.length - 1 && peek() == CHAR_CODE_DOUBLE_QUOTE) { + i++; + charCodes.push(CHAR_CODE_DOUBLE_QUOTE); + continue; + } + } + break; // End of string. } else if (next == CHAR_CODE_BACKSLASH) { previousWasBackslash = true; @@ -148,12 +161,25 @@ export function decodeSequence(options: DecodeSequenceOptions) { if (charCode == CHAR_CODE_DOUBLE_QUOTE) { const value = quotedString(); listener.onValue(value); + } else if (charCode == delimiters.delimiterCharCode || charCode == delimiters.closingCharCode) { + if (!delimiters.allowEmpty) { + error('invalid empty element'); + } + + listener.onValue('' == delimiters.nullLiteral ? null : ''); + if (charCode == delimiters.delimiterCharCode) { + // Since this is a comma, there'll be an element afterwards + currentState = SequenceDecoderState.BEFORE_ELEMENT; + } else { + endStructure(); + } + break; } else { const behavior = listener.maybeParseSubStructure(charCode); if (behavior == null) { // Parse the current cell as one value const value = unquotedString(); - listener.onValue(value == 'NULL' ? null : value); + listener.onValue(value == delimiters.nullLiteral ? null : value); } else { currentState = SequenceDecoderState.AFTER_ELEMENT; listener.onStructureStart(); @@ -198,6 +224,28 @@ export const CHAR_CODE_RIGHT_BRACE = 0x7d; export const CHAR_CODE_LEFT_PAREN = 0x28; export const CHAR_CODE_RIGHT_PAREN = 0x29; +// https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO +export function arrayDelimiters(delimiterCharCode: number = CHAR_CODE_COMMA): Delimiters { + return { + openingCharCode: CHAR_CODE_LEFT_BRACE, + closingCharCode: CHAR_CODE_RIGHT_BRACE, + allowEscapingWithDoubleDoubleQuote: false, + nullLiteral: 'NULL', + allowEmpty: false, // Empty values must be escaped + delimiterCharCode + }; +} + +// https://www.postgresql.org/docs/current/rowtypes.html#ROWTYPES-IO-SYNTAX +export const COMPOSITE_DELIMITERS = Object.freeze({ + openingCharCode: CHAR_CODE_LEFT_PAREN, + closingCharCode: CHAR_CODE_RIGHT_PAREN, + delimiterCharCode: CHAR_CODE_COMMA, + allowEscapingWithDoubleDoubleQuote: true, + allowEmpty: true, // Empty values encode NULL + nullLiteral: '' +} satisfies Delimiters); + enum SequenceDecoderState { BEFORE_SEQUENCE = 1, BEFORE_ELEMENT_OR_END = 2, diff --git a/packages/jpgwire/test/sequence_tokenizer.test.ts b/packages/jpgwire/test/sequence_tokenizer.test.ts index d21af80a5..0eb9be0dd 100644 --- a/packages/jpgwire/test/sequence_tokenizer.test.ts +++ b/packages/jpgwire/test/sequence_tokenizer.test.ts @@ -2,20 +2,18 @@ import { describe, expect, test } from 'vitest'; import { SequenceListener, Delimiters, - CHAR_CODE_COMMA, - CHAR_CODE_RIGHT_BRACE, CHAR_CODE_LEFT_BRACE, decodeSequence, - CHAR_CODE_LEFT_PAREN, - CHAR_CODE_RIGHT_PAREN + arrayDelimiters, + COMPOSITE_DELIMITERS } from '../src/index'; test('empty array', () => { - expect(recordParseEvents('{}', arrayDelimiters)).toStrictEqual(['structureStart', 'structureEnd']); + expect(recordParseEvents('{}', arrayDelimiters())).toStrictEqual(['structureStart', 'structureEnd']); }); test('regular array', () => { - expect(recordParseEvents('{foo,bar}', arrayDelimiters)).toStrictEqual([ + expect(recordParseEvents('{foo,bar}', arrayDelimiters())).toStrictEqual([ 'structureStart', 'foo', 'bar', @@ -24,15 +22,19 @@ test('regular array', () => { }); test('null', () => { - expect(recordParseEvents('{NULL}', arrayDelimiters)).toStrictEqual(['structureStart', null, 'structureEnd']); - expect(recordParseEvents('{null}', arrayDelimiters)).toStrictEqual(['structureStart', 'null', 'structureEnd']); + expect(recordParseEvents('{NULL}', arrayDelimiters())).toStrictEqual(['structureStart', null, 'structureEnd']); + expect(recordParseEvents('{null}', arrayDelimiters())).toStrictEqual(['structureStart', 'null', 'structureEnd']); }); test('escaped', () => { - expect(recordParseEvents('{""}', arrayDelimiters)).toStrictEqual(['structureStart', '', 'structureEnd']); - expect(recordParseEvents('{"foo"}', arrayDelimiters)).toStrictEqual(['structureStart', 'foo', 'structureEnd']); - expect(recordParseEvents('{"fo\\"o,"}', arrayDelimiters)).toStrictEqual(['structureStart', 'fo"o,', 'structureEnd']); - expect(recordParseEvents('{"fo\\\\o,"}', arrayDelimiters)).toStrictEqual([ + expect(recordParseEvents('{""}', arrayDelimiters())).toStrictEqual(['structureStart', '', 'structureEnd']); + expect(recordParseEvents('{"foo"}', arrayDelimiters())).toStrictEqual(['structureStart', 'foo', 'structureEnd']); + expect(recordParseEvents('{"fo\\"o,"}', arrayDelimiters())).toStrictEqual([ + 'structureStart', + 'fo"o,', + 'structureEnd' + ]); + expect(recordParseEvents('{"fo\\\\o,"}', arrayDelimiters())).toStrictEqual([ 'structureStart', 'fo\\o,', 'structureEnd' @@ -41,7 +43,7 @@ test('escaped', () => { test('nested array', () => { expect( - recordParseEvents('{0,{0,{}}}', arrayDelimiters, (c) => (c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters : null)) + recordParseEvents('{0,{0,{}}}', arrayDelimiters(), (c) => (c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters() : null)) ).toStrictEqual([ 'structureStart', '0', @@ -55,52 +57,68 @@ test('nested array', () => { }); test('other structures', () => { - const outerDelimiters: Delimiters = { - openingCharCode: CHAR_CODE_LEFT_PAREN, - closingCharCode: CHAR_CODE_RIGHT_PAREN, - delimiterCharCode: CHAR_CODE_COMMA - }; - - expect(recordParseEvents('()', outerDelimiters)).toStrictEqual(['structureStart', 'structureEnd']); + expect(recordParseEvents('()', COMPOSITE_DELIMITERS)).toStrictEqual(['structureStart', 'structureEnd']); expect( - recordParseEvents('(foo,bar,{baz})', outerDelimiters, (c) => (c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters : null)) + recordParseEvents('(foo,bar,{baz})', COMPOSITE_DELIMITERS, (c) => + c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters() : null + ) ).toStrictEqual(['structureStart', 'foo', 'bar', 'structureStart', 'baz', 'structureEnd', 'structureEnd']); }); +test('composite null entries', () => { + // CREATE TYPE nested AS (a BOOLEAN, b BOOLEAN); SELECT (NULL, NULL)::nested; + expect(recordParseEvents('(,)', COMPOSITE_DELIMITERS)).toStrictEqual(['structureStart', null, null, 'structureEnd']); + + // CREATE TYPE triple AS (a BOOLEAN, b BOOLEAN, c BOOLEAN); SELECT (NULL, NULL, NULL)::triple + expect(recordParseEvents('(,,)', COMPOSITE_DELIMITERS)).toStrictEqual([ + 'structureStart', + null, + null, + null, + 'structureEnd' + ]); + + // NOTE: It looks like a single-element composite type has (NULL) encoded as NULL instead of a string like () +}); + +test('composite string escaping', () => { + expect(recordParseEvents('("foo""bar")', COMPOSITE_DELIMITERS)).toStrictEqual([ + 'structureStart', + 'foo"bar', + 'structureEnd' + ]); + + expect(recordParseEvents('("")', COMPOSITE_DELIMITERS)).toStrictEqual(['structureStart', '', 'structureEnd']); +}); + describe('errors', () => { test('unclosed array', () => { - expect(() => recordParseEvents('{', arrayDelimiters)).toThrow(/Unexpected end of input/); + expect(() => recordParseEvents('{', arrayDelimiters())).toThrow(/Unexpected end of input/); }); test('trailing data', () => { - expect(() => recordParseEvents('{foo,bar}baz', arrayDelimiters)).toThrow(/Unexpected trailing text/); + expect(() => recordParseEvents('{foo,bar}baz', arrayDelimiters())).toThrow(/Unexpected trailing text/); }); test('improper escaped string', () => { - expect(() => recordParseEvents('{foo,"bar}', arrayDelimiters)).toThrow(/Unexpected end of input/); + expect(() => recordParseEvents('{foo,"bar}', arrayDelimiters())).toThrow(/Unexpected end of input/); }); test('illegal escape sequence', () => { - expect(() => recordParseEvents('{foo,"b\\ar"}', arrayDelimiters)).toThrow( + expect(() => recordParseEvents('{foo,"b\\ar"}', arrayDelimiters())).toThrow( /Expected escaped double quote or escaped backslash/ ); }); test('illegal delimiter in value', () => { - expect(() => recordParseEvents('{foo{}', arrayDelimiters)).toThrow(/illegal char, should require escaping/); + expect(() => recordParseEvents('{foo{}', arrayDelimiters())).toThrow(/illegal char, should require escaping/); }); test('illegal quote in value', () => { - expect(() => recordParseEvents('{foo"}', arrayDelimiters)).toThrow(/illegal char, should require escaping/); + expect(() => recordParseEvents('{foo"}', arrayDelimiters())).toThrow(/illegal char, should require escaping/); }); }); -const arrayDelimiters: Delimiters = { - openingCharCode: CHAR_CODE_LEFT_BRACE, - closingCharCode: CHAR_CODE_RIGHT_BRACE, - delimiterCharCode: CHAR_CODE_COMMA -}; - function recordParseEvents( source: string, delimiters: Delimiters, From de7f399778c6cdf14a8b23fc16aef07b983748f0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 26 Aug 2025 16:21:57 +0200 Subject: [PATCH 03/14] Simplify --- .../src/replication/PgManager.ts | 2 +- .../src/replication/PgRelation.ts | 9 ++ .../src/replication/WalStream.ts | 70 +++++++---- .../src/types/{custom.ts => cache.ts} | 2 +- modules/module-postgres/src/types/registry.ts | 119 ++++-------------- packages/jpgwire/src/pgwire_types.ts | 40 +----- packages/jpgwire/src/sequence_tokenizer.ts | 105 +++++++++++----- 7 files changed, 161 insertions(+), 186 deletions(-) rename modules/module-postgres/src/types/{custom.ts => cache.ts} (98%) diff --git a/modules/module-postgres/src/replication/PgManager.ts b/modules/module-postgres/src/replication/PgManager.ts index caa54b23f..6acf9638d 100644 --- a/modules/module-postgres/src/replication/PgManager.ts +++ b/modules/module-postgres/src/replication/PgManager.ts @@ -2,7 +2,7 @@ import * as pgwire from '@powersync/service-jpgwire'; import semver from 'semver'; import { NormalizedPostgresConnectionConfig } from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; -import { PostgresTypeCache } from '../types/custom.js'; +import { PostgresTypeCache } from '../types/cache.js'; /** * Shorter timeout for snapshot connections than for replication connections. diff --git a/modules/module-postgres/src/replication/PgRelation.ts b/modules/module-postgres/src/replication/PgRelation.ts index cc3d9a840..3e665c5f2 100644 --- a/modules/module-postgres/src/replication/PgRelation.ts +++ b/modules/module-postgres/src/replication/PgRelation.ts @@ -30,3 +30,12 @@ export function getPgOutputRelation(source: PgoutputRelation): storage.SourceEnt replicaIdColumns: getReplicaIdColumns(source) } satisfies storage.SourceEntityDescriptor; } + +export function referencedColumnTypeIds(source: PgoutputRelation): number[] { + const oids = new Set(); + for (const column of source.columns) { + oids.add(column.typeOid); + } + + return [...oids]; +} diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 1afe74fe3..fc4419812 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -31,7 +31,7 @@ import { } from '@powersync/service-sync-rules'; import { PgManager } from './PgManager.js'; -import { getPgOutputRelation, getRelId } from './PgRelation.js'; +import { getPgOutputRelation, getRelId, referencedColumnTypeIds } from './PgRelation.js'; import { checkSourceConfiguration, checkTableRls, getReplicationIdentityColumns } from './replication-utils.js'; import { ReplicationMetric } from '@powersync/service-types'; import { @@ -188,28 +188,30 @@ export class WalStream { let tableRows: any[]; const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined; - if (tablePattern.isWildcard) { - const result = await db.query({ - statement: `SELECT c.oid AS relid, c.relname AS table_name + + { + let query = ` + SELECT + c.oid AS relid, + c.relname AS table_name, + (SELECT + json_agg(DISTINCT a.atttypid) + FROM pg_attribute a + WHERE a.attnum > 0 AND NOT a.attisdropped AND a.attrelid = c.oid) + AS column_types FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = $1 - AND c.relkind = 'r' - AND c.relname LIKE $2`, - params: [ - { type: 'varchar', value: schema }, - { type: 'varchar', value: tablePattern.tablePattern } - ] - }); - tableRows = pgwire.pgwireRows(result); - } else { + AND c.relkind = 'r'`; + + if (tablePattern.isWildcard) { + query += ' AND c.relname LIKE $2'; + } else { + query += ' AND c.relname = $2'; + } + const result = await db.query({ - statement: `SELECT c.oid AS relid, c.relname AS table_name - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 - AND c.relkind = 'r' - AND c.relname = $2`, + statement: query, params: [ { type: 'varchar', value: schema }, { type: 'varchar', value: tablePattern.tablePattern } @@ -218,6 +220,7 @@ export class WalStream { tableRows = pgwire.pgwireRows(result); } + let result: storage.SourceTable[] = []; for (let row of tableRows) { @@ -257,16 +260,18 @@ export class WalStream { const cresult = await getReplicationIdentityColumns(db, relid); - const table = await this.handleRelation( + const columnTypes = (JSON.parse(row.column_types) as string[]).map((e) => Number(e)); + const table = await this.handleRelation({ batch, - { + descriptor: { name, schema, objectId: relid, replicaIdColumns: cresult.replicationColumns } as SourceEntityDescriptor, - false - ); + snapshot: false, + referencedTypeIds: columnTypes + }); result.push(table); } @@ -682,7 +687,14 @@ WHERE oid = $1::regclass`, } } - async handleRelation(batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor, snapshot: boolean) { + async handleRelation(options: { + batch: storage.BucketStorageBatch; + descriptor: SourceEntityDescriptor; + snapshot: boolean; + referencedTypeIds: number[]; + }) { + const { batch, descriptor, snapshot, referencedTypeIds } = options; + if (!descriptor.objectId && typeof descriptor.objectId != 'number') { throw new ReplicationAssertionError(`objectId expected, got ${typeof descriptor.objectId}`); } @@ -708,6 +720,9 @@ WHERE oid = $1::regclass`, // Truncate this table, in case a previous snapshot was interrupted. await batch.truncate([result.table]); + // Ensure we have a description for custom types referenced in the table. + await this.connections.types.fetchTypes(referencedTypeIds); + // Start the snapshot inside a transaction. // We use a dedicated connection for this. const db = await this.connections.snapshotConnection(); @@ -954,7 +969,12 @@ WHERE oid = $1::regclass`, for (const msg of messages) { if (msg.tag == 'relation') { - await this.handleRelation(batch, getPgOutputRelation(msg), true); + await this.handleRelation({ + batch, + descriptor: getPgOutputRelation(msg), + snapshot: true, + referencedTypeIds: referencedColumnTypeIds(msg) + }); } else if (msg.tag == 'begin') { // This may span multiple transactions in the same chunk, or even across chunks. skipKeepalive = true; diff --git a/modules/module-postgres/src/types/custom.ts b/modules/module-postgres/src/types/cache.ts similarity index 98% rename from modules/module-postgres/src/types/custom.ts rename to modules/module-postgres/src/types/cache.ts index d536ed57d..a281f64a4 100644 --- a/modules/module-postgres/src/types/custom.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -10,7 +10,7 @@ export class PostgresTypeCache { } public async fetchTypes(oids: number[]) { - let pending = oids.filter((id) => !(id in Object.values(pgwire.PgTypeOid))); + let pending = oids.filter((id) => !this.registry.knows(id)); // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html const statement = ` SELECT oid, t.typtype, diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index 10bbf1299..d495ba8d7 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -9,6 +9,7 @@ import { toSyncRulesValue } from '@powersync/service-sync-rules'; import * as pgwire from '@powersync/service-jpgwire'; +import { JsonContainer } from '@powersync/service-jsonbig'; interface BaseType { sqliteType: () => SqliteValueType; @@ -167,102 +168,34 @@ export class CustomTypeRegistry { return pgwire.PgType.decode(raw, oid); case 'domain': return this.decodeWithCustomTypes(raw, resolved.innerId); - } - - type StructureState = (ArrayType & { parsed: any[] }) | (CompositeType & { parsed: [string, any][] }); - const stateStack: StructureState[] = []; - let pendingNestedStructure: ArrayType | CompositeType | null = resolved; - - const pushParsedValue = (value: any) => { - const top = stateStack[stateStack.length - 1]; - if (top.type == 'array') { - top.parsed.push(value); - } else { - const nextMember = top.members[top.parsed.length]; - if (nextMember) { - top.parsed.push([nextMember.name, value]); - } - } - }; - - const resolveCurrentStructureTypeId = () => { - const top = stateStack[stateStack.length - 1]; - if (top.type == 'array') { - return top.innerId; - } else { - const nextMember = top.members[top.parsed.length]; - if (nextMember) { - return nextMember.typeId; - } else { - return -1; - } - } - }; - - let result: any; - pgwire.decodeSequence({ - source: raw, - delimiters: this.delimitersFor(resolved), - listener: { - onStructureStart: () => { - stateStack.push({ - ...pendingNestedStructure!, - parsed: [] - }); - pendingNestedStructure = null; - }, - onValue: (raw) => { - pushParsedValue(raw == null ? null : this.decodeWithCustomTypes(raw, resolveCurrentStructureTypeId())); - }, - onStructureEnd: () => { - const top = stateStack.pop()!; - // For arrays, pop the parsed array. For compounds, create an object from the key-value entries. - const parsedValue = top.type == 'array' ? top.parsed : Object.fromEntries(top.parsed); - - if (stateStack.length == 0) { - // We have exited the outermost structure, parsedValue is the result. - result = parsedValue; - } else { - // Add the result of parsing a nested structure to the current outer structure. - pushParsedValue(parsedValue); - } - }, - maybeParseSubStructure: (firstChar: number) => { - const top = stateStack[stateStack.length - 1]; - if (top.type == 'array' && firstChar == pgwire.CHAR_CODE_LEFT_BRACE) { - // Postgres arrays are natively multidimensional - so if we're in an array, we can always parse sub-arrays - // of the same type. - pendingNestedStructure = top; - return this.delimitersFor(top); + case 'array': + return pgwire.decodeArray({ + source: raw, + decodeElement: (source) => this.decodeWithCustomTypes(source, resolved.innerId), + delimiterCharCode: resolved.separatorCharCode + }); + case 'composite': { + const parsed: [string, any][] = []; + + pgwire.decodeSequence({ + source: raw, + delimiters: pgwire.COMPOSITE_DELIMITERS, + listener: { + onValue: (raw) => { + const nextMember = resolved.members[parsed.length]; + if (nextMember) { + const value = raw == null ? null : this.decodeWithCustomTypes(raw, nextMember.typeId); + parsed.push([nextMember.name, value]); + } + }, + // These are only used for nested arrays + onStructureStart: () => {}, + onStructureEnd: () => {} } + }); - // If we're in a compound type, nested compound values or arrays are encoded as strings. - return null; - } + return Object.fromEntries(parsed); } - }); - - return result; - } - - private resolveStructure(type: MaybeKnownType): [ArrayType | CompositeType, pgwire.Delimiters] | null { - switch (type.type) { - case 'builtin': - case 'unknown': - return null; - case 'domain': - return this.resolveStructure(this.lookupType(type.innerId)); - case 'array': - case 'composite': - return [type, this.delimitersFor(type)]; - } - } - - private delimitersFor(type: ArrayType | CompositeType): pgwire.Delimiters { - if (type.type == 'array') { - return pgwire.arrayDelimiters(type.separatorCharCode); - } else { - return pgwire.COMPOSITE_DELIMITERS; } } diff --git a/packages/jpgwire/src/pgwire_types.ts b/packages/jpgwire/src/pgwire_types.ts index 3908bb019..557512c31 100644 --- a/packages/jpgwire/src/pgwire_types.ts +++ b/packages/jpgwire/src/pgwire_types.ts @@ -1,13 +1,14 @@ // Adapted from https://github.com/kagis/pgwire/blob/0dc927f9f8990a903f238737326e53ba1c8d094f/mod.js#L2218 import { JsonContainer } from '@powersync/service-jsonbig'; -import { TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules'; +import { CustomSqliteValue, TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules'; import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js'; import { arrayDelimiters, CHAR_CODE_COMMA, CHAR_CODE_LEFT_BRACE, CHAR_CODE_RIGHT_BRACE, + decodeArray, decodeSequence, Delimiters, SequenceListener @@ -164,43 +165,10 @@ export class PgType { static _decodeArray(text: string, elemTypeOid: number): DatabaseInputValue[] { text = text.replace(/^\[.+=/, ''); // skip dimensions - - let results: DatabaseInputValue[]; - const stack: DatabaseInputValue[][] = []; - const delimiters = arrayDelimiters(); - - const listener: SequenceListener = { - maybeParseSubStructure: function (firstChar: number): Delimiters | null { - return firstChar == CHAR_CODE_LEFT_BRACE ? delimiters : null; - }, - onStructureStart: () => { - // We're parsing a new array - stack.push([]); - }, - onValue: function (value: string | null): void { - // Atomic (non-array) value, add to current array. - stack[stack.length - 1].push(value && PgType.decode(value, elemTypeOid)); - }, - onStructureEnd: () => { - // We're done parsing an array. - const subarray = stack.pop()!; - if (stack.length == 0) { - // We are done with the outermost array, set results. - results = subarray; - } else { - // We were busy parsing a nested array, continue outer array. - stack[stack.length - 1].push(subarray); - } - } - }; - - decodeSequence({ + return decodeArray({ source: text, - listener, - delimiters + decodeElement: (raw) => PgType.decode(raw, elemTypeOid) }); - - return results!; } static _decodeBytea(text: string): Uint8Array { diff --git a/packages/jpgwire/src/sequence_tokenizer.ts b/packages/jpgwire/src/sequence_tokenizer.ts index 80331e7ae..28dff52b7 100644 --- a/packages/jpgwire/src/sequence_tokenizer.ts +++ b/packages/jpgwire/src/sequence_tokenizer.ts @@ -1,14 +1,4 @@ export interface SequenceListener { - /** - * Using the context of the listener, determine whether the given character starts a sub-sequence. If so, return the - * {@link Delimiters} for that structure. - * - * For nested arrays, the inner delimiters would always match the outer delimiters. But for other structures (e.g. - * a compound type where one element is an array, that's not the case). That's also why this information is part - * of the listener, as it is inherently stateful! If a compount type has another compound type as a field and an array - * as another, the behavior of this callback depends on the index in the outer compound. - */ - maybeParseSubStructure(firstChar: number): Delimiters | null; /** * Invoked whenever the tokenizer has begun decoding a structure (that is, once in the beginning and then for * every sub-structure). @@ -28,12 +18,23 @@ export interface SequenceListener { } export interface Delimiters { + /** The char code opening the structure, e.g. `{` for arrays */ openingCharCode: number; + /** The char code opening the structure, e.g. `}` for arrays */ closingCharCode: number; + /** The char code opening the structure, e.g. `,` */ delimiterCharCode: number; + /** + * Whether two subsequent double quotes are allowed to escape values. + * + * This is the case for composite values, but not for arrays. */ allowEscapingWithDoubleDoubleQuote: boolean; + /** Whether empty values are allowed, e.g. `(,)` */ allowEmpty: boolean; + /** The string literal that denotes a `NULL` value. */ nullLiteral: string; + /** Whether values can be nested sub-structures. */ + multiDimensional: boolean; } export interface DecodeSequenceOptions { @@ -48,14 +49,12 @@ export interface DecodeSequenceOptions { /** * Decodes a sequence of values, such as arrays or composite types represented as text. * - * It supports nested arrays, composite types with nested array types, and so on. However, it does not know how to - * parse + * It supports nested arrays and different options for escaping values needed for arrays and composites. */ export function decodeSequence(options: DecodeSequenceOptions) { let { source, delimiters, listener } = options; - const olderStateStack: SequenceDecoderState[] = []; - const olderDelimiterStack: Delimiters[] = []; + const stateStackTail: SequenceDecoderState[] = []; let currentState: SequenceDecoderState = SequenceDecoderState.BEFORE_SEQUENCE as SequenceDecoderState; consumeChar: for (let i = 0; i < source.length; i++) { @@ -63,8 +62,6 @@ export function decodeSequence(options: DecodeSequenceOptions) { throw new Error(`Error decoding Postgres sequence at position ${i}: ${msg}`); } - const charCode = source.charCodeAt(i); - function check(expected: number) { if (charCode != expected) { error(`Expected ${String.fromCharCode(expected)}, got ${String.fromCharCode(charCode)}`); @@ -137,12 +134,12 @@ export function decodeSequence(options: DecodeSequenceOptions) { function endStructure() { currentState = SequenceDecoderState.AFTER_SEQUENCE; listener.onStructureEnd(); - if (olderStateStack.length > 0) { - currentState = olderStateStack.pop()!; - delimiters = olderDelimiterStack.pop()!; + if (stateStackTail.length > 0) { + currentState = stateStackTail.pop()!; } } + const charCode = source.charCodeAt(i); switch (currentState) { case SequenceDecoderState.BEFORE_SEQUENCE: check(delimiters.openingCharCode); @@ -175,22 +172,19 @@ export function decodeSequence(options: DecodeSequenceOptions) { } break; } else { - const behavior = listener.maybeParseSubStructure(charCode); - if (behavior == null) { - // Parse the current cell as one value - const value = unquotedString(); - listener.onValue(value == delimiters.nullLiteral ? null : value); - } else { + if (delimiters.multiDimensional && charCode == delimiters.openingCharCode) { currentState = SequenceDecoderState.AFTER_ELEMENT; listener.onStructureStart(); - olderDelimiterStack.push(delimiters); - olderStateStack.push(currentState); + stateStackTail.push(currentState); - delimiters = behavior; // We've consumed the opening delimiter already, so the inner state can either parse an element or // immediately close. currentState = SequenceDecoderState.BEFORE_ELEMENT_OR_END; continue consumeChar; + } else { + // Parse the current cell as one value + const value = unquotedString(); + listener.onValue(value == delimiters.nullLiteral ? null : value); } } currentState = SequenceDecoderState.AFTER_ELEMENT; @@ -216,6 +210,55 @@ export function decodeSequence(options: DecodeSequenceOptions) { } } +export type ElementOrArray = null | T | ElementOrArray[]; + +export interface DecodeArrayOptions { + source: string; + delimiterCharCode?: number; + decodeElement: (source: string) => T; +} + +/** + * A variant of {@link decodeSequence} that specifically decodes arrays. + * + * The {@link DecodeArrayOptions.decodeElement} method is responsible for parsing individual values with the array, + * this method automatically recognizes multidimensional arrays and parses them appropriately. + */ +export function decodeArray(options: DecodeArrayOptions): ElementOrArray[] { + let results: ElementOrArray[] = []; + const stack: ElementOrArray[][] = []; + + const listener: SequenceListener = { + onStructureStart: () => { + // We're parsing a new array + stack.push([]); + }, + onValue: function (value: string | null): void { + // Atomic (non-array) value, add to current array. + stack[stack.length - 1].push(value != null ? options.decodeElement(value) : null); + }, + onStructureEnd: () => { + // We're done parsing an array. + const subarray = stack.pop()!; + if (stack.length == 0) { + // We are done with the outermost array, set results. + results = subarray; + } else { + // We were busy parsing a nested array, continue outer array. + stack[stack.length - 1].push(subarray); + } + } + }; + + decodeSequence({ + source: options.source, + listener, + delimiters: arrayDelimiters(options.delimiterCharCode) + }); + + return results!; +} + const CHAR_CODE_DOUBLE_QUOTE = 0x22; const CHAR_CODE_BACKSLASH = 0x5c; export const CHAR_CODE_COMMA = 0x2c; @@ -232,7 +275,8 @@ export function arrayDelimiters(delimiterCharCode: number = CHAR_CODE_COMMA): De allowEscapingWithDoubleDoubleQuote: false, nullLiteral: 'NULL', allowEmpty: false, // Empty values must be escaped - delimiterCharCode + delimiterCharCode, + multiDimensional: true }; } @@ -243,7 +287,8 @@ export const COMPOSITE_DELIMITERS = Object.freeze({ delimiterCharCode: CHAR_CODE_COMMA, allowEscapingWithDoubleDoubleQuote: true, allowEmpty: true, // Empty values encode NULL - nullLiteral: '' + nullLiteral: '', + multiDimensional: false } satisfies Delimiters); enum SequenceDecoderState { From b6a1853004d4ab749eec4a39b7a3cc64925a5490 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 26 Aug 2025 16:49:33 +0200 Subject: [PATCH 04/14] Docs --- modules/module-postgres/src/types/cache.ts | 10 ++++ modules/module-postgres/src/types/registry.ts | 52 ++++++++++++++++--- .../test/src/types/registry.test.ts | 3 ++ packages/jpgwire/src/sequence_tokenizer.ts | 2 +- packages/sync-rules/src/compatibility.ts | 6 +-- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/cache.ts index a281f64a4..b8d7f185b 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -2,6 +2,9 @@ import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/ser import * as pgwire from '@powersync/service-jpgwire'; import { CustomTypeRegistry } from './registry.js'; +/** + * A cache of custom types for which information can be crawled from the source database. + */ export class PostgresTypeCache { readonly registry: CustomTypeRegistry; @@ -9,6 +12,12 @@ export class PostgresTypeCache { this.registry = new CustomTypeRegistry(); } + /** + * Fetches information about indicated types. + * + * If a type references another custom type (e.g. because it's a composite type with a custom field), these are + * automatically crawled as well. + */ public async fetchTypes(oids: number[]) { let pending = oids.filter((id) => !this.registry.knows(id)); // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html @@ -30,6 +39,7 @@ WHERE t.oid = ANY($1) `; while (pending.length != 0) { + // 1016: int8 array const query = await this.pool.query({ statement, params: [{ type: 1016, value: pending }] }); const stillPending: number[] = []; diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index d495ba8d7..75526e67c 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -9,28 +9,41 @@ import { toSyncRulesValue } from '@powersync/service-sync-rules'; import * as pgwire from '@powersync/service-jpgwire'; -import { JsonContainer } from '@powersync/service-jsonbig'; interface BaseType { sqliteType: () => SqliteValueType; } +/** A type natively supported by {@link pgwire.PgType.decode}. */ interface BuiltinType extends BaseType { type: 'builtin'; oid: number; } +/** + * An array type. + */ interface ArrayType extends BaseType { type: 'array'; innerId: number; separatorCharCode: number; } +/** + * A domain type, like `CREATE DOMAIN api.rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);` + * + * This type gets decoded and synced as the inner type (`FLOAT` in the example above). + */ interface DomainType extends BaseType { type: 'domain'; innerId: number; } +/** + * A composite type as created by `CREATE TYPE AS`. + * + * These types are encoded as a tuple of values, so we recover attribute names to restore them as a JSON object. + */ interface CompositeType extends BaseType { type: 'composite'; members: { name: string; typeId: number }[]; @@ -81,6 +94,12 @@ class CustomTypeValue extends CustomSqliteValue { } } +/** + * A registry of custom types. + * + * These extend the builtin decoding behavior in {@link pgwire.PgType.decode} for user-defined types like `DOMAIN`s or + * composite types. + */ export class CustomTypeRegistry { private readonly byOid: Map; @@ -89,6 +108,7 @@ export class CustomTypeRegistry { for (const builtin of Object.values(pgwire.PgTypeOid)) { if (typeof builtin == 'number') { + // We need to know the SQLite type of builtins to implement CustomSqliteValue.sqliteType for DOMAIN types. let sqliteType: SqliteValueType; switch (builtin) { case pgwire.PgTypeOid.TEXT: @@ -168,12 +188,6 @@ export class CustomTypeRegistry { return pgwire.PgType.decode(raw, oid); case 'domain': return this.decodeWithCustomTypes(raw, resolved.innerId); - case 'array': - return pgwire.decodeArray({ - source: raw, - decodeElement: (source) => this.decodeWithCustomTypes(source, resolved.innerId), - delimiterCharCode: resolved.separatorCharCode - }); case 'composite': { const parsed: [string, any][] = []; @@ -196,6 +210,28 @@ export class CustomTypeRegistry { return Object.fromEntries(parsed); } + case 'array': { + // Nornalize "array of array of T" types into just "array of T", because Postgres arrays are natively multi- + // dimensional. This may be required when we have a DOMAIN wrapper around an array followed by another array + // around that domain. + let innerId = resolved.innerId; + while (true) { + const resolvedInner = this.lookupType(innerId); + if (resolvedInner.type == 'domain') { + innerId = resolvedInner.innerId; + } else if (resolvedInner.type == 'array') { + innerId = resolvedInner.innerId; + } else { + break; + } + } + + return pgwire.decodeArray({ + source: raw, + decodeElement: (source) => this.decodeWithCustomTypes(source, innerId), + delimiterCharCode: resolved.separatorCharCode + }); + } } } @@ -217,6 +253,8 @@ export class CustomTypeRegistry { decodeDatabaseValue(value: string, oid: number): DatabaseInputValue { const resolved = this.lookupType(oid); + // For backwards-compatibility, some types are only properly parsed with a compatibility option. Others are synced + // in the raw text representation by default, and are only parsed as JSON values when necessary. if (this.isParsedWithoutCustomTypesSupport(resolved)) { return pgwire.PgType.decode(value, oid); } else { diff --git a/modules/module-postgres/test/src/types/registry.test.ts b/modules/module-postgres/test/src/types/registry.test.ts index e2891547f..68b9d2871 100644 --- a/modules/module-postgres/test/src/types/registry.test.ts +++ b/modules/module-postgres/test/src/types/registry.test.ts @@ -43,6 +43,9 @@ describe('custom type registry', () => { registry.setDomainType(1339, 1338); checkResult('{1,2,3}', 1339, '{1,2,3}', '[1,2,3]'); + + registry.set(1400, { type: 'array', separatorCharCode: CHAR_CODE_COMMA, innerId: 1339, sqliteType: () => 'text' }); + checkResult('{{1,2,3}}', 1400, '{{1,2,3}}', '[[1,2,3]]'); }); test('structure', () => { diff --git a/packages/jpgwire/src/sequence_tokenizer.ts b/packages/jpgwire/src/sequence_tokenizer.ts index 28dff52b7..133098f1e 100644 --- a/packages/jpgwire/src/sequence_tokenizer.ts +++ b/packages/jpgwire/src/sequence_tokenizer.ts @@ -8,7 +8,7 @@ export interface SequenceListener { * Invoked whenever the tokenizer has finished parsing a value that isn't a nested structure. * * @param value the raw value, with escape characters related to the outer structure being removed. `null` for the - * literal text `NULL`. + * literal text {@link Delimiters.nullLiteral}. */ onValue: (value: string | null) => void; /** diff --git a/packages/sync-rules/src/compatibility.ts b/packages/sync-rules/src/compatibility.ts index 630256a8b..24ba9738b 100644 --- a/packages/sync-rules/src/compatibility.ts +++ b/packages/sync-rules/src/compatibility.ts @@ -35,8 +35,8 @@ export class CompatibilityOption { ); static customTypes = new CompatibilityOption( - 'custom_types', - 'Map custom types into appropriate structures.', + 'custom_postgres_types', + 'Map custom Postgres types into appropriate structures instead of syncing the raw string.', CompatibilityEdition.SYNC_STREAMS ); @@ -44,7 +44,7 @@ export class CompatibilityOption { timestamps_iso8601: this.timestampsIso8601, versioned_bucket_ids: this.versionedBucketIds, fixed_json_extract: this.fixedJsonExtract, - custom_types: this.customTypes + custom_postgres_types: this.customTypes }); } From e5de28a6ed8f31a4c6a67858ef5503dcc8c0ffde Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 27 Aug 2025 12:31:40 +0200 Subject: [PATCH 05/14] Fix decoding box[] type --- modules/module-postgres/src/types/registry.ts | 49 ++++++++++++------- .../module-postgres/test/src/pg_test.test.ts | 16 +++--- packages/jpgwire/src/pgwire_types.ts | 2 +- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index 75526e67c..08550ec44 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -75,18 +75,21 @@ class CustomTypeValue extends CustomSqliteValue { return this.cache.lookupType(this.oid); } - toSqliteValue(context: CompatibilityContext): SqliteValue { + private decodeToDatabaseInputValue(context: CompatibilityContext): DatabaseInputValue { if (context.isEnabled(CompatibilityOption.customTypes)) { try { - const rawValue = this.cache.decodeWithCustomTypes(this.rawValue, this.oid); - const value = toSyncRulesValue(rawValue); - return applyValueContext(value, context); + return this.cache.decodeWithCustomTypes(this.rawValue, this.oid); } catch (_e) { return this.rawValue; } + } else { + return pgwire.PgType.decode(this.rawValue, this.oid); } + } - return this.rawValue; + toSqliteValue(context: CompatibilityContext): SqliteValue { + const value = toSyncRulesValue(this.decodeToDatabaseInputValue(context)); + return applyValueContext(value, context); } get sqliteType(): SqliteValueType { @@ -146,20 +149,25 @@ export class CustomTypeRegistry { oid: builtin, sqliteType: () => sqliteType }); + } + } - const arrayVariant = pgwire.PgType.getArrayType(builtin); - if (arrayVariant != null) { - // NOTE: We could use builtin for this, since PgType.decode can decode arrays. Especially in the presence of - // nested arrays (or arrays in compounds) though, we prefer to keep a common decoder state across everything - // (since it's otherwise hard to decode inner separators properly). So, this ships its own array decoder. - this.byOid.set(arrayVariant, { - type: 'array', - innerId: builtin, - sqliteType: () => sqliteType, - // We assume builtin arrays use commas as a separator (the default) - separatorCharCode: pgwire.CHAR_CODE_COMMA - }); - } + for (const [arrayId, innerId] of pgwire.ARRAY_TO_ELEM_OID.entries()) { + // We can just use the default decoder, except for box[] because those use a different delimiter. We don't fix + // this in PgType._decodeArray for backwards-compatibility. + if (innerId == 603) { + this.byOid.set(arrayId, { + type: 'array', + innerId, + sqliteType: () => 'text', // these get encoded as JSON arrays + separatorCharCode: 0x3b // ";" + }); + } else { + this.byOid.set(arrayId, { + type: 'builtin', + oid: arrayId, + sqliteType: () => 'text' // these get encoded as JSON arrays + }); } } } @@ -245,7 +253,10 @@ export class CustomTypeRegistry { case 'unknown': return true; case 'array': - return this.isParsedWithoutCustomTypesSupport(this.lookupType(type.innerId)); + return ( + type.separatorCharCode == pgwire.CHAR_CODE_COMMA && + this.isParsedWithoutCustomTypesSupport(this.lookupType(type.innerId)) + ); default: return false; } diff --git a/modules/module-postgres/test/src/pg_test.test.ts b/modules/module-postgres/test/src/pg_test.test.ts index 63abccb9c..a139cb7d9 100644 --- a/modules/module-postgres/test/src/pg_test.test.ts +++ b/modules/module-postgres/test/src/pg_test.test.ts @@ -10,7 +10,7 @@ import { import { describe, expect, test } from 'vitest'; import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js'; import { WalStream } from '@module/replication/WalStream.js'; -import { PostgresTypeCache } from '@module/types/custom.js'; +import { PostgresTypeCache } from '@module/types/cache.js'; describe('pg data types', () => { async function setupTable(db: pgwire.PgClient) { @@ -483,7 +483,8 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' id serial primary key, rating rating_value, composite composite, - nested_composite nested_composite + nested_composite nested_composite, + boxes box[] );`); const slotName = 'test_slot'; @@ -500,11 +501,12 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' await db.query(` INSERT INTO test_custom - (rating, composite, nested_composite) + (rating, composite, nested_composite, boxes) VALUES ( 1, (ARRAY[2,3], 'bar'), - (TRUE, (ARRAY[2,3], 'bar')) + (TRUE, (ARRAY[2,3], 'bar')), + ARRAY[box(point '(1,2)', point '(3,4)'), box(point '(5, 6)', point '(7,8)')] ); `); @@ -524,14 +526,16 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' expect(oldFormat).toMatchObject({ rating: '1', composite: '("{2,3}",bar)', - nested_composite: '(t,"(""{2,3}"",bar)")' + nested_composite: '(t,"(""{2,3}"",bar)")', + boxes: '["(3","4)","(1","2);(7","8)","(5","6)"]' }); const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)); expect(newFormat).toMatchObject({ rating: 1, composite: '{"foo":[2.0,3.0],"bar":"bar"}', - nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}' + nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}', + boxes: JSON.stringify(['(3,4),(1,2)', '(7,8),(5,6)']) }); } finally { await db.end(); diff --git a/packages/jpgwire/src/pgwire_types.ts b/packages/jpgwire/src/pgwire_types.ts index 557512c31..4f520d6bb 100644 --- a/packages/jpgwire/src/pgwire_types.ts +++ b/packages/jpgwire/src/pgwire_types.ts @@ -37,7 +37,7 @@ export enum PgTypeOid { // Generate using: // select '[' || typarray || ', ' || oid || '], // ' || typname from pg_catalog.pg_type WHERE typarray != 0; -const ARRAY_TO_ELEM_OID = new Map([ +export const ARRAY_TO_ELEM_OID = new Map([ [1000, 16], // bool [1001, 17], // bytea [1002, 18], // char From 3199e15eb24299ddc6624361dcd79978acdb28f4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 27 Aug 2025 16:18:45 +0200 Subject: [PATCH 06/14] Support ranges --- modules/module-postgres/src/types/cache.ts | 16 +- modules/module-postgres/src/types/registry.ts | 45 +-- .../module-postgres/test/src/pg_test.test.ts | 24 +- .../test/src/types/registry.test.ts | 25 ++ packages/jpgwire/src/index.ts | 2 +- packages/jpgwire/src/pgwire_types.ts | 16 +- packages/jpgwire/src/sequence_tokenizer.ts | 300 ------------------ packages/jpgwire/src/structure_parser.ts | 300 ++++++++++++++++++ .../jpgwire/test/sequence_tokenizer.test.ts | 141 -------- .../jpgwire/test/structure_parser.test.ts | 203 ++++++++++++ packages/sync-rules/src/ExpressionType.ts | 2 +- 11 files changed, 589 insertions(+), 485 deletions(-) delete mode 100644 packages/jpgwire/src/sequence_tokenizer.ts create mode 100644 packages/jpgwire/src/structure_parser.ts delete mode 100644 packages/jpgwire/test/sequence_tokenizer.test.ts create mode 100644 packages/jpgwire/test/structure_parser.test.ts diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/cache.ts index b8d7f185b..bb9de1d51 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -32,6 +32,8 @@ SELECT oid, t.typtype, FROM pg_attribute a WHERE a.attrelid = t.typrelid) ) + WHEN 'r' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngtypid = t.oid)) + WHEN 'm' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngmultitypid = t.oid)) ELSE NULL END AS desc FROM pg_type t @@ -59,9 +61,11 @@ WHERE t.oid = ANY($1) if (!this.registry.knows(oid)) { // This type is an array of another custom type. + const inner = Number(element_type); + requireType(inner); this.registry.set(oid, { type: 'array', - innerId: Number(element_type), + innerId: inner, separatorCharCode: (delim as string).charCodeAt(0), sqliteType: () => 'text' // Since it's JSON }); @@ -89,6 +93,15 @@ WHERE t.oid = ANY($1) this.registry.setDomainType(oid, inner); requireType(inner); break; + case 'r': + case 'm': { + const inner = Number(desc.inner); + this.registry.set(oid, { + type: row.typtype == 'r' ? 'range' : 'multirange', + innerId: inner, + sqliteType: () => 'text' // Since it's JSON + }); + } } } @@ -110,7 +123,6 @@ JOIN pg_namespace tn ON tn.oid = t.typnamespace WHERE a.attnum > 0 AND NOT a.attisdropped AND cn.nspname = $1 - AND tn.nspname NOT IN ('pg_catalog', 'information_schema'); `; const query = await this.pool.query({ statement: sql, params: [{ type: 'varchar', value: schema }] }); diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index 08550ec44..5ab6b9231 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -49,7 +49,17 @@ interface CompositeType extends BaseType { members: { name: string; typeId: number }[]; } -type KnownType = BuiltinType | ArrayType | DomainType | DomainType | CompositeType; +/** + * A type created with `CREATE TYPE AS RANGE`. + * + * Ranges are represented as {@link pgwire.Range}. Multiranges are represented as arrays thereof. + */ +interface RangeType extends BaseType { + type: 'range' | 'multirange'; + innerId: number; +} + +type KnownType = BuiltinType | ArrayType | DomainType | DomainType | CompositeType | RangeType; interface UnknownType extends BaseType { type: 'unknown'; @@ -199,23 +209,13 @@ export class CustomTypeRegistry { case 'composite': { const parsed: [string, any][] = []; - pgwire.decodeSequence({ - source: raw, - delimiters: pgwire.COMPOSITE_DELIMITERS, - listener: { - onValue: (raw) => { - const nextMember = resolved.members[parsed.length]; - if (nextMember) { - const value = raw == null ? null : this.decodeWithCustomTypes(raw, nextMember.typeId); - parsed.push([nextMember.name, value]); - } - }, - // These are only used for nested arrays - onStructureStart: () => {}, - onStructureEnd: () => {} + new pgwire.StructureParser(raw).parseComposite((raw) => { + const nextMember = resolved.members[parsed.length]; + if (nextMember) { + const value = raw == null ? null : this.decodeWithCustomTypes(raw, nextMember.typeId); + parsed.push([nextMember.name, value]); } }); - return Object.fromEntries(parsed); } case 'array': { @@ -234,12 +234,15 @@ export class CustomTypeRegistry { } } - return pgwire.decodeArray({ - source: raw, - decodeElement: (source) => this.decodeWithCustomTypes(source, innerId), - delimiterCharCode: resolved.separatorCharCode - }); + return new pgwire.StructureParser(raw).parseArray( + (source) => this.decodeWithCustomTypes(source, innerId), + resolved.separatorCharCode + ); } + case 'range': + return new pgwire.StructureParser(raw).parseRange((s) => this.decodeWithCustomTypes(s, resolved.innerId)); + case 'multirange': + return new pgwire.StructureParser(raw).parseMultiRange((s) => this.decodeWithCustomTypes(s, resolved.innerId)); } } diff --git a/modules/module-postgres/test/src/pg_test.test.ts b/modules/module-postgres/test/src/pg_test.test.ts index a139cb7d9..0d6367790 100644 --- a/modules/module-postgres/test/src/pg_test.test.ts +++ b/modules/module-postgres/test/src/pg_test.test.ts @@ -478,13 +478,16 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5);`); await db.query(`CREATE TYPE composite AS (foo rating_value[], bar TEXT);`); await db.query(`CREATE TYPE nested_composite AS (a BOOLEAN, b composite);`); + await db.query(`CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')`); await db.query(`CREATE TABLE test_custom( id serial primary key, rating rating_value, composite composite, nested_composite nested_composite, - boxes box[] + boxes box[], + mood mood, + ranges int4multirange[] );`); const slotName = 'test_slot'; @@ -501,12 +504,14 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' await db.query(` INSERT INTO test_custom - (rating, composite, nested_composite, boxes) + (rating, composite, nested_composite, boxes, mood, ranges) VALUES ( 1, (ARRAY[2,3], 'bar'), (TRUE, (ARRAY[2,3], 'bar')), - ARRAY[box(point '(1,2)', point '(3,4)'), box(point '(5, 6)', point '(7,8)')] + ARRAY[box(point '(1,2)', point '(3,4)'), box(point '(5, 6)', point '(7,8)')], + 'happy', + ARRAY[int4multirange(int4range(2, 4), int4range(5, 7, '(]'))]::int4multirange[] ); `); @@ -527,7 +532,9 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' rating: '1', composite: '("{2,3}",bar)', nested_composite: '(t,"(""{2,3}"",bar)")', - boxes: '["(3","4)","(1","2);(7","8)","(5","6)"]' + boxes: '["(3","4)","(1","2);(7","8)","(5","6)"]', + mood: 'happy', + ranges: '{"{[2,4),[6,8)}"}' }); const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)); @@ -535,7 +542,14 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' rating: 1, composite: '{"foo":[2.0,3.0],"bar":"bar"}', nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}', - boxes: JSON.stringify(['(3,4),(1,2)', '(7,8),(5,6)']) + boxes: JSON.stringify(['(3,4),(1,2)', '(7,8),(5,6)']), + mood: 'happy', + ranges: JSON.stringify([ + [ + { lower: 2, upper: 4, lower_exclusive: 0, upper_exclusive: 1 }, + { lower: 6, upper: 8, lower_exclusive: 0, upper_exclusive: 1 } + ] + ]) }); } finally { await db.end(); diff --git a/modules/module-postgres/test/src/types/registry.test.ts b/modules/module-postgres/test/src/types/registry.test.ts index 68b9d2871..d76f78f7e 100644 --- a/modules/module-postgres/test/src/types/registry.test.ts +++ b/modules/module-postgres/test/src/types/registry.test.ts @@ -121,4 +121,29 @@ describe('custom type registry', () => { // SELECT ROW(ARRAY[(FALSE,2)]::c2[])::c3; checkResult('("{""(f,2)""}")', 1339, '("{""(f,2)""}")', '{"c":[{"a":0,"b":2}]}'); }); + + test('range', () => { + registry.set(1337, { + type: 'range', + sqliteType: () => 'text', + innerId: PgTypeOid.INT2 + }); + + checkResult('[1,2]', 1337, '[1,2]', '{"lower":1,"upper":2,"lower_exclusive":0,"upper_exclusive":0}'); + }); + + test('multirange', () => { + registry.set(1337, { + type: 'multirange', + sqliteType: () => 'text', + innerId: PgTypeOid.INT2 + }); + + checkResult( + '{[1,2),[3,4)}', + 1337, + '{[1,2),[3,4)}', + '[{"lower":1,"upper":2,"lower_exclusive":0,"upper_exclusive":1},{"lower":3,"upper":4,"lower_exclusive":0,"upper_exclusive":1}]' + ); + }); }); diff --git a/packages/jpgwire/src/index.ts b/packages/jpgwire/src/index.ts index d7c5e461c..53a02fe44 100644 --- a/packages/jpgwire/src/index.ts +++ b/packages/jpgwire/src/index.ts @@ -3,4 +3,4 @@ export * from './certs.js'; export * from './util.js'; export * from './metrics.js'; export * from './pgwire_types.js'; -export * from './sequence_tokenizer.js'; +export * from './structure_parser.js'; diff --git a/packages/jpgwire/src/pgwire_types.ts b/packages/jpgwire/src/pgwire_types.ts index 4f520d6bb..7842ea452 100644 --- a/packages/jpgwire/src/pgwire_types.ts +++ b/packages/jpgwire/src/pgwire_types.ts @@ -3,16 +3,7 @@ import { JsonContainer } from '@powersync/service-jsonbig'; import { CustomSqliteValue, TimeValue, type DatabaseInputValue } from '@powersync/service-sync-rules'; import { dateToSqlite, lsnMakeComparable, timestampToSqlite, timestamptzToSqlite } from './util.js'; -import { - arrayDelimiters, - CHAR_CODE_COMMA, - CHAR_CODE_LEFT_BRACE, - CHAR_CODE_RIGHT_BRACE, - decodeArray, - decodeSequence, - Delimiters, - SequenceListener -} from './sequence_tokenizer.js'; +import { StructureParser } from './structure_parser.js'; export enum PgTypeOid { TEXT = 25, @@ -165,10 +156,7 @@ export class PgType { static _decodeArray(text: string, elemTypeOid: number): DatabaseInputValue[] { text = text.replace(/^\[.+=/, ''); // skip dimensions - return decodeArray({ - source: text, - decodeElement: (raw) => PgType.decode(raw, elemTypeOid) - }); + return new StructureParser(text).parseArray((raw) => (raw == null ? null : PgType.decode(raw, elemTypeOid))); } static _decodeBytea(text: string): Uint8Array { diff --git a/packages/jpgwire/src/sequence_tokenizer.ts b/packages/jpgwire/src/sequence_tokenizer.ts deleted file mode 100644 index 133098f1e..000000000 --- a/packages/jpgwire/src/sequence_tokenizer.ts +++ /dev/null @@ -1,300 +0,0 @@ -export interface SequenceListener { - /** - * Invoked whenever the tokenizer has begun decoding a structure (that is, once in the beginning and then for - * every sub-structure). - */ - onStructureStart: () => void; - /** - * Invoked whenever the tokenizer has finished parsing a value that isn't a nested structure. - * - * @param value the raw value, with escape characters related to the outer structure being removed. `null` for the - * literal text {@link Delimiters.nullLiteral}. - */ - onValue: (value: string | null) => void; - /** - * Invoked whenever a tokenizer has completed a structure (meaning that it's closing brace has been consumed). - */ - onStructureEnd: () => void; -} - -export interface Delimiters { - /** The char code opening the structure, e.g. `{` for arrays */ - openingCharCode: number; - /** The char code opening the structure, e.g. `}` for arrays */ - closingCharCode: number; - /** The char code opening the structure, e.g. `,` */ - delimiterCharCode: number; - /** - * Whether two subsequent double quotes are allowed to escape values. - * - * This is the case for composite values, but not for arrays. */ - allowEscapingWithDoubleDoubleQuote: boolean; - /** Whether empty values are allowed, e.g. `(,)` */ - allowEmpty: boolean; - /** The string literal that denotes a `NULL` value. */ - nullLiteral: string; - /** Whether values can be nested sub-structures. */ - multiDimensional: boolean; -} - -export interface DecodeSequenceOptions { - /** The original text to parse */ - source: string; - /** Delimiters for the outermost structure */ - delimiters: Delimiters; - /** Callbacks to control how values are interpreted and how substructures should be parsed. */ - listener: SequenceListener; -} - -/** - * Decodes a sequence of values, such as arrays or composite types represented as text. - * - * It supports nested arrays and different options for escaping values needed for arrays and composites. - */ -export function decodeSequence(options: DecodeSequenceOptions) { - let { source, delimiters, listener } = options; - - const stateStackTail: SequenceDecoderState[] = []; - let currentState: SequenceDecoderState = SequenceDecoderState.BEFORE_SEQUENCE as SequenceDecoderState; - - consumeChar: for (let i = 0; i < source.length; i++) { - function error(msg: string): never { - throw new Error(`Error decoding Postgres sequence at position ${i}: ${msg}`); - } - - function check(expected: number) { - if (charCode != expected) { - error(`Expected ${String.fromCharCode(expected)}, got ${String.fromCharCode(charCode)}`); - } - } - - function peek(): number { - if (i == source.length - 1) { - error('Unexpected end of input'); - } - - return source.charCodeAt(i + 1); - } - - function advance(): number { - const value = peek(); - i++; - return value; - } - - function quotedString(): string { - const start = i; - const charCodes: number[] = []; - let previousWasBackslash = false; - - while (true) { - const next = advance(); - if (previousWasBackslash) { - if (next != CHAR_CODE_DOUBLE_QUOTE && next != CHAR_CODE_BACKSLASH) { - error('Expected escaped double quote or escaped backslash'); - } - charCodes.push(next); - previousWasBackslash = false; - } else if (next == CHAR_CODE_DOUBLE_QUOTE) { - if (i != start && delimiters.allowEscapingWithDoubleDoubleQuote) { - // If the next character is also a double quote, that escapes a single double quote - if (i < source.length - 1 && peek() == CHAR_CODE_DOUBLE_QUOTE) { - i++; - charCodes.push(CHAR_CODE_DOUBLE_QUOTE); - continue; - } - } - - break; // End of string. - } else if (next == CHAR_CODE_BACKSLASH) { - previousWasBackslash = true; - } else { - charCodes.push(next); - } - } - - return String.fromCharCode(...charCodes); - } - - function unquotedString(): string { - const start = i; - let next = peek(); - while (next != delimiters.delimiterCharCode && next != delimiters.closingCharCode) { - if (next == delimiters.openingCharCode || next == CHAR_CODE_DOUBLE_QUOTE) { - error('illegal char, should require escaping'); - } - - i++; - next = peek(); - } - - return source.substring(start, i + 1); - } - - function endStructure() { - currentState = SequenceDecoderState.AFTER_SEQUENCE; - listener.onStructureEnd(); - if (stateStackTail.length > 0) { - currentState = stateStackTail.pop()!; - } - } - - const charCode = source.charCodeAt(i); - switch (currentState) { - case SequenceDecoderState.BEFORE_SEQUENCE: - check(delimiters.openingCharCode); - currentState = SequenceDecoderState.BEFORE_ELEMENT_OR_END; - listener.onStructureStart(); - break; - case SequenceDecoderState.BEFORE_ELEMENT_OR_END: - if (charCode == delimiters.closingCharCode) { - endStructure(); - continue consumeChar; - } - // No break between these, end has been handled. - case SequenceDecoderState.BEFORE_ELEMENT: - // What follows is either NULL, a non-empty string value that does not contain delimiters, or an escaped string - // value. - if (charCode == CHAR_CODE_DOUBLE_QUOTE) { - const value = quotedString(); - listener.onValue(value); - } else if (charCode == delimiters.delimiterCharCode || charCode == delimiters.closingCharCode) { - if (!delimiters.allowEmpty) { - error('invalid empty element'); - } - - listener.onValue('' == delimiters.nullLiteral ? null : ''); - if (charCode == delimiters.delimiterCharCode) { - // Since this is a comma, there'll be an element afterwards - currentState = SequenceDecoderState.BEFORE_ELEMENT; - } else { - endStructure(); - } - break; - } else { - if (delimiters.multiDimensional && charCode == delimiters.openingCharCode) { - currentState = SequenceDecoderState.AFTER_ELEMENT; - listener.onStructureStart(); - stateStackTail.push(currentState); - - // We've consumed the opening delimiter already, so the inner state can either parse an element or - // immediately close. - currentState = SequenceDecoderState.BEFORE_ELEMENT_OR_END; - continue consumeChar; - } else { - // Parse the current cell as one value - const value = unquotedString(); - listener.onValue(value == delimiters.nullLiteral ? null : value); - } - } - currentState = SequenceDecoderState.AFTER_ELEMENT; - break; - case SequenceDecoderState.AFTER_ELEMENT: - // There can be another element here, or a closing brace - if (charCode == delimiters.closingCharCode) { - endStructure(); - } else { - check(delimiters.delimiterCharCode); - currentState = SequenceDecoderState.BEFORE_ELEMENT; - } - break; - case SequenceDecoderState.AFTER_SEQUENCE: - error('Unexpected trailing text'); - default: - error('Internal error: Unknown state'); - } - } - - if (currentState != SequenceDecoderState.AFTER_SEQUENCE) { - throw Error('Unexpected end of input'); - } -} - -export type ElementOrArray = null | T | ElementOrArray[]; - -export interface DecodeArrayOptions { - source: string; - delimiterCharCode?: number; - decodeElement: (source: string) => T; -} - -/** - * A variant of {@link decodeSequence} that specifically decodes arrays. - * - * The {@link DecodeArrayOptions.decodeElement} method is responsible for parsing individual values with the array, - * this method automatically recognizes multidimensional arrays and parses them appropriately. - */ -export function decodeArray(options: DecodeArrayOptions): ElementOrArray[] { - let results: ElementOrArray[] = []; - const stack: ElementOrArray[][] = []; - - const listener: SequenceListener = { - onStructureStart: () => { - // We're parsing a new array - stack.push([]); - }, - onValue: function (value: string | null): void { - // Atomic (non-array) value, add to current array. - stack[stack.length - 1].push(value != null ? options.decodeElement(value) : null); - }, - onStructureEnd: () => { - // We're done parsing an array. - const subarray = stack.pop()!; - if (stack.length == 0) { - // We are done with the outermost array, set results. - results = subarray; - } else { - // We were busy parsing a nested array, continue outer array. - stack[stack.length - 1].push(subarray); - } - } - }; - - decodeSequence({ - source: options.source, - listener, - delimiters: arrayDelimiters(options.delimiterCharCode) - }); - - return results!; -} - -const CHAR_CODE_DOUBLE_QUOTE = 0x22; -const CHAR_CODE_BACKSLASH = 0x5c; -export const CHAR_CODE_COMMA = 0x2c; -export const CHAR_CODE_LEFT_BRACE = 0x7b; -export const CHAR_CODE_RIGHT_BRACE = 0x7d; -export const CHAR_CODE_LEFT_PAREN = 0x28; -export const CHAR_CODE_RIGHT_PAREN = 0x29; - -// https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO -export function arrayDelimiters(delimiterCharCode: number = CHAR_CODE_COMMA): Delimiters { - return { - openingCharCode: CHAR_CODE_LEFT_BRACE, - closingCharCode: CHAR_CODE_RIGHT_BRACE, - allowEscapingWithDoubleDoubleQuote: false, - nullLiteral: 'NULL', - allowEmpty: false, // Empty values must be escaped - delimiterCharCode, - multiDimensional: true - }; -} - -// https://www.postgresql.org/docs/current/rowtypes.html#ROWTYPES-IO-SYNTAX -export const COMPOSITE_DELIMITERS = Object.freeze({ - openingCharCode: CHAR_CODE_LEFT_PAREN, - closingCharCode: CHAR_CODE_RIGHT_PAREN, - delimiterCharCode: CHAR_CODE_COMMA, - allowEscapingWithDoubleDoubleQuote: true, - allowEmpty: true, // Empty values encode NULL - nullLiteral: '', - multiDimensional: false -} satisfies Delimiters); - -enum SequenceDecoderState { - BEFORE_SEQUENCE = 1, - BEFORE_ELEMENT_OR_END = 2, - BEFORE_ELEMENT = 3, - AFTER_ELEMENT = 4, - AFTER_SEQUENCE = 5 -} diff --git a/packages/jpgwire/src/structure_parser.ts b/packages/jpgwire/src/structure_parser.ts new file mode 100644 index 000000000..a9acfeba0 --- /dev/null +++ b/packages/jpgwire/src/structure_parser.ts @@ -0,0 +1,300 @@ +import { delimiter } from 'path'; + +/** + * Utility to parse encoded structural values, such as arrays, composite types, ranges and multiranges. + */ +export class StructureParser { + private offset: number; + + constructor(readonly source: string) { + this.offset = 0; + } + + private currentCharCode(): number { + return this.source.charCodeAt(this.offset); + } + + private get isAtEnd(): boolean { + return this.offset == this.source.length; + } + + private checkNotAtEnd() { + if (this.isAtEnd) { + this.error('Unexpected end of input'); + } + } + + private error(msg: string): never { + throw new Error(`Error decoding Postgres sequence at position ${this.offset}: ${msg}`); + } + + private check(expected: number) { + if (this.currentCharCode() != expected) { + this.error(`Expected ${String.fromCharCode(expected)}, got ${String.fromCharCode(this.currentCharCode())}`); + } + } + + private peek(): number { + this.checkNotAtEnd(); + + return this.source.charCodeAt(this.offset + 1); + } + + private advance() { + this.checkNotAtEnd(); + this.offset++; + } + + private consume(expected: number) { + this.check(expected); + this.advance(); + } + + private maybeConsume(expected: number): boolean { + if (this.currentCharCode() == expected) { + this.advance(); + return true; + } else { + return false; + } + } + + /** + * Assuming that the current position contains a opening double quote for an escaped string, parses the value until + * the closing quote. + * + * The returned value applies escape characters, so `"foo\"bar"` would return the string `foo"bar"`. + */ + private quotedString(allowEscapingWithDoubleDoubleQuote: boolean = false): string { + this.consume(CHAR_CODE_DOUBLE_QUOTE); + + const start = this.offset; + const charCodes: number[] = []; + let previousWasBackslash = false; + + while (true) { + const char = this.currentCharCode(); + + if (previousWasBackslash) { + if (char != CHAR_CODE_DOUBLE_QUOTE && char != CHAR_CODE_BACKSLASH) { + this.error('Expected escaped double quote or escaped backslash'); + } + charCodes.push(char); + previousWasBackslash = false; + } else if (char == CHAR_CODE_DOUBLE_QUOTE) { + if (this.offset != start && allowEscapingWithDoubleDoubleQuote) { + // If the next character is also a double quote, that escapes a single double quote + if (this.offset < this.source.length - 1 && this.peek() == CHAR_CODE_DOUBLE_QUOTE) { + this.offset += 2; + charCodes.push(CHAR_CODE_DOUBLE_QUOTE); + continue; + } + } + + break; // End of string. + } else if (char == CHAR_CODE_BACKSLASH) { + previousWasBackslash = true; + } else { + charCodes.push(char); + } + + this.advance(); + } + + this.consume(CHAR_CODE_DOUBLE_QUOTE); + return String.fromCharCode(...charCodes); + } + + unquotedString(endedBy: number[], illegal: number[]): string { + const start = this.offset; + this.advance(); + + let next = this.currentCharCode(); + while (endedBy.indexOf(next) == -1) { + if (illegal.indexOf(next) != -1) { + this.error('illegal char, should require escaping'); + } + + this.advance(); + next = this.currentCharCode(); + } + + return this.source.substring(start, this.offset); + } + + checkAtEnd() { + if (this.offset < this.source.length) { + this.error('Unexpected trailing text'); + } + } + + // https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO + parseArray(parseElement: (value: string) => T, delimiter: number = CHAR_CODE_COMMA): ElementOrArray[] { + const array = this.parseArrayInner(delimiter, parseElement); + this.checkAtEnd(); + return array; + } + + // Recursively parses a (potentially multi-dimensional) array. + private parseArrayInner(delimiter: number, parseElement: (value: string) => T): ElementOrArray[] { + this.consume(CHAR_CODE_LEFT_BRACE); + if (this.maybeConsume(CHAR_CODE_RIGHT_BRACE)) { + return []; // Empty array ({}) + } + + const elements: ElementOrArray[] = []; + do { + // Parse a value in the array. This can either be an escaped string, an unescaped string, or a nested array. + const currentChar = this.currentCharCode(); + if (currentChar == CHAR_CODE_LEFT_BRACE) { + // Nested array + elements.push(this.parseArrayInner(delimiter, parseElement)); + } else if (currentChar == CHAR_CODE_DOUBLE_QUOTE) { + elements.push(parseElement(this.quotedString())); + } else { + const value = this.unquotedString( + [delimiter, CHAR_CODE_RIGHT_BRACE], + [CHAR_CODE_DOUBLE_QUOTE, CHAR_CODE_LEFT_BRACE] + ); + elements.push(value == 'NULL' ? null : parseElement(value)); + } + } while (this.maybeConsume(delimiter)); + + this.consume(CHAR_CODE_RIGHT_BRACE); + return elements; + } + + // https://www.postgresql.org/docs/current/rowtypes.html#ROWTYPES-IO-SYNTAX + parseComposite(onElement: (value: string | null) => void) { + this.consume(CHAR_CODE_LEFT_PAREN); + do { + // Parse a composite value. This can either be an escaped string, an unescaped string, or an empty string. + const currentChar = this.currentCharCode(); + if (currentChar == CHAR_CODE_COMMA) { + // Empty value. The comma is consumed by the while() below. + onElement(null); + } else if (currentChar == CHAR_CODE_RIGHT_PAREN) { + // Empty value before end. The right parent is consumed by the line after the loop. + onElement(null); + } else if (currentChar == CHAR_CODE_DOUBLE_QUOTE) { + onElement(this.quotedString(true)); + } else { + const value = this.unquotedString( + [CHAR_CODE_COMMA, CHAR_CODE_RIGHT_PAREN], + [CHAR_CODE_DOUBLE_QUOTE, CHAR_CODE_LEFT_PAREN] + ); + onElement(value); + } + } while (this.maybeConsume(CHAR_CODE_COMMA)); + this.consume(CHAR_CODE_RIGHT_PAREN); + this.checkAtEnd(); + } + + // https://www.postgresql.org/docs/current/rangetypes.html#RANGETYPES-IO + private parseRangeInner(parseInner: (value: string) => T): Range { + const empty = 'empty'; + + // Parse [ or ( to start the range + let lowerBoundExclusive; + switch (this.currentCharCode()) { + case CHAR_CODE_LEFT_PAREN: + lowerBoundExclusive = true; + this.advance(); + break; + case CHAR_CODE_LEFT_BRACKET: + lowerBoundExclusive = false; + this.advance(); + break; + case empty.charCodeAt(0): + // Consume the string "empty" + for (let i = 0; i < empty.length; i++) { + this.consume(empty.charCodeAt(i)); + } + return empty; + default: + this.error('Expected [, ( or string empty'); + } + + // Parse value until comma (which may be empty) + let lower = null; + if (this.currentCharCode() == CHAR_CODE_DOUBLE_QUOTE) { + lower = parseInner(this.quotedString()); + } else if (this.currentCharCode() != CHAR_CODE_COMMA) { + lower = parseInner(this.unquotedString([CHAR_CODE_COMMA], [])); + } + + this.consume(CHAR_CODE_COMMA); + + let upper = null; + if (this.currentCharCode() == CHAR_CODE_DOUBLE_QUOTE) { + upper = parseInner(this.quotedString()); + } else if (this.currentCharCode() != CHAR_CODE_RIGHT_PAREN && this.currentCharCode() != CHAR_CODE_RIGHT_BRACKET) { + upper = parseInner(this.unquotedString([CHAR_CODE_RIGHT_PAREN, CHAR_CODE_RIGHT_BRACKET], [])); + } + + let upperBoundExclusive; + switch (this.currentCharCode()) { + case CHAR_CODE_RIGHT_PAREN: + upperBoundExclusive = true; + this.advance(); + break; + case CHAR_CODE_RIGHT_BRACKET: + upperBoundExclusive = false; + this.advance(); + break; + default: + this.error('Expected ] or )'); + } + + return { + lower: lower, + upper: upper, + lower_exclusive: lowerBoundExclusive, + upper_exclusive: upperBoundExclusive + }; + } + + parseRange(parseInner: (value: string) => T): Range { + const range = this.parseRangeInner(parseInner); + this.checkAtEnd(); + return range; + } + + parseMultiRange(parseInner: (value: string) => T): Range[] { + this.consume(CHAR_CODE_LEFT_BRACE); + if (this.maybeConsume(CHAR_CODE_RIGHT_BRACE)) { + return []; + } + + const values: Range[] = []; + do { + values.push(this.parseRangeInner(parseInner)); + } while (this.maybeConsume(CHAR_CODE_COMMA)); + + this.consume(CHAR_CODE_RIGHT_BRACE); + this.checkAtEnd(); + return values; + } +} + +export type Range = + | { + lower: T | null; + upper: T | null; + lower_exclusive: boolean; + upper_exclusive: boolean; + } + | 'empty'; + +export type ElementOrArray = null | T | ElementOrArray[]; + +const CHAR_CODE_DOUBLE_QUOTE = 0x22; +const CHAR_CODE_BACKSLASH = 0x5c; +export const CHAR_CODE_COMMA = 0x2c; +export const CHAR_CODE_SEMICOLON = 0x3b; +const CHAR_CODE_LEFT_BRACE = 0x7b; +const CHAR_CODE_RIGHT_BRACE = 0x7d; +const CHAR_CODE_LEFT_PAREN = 0x28; +const CHAR_CODE_RIGHT_PAREN = 0x29; +const CHAR_CODE_LEFT_BRACKET = 0x5b; +const CHAR_CODE_RIGHT_BRACKET = 0x5d; diff --git a/packages/jpgwire/test/sequence_tokenizer.test.ts b/packages/jpgwire/test/sequence_tokenizer.test.ts deleted file mode 100644 index 0eb9be0dd..000000000 --- a/packages/jpgwire/test/sequence_tokenizer.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { - SequenceListener, - Delimiters, - CHAR_CODE_LEFT_BRACE, - decodeSequence, - arrayDelimiters, - COMPOSITE_DELIMITERS -} from '../src/index'; - -test('empty array', () => { - expect(recordParseEvents('{}', arrayDelimiters())).toStrictEqual(['structureStart', 'structureEnd']); -}); - -test('regular array', () => { - expect(recordParseEvents('{foo,bar}', arrayDelimiters())).toStrictEqual([ - 'structureStart', - 'foo', - 'bar', - 'structureEnd' - ]); -}); - -test('null', () => { - expect(recordParseEvents('{NULL}', arrayDelimiters())).toStrictEqual(['structureStart', null, 'structureEnd']); - expect(recordParseEvents('{null}', arrayDelimiters())).toStrictEqual(['structureStart', 'null', 'structureEnd']); -}); - -test('escaped', () => { - expect(recordParseEvents('{""}', arrayDelimiters())).toStrictEqual(['structureStart', '', 'structureEnd']); - expect(recordParseEvents('{"foo"}', arrayDelimiters())).toStrictEqual(['structureStart', 'foo', 'structureEnd']); - expect(recordParseEvents('{"fo\\"o,"}', arrayDelimiters())).toStrictEqual([ - 'structureStart', - 'fo"o,', - 'structureEnd' - ]); - expect(recordParseEvents('{"fo\\\\o,"}', arrayDelimiters())).toStrictEqual([ - 'structureStart', - 'fo\\o,', - 'structureEnd' - ]); -}); - -test('nested array', () => { - expect( - recordParseEvents('{0,{0,{}}}', arrayDelimiters(), (c) => (c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters() : null)) - ).toStrictEqual([ - 'structureStart', - '0', - 'structureStart', - '0', - 'structureStart', - 'structureEnd', - 'structureEnd', - 'structureEnd' - ]); -}); - -test('other structures', () => { - expect(recordParseEvents('()', COMPOSITE_DELIMITERS)).toStrictEqual(['structureStart', 'structureEnd']); - expect( - recordParseEvents('(foo,bar,{baz})', COMPOSITE_DELIMITERS, (c) => - c == CHAR_CODE_LEFT_BRACE ? arrayDelimiters() : null - ) - ).toStrictEqual(['structureStart', 'foo', 'bar', 'structureStart', 'baz', 'structureEnd', 'structureEnd']); -}); - -test('composite null entries', () => { - // CREATE TYPE nested AS (a BOOLEAN, b BOOLEAN); SELECT (NULL, NULL)::nested; - expect(recordParseEvents('(,)', COMPOSITE_DELIMITERS)).toStrictEqual(['structureStart', null, null, 'structureEnd']); - - // CREATE TYPE triple AS (a BOOLEAN, b BOOLEAN, c BOOLEAN); SELECT (NULL, NULL, NULL)::triple - expect(recordParseEvents('(,,)', COMPOSITE_DELIMITERS)).toStrictEqual([ - 'structureStart', - null, - null, - null, - 'structureEnd' - ]); - - // NOTE: It looks like a single-element composite type has (NULL) encoded as NULL instead of a string like () -}); - -test('composite string escaping', () => { - expect(recordParseEvents('("foo""bar")', COMPOSITE_DELIMITERS)).toStrictEqual([ - 'structureStart', - 'foo"bar', - 'structureEnd' - ]); - - expect(recordParseEvents('("")', COMPOSITE_DELIMITERS)).toStrictEqual(['structureStart', '', 'structureEnd']); -}); - -describe('errors', () => { - test('unclosed array', () => { - expect(() => recordParseEvents('{', arrayDelimiters())).toThrow(/Unexpected end of input/); - }); - - test('trailing data', () => { - expect(() => recordParseEvents('{foo,bar}baz', arrayDelimiters())).toThrow(/Unexpected trailing text/); - }); - - test('improper escaped string', () => { - expect(() => recordParseEvents('{foo,"bar}', arrayDelimiters())).toThrow(/Unexpected end of input/); - }); - - test('illegal escape sequence', () => { - expect(() => recordParseEvents('{foo,"b\\ar"}', arrayDelimiters())).toThrow( - /Expected escaped double quote or escaped backslash/ - ); - }); - - test('illegal delimiter in value', () => { - expect(() => recordParseEvents('{foo{}', arrayDelimiters())).toThrow(/illegal char, should require escaping/); - }); - - test('illegal quote in value', () => { - expect(() => recordParseEvents('{foo"}', arrayDelimiters())).toThrow(/illegal char, should require escaping/); - }); -}); - -function recordParseEvents( - source: string, - delimiters: Delimiters, - maybeParseSubStructure?: (firstChar: number) => Delimiters | null -) { - maybeParseSubStructure ??= (_) => null; - - const events: any[] = []; - const listener: SequenceListener = { - maybeParseSubStructure, - onStructureStart: () => events.push('structureStart'), - onValue: (value) => { - events.push(value); - }, - onStructureEnd: () => events.push('structureEnd') - }; - - decodeSequence({ source, delimiters, listener }); - return events; -} diff --git a/packages/jpgwire/test/structure_parser.test.ts b/packages/jpgwire/test/structure_parser.test.ts new file mode 100644 index 000000000..d8dbbfc41 --- /dev/null +++ b/packages/jpgwire/test/structure_parser.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, test } from 'vitest'; +import { CHAR_CODE_COMMA, CHAR_CODE_SEMICOLON, StructureParser } from '../src/index'; + +describe('StructureParser', () => { + describe('array', () => { + const parseArray = (source: string, delimiter: number = CHAR_CODE_COMMA) => { + return new StructureParser(source).parseArray((source) => source, delimiter); + }; + + test('empty', () => { + expect(parseArray('{}')).toStrictEqual([]); + }); + + test('regular', () => { + expect(parseArray('{foo,bar}')).toStrictEqual(['foo', 'bar']); + }); + + test('custom delimiter', () => { + expect(parseArray('{foo;bar}', CHAR_CODE_SEMICOLON)).toStrictEqual(['foo', 'bar']); + }); + + test('null elements', () => { + expect(parseArray('{null}')).toStrictEqual(['null']); + expect(parseArray('{NULL}')).toStrictEqual([null]); + }); + + test('escaped', () => { + expect(parseArray('{""}')).toStrictEqual(['']); + expect(parseArray('{"foo"}')).toStrictEqual(['foo']); + expect(parseArray('{"fo\\"o"}')).toStrictEqual(['fo"o']); + expect(parseArray('{"fo\\\\o"}')).toStrictEqual(['fo\\o']); + }); + + test('nested', () => { + expect(parseArray('{0,{0,{}}}')).toStrictEqual(['0', ['0', []]]); + }); + + test('trailing data', () => { + expect(() => parseArray('{foo}bar')).toThrow(/Unexpected trailing text/); + }); + + test('unclosed array', () => { + expect(() => parseArray('{')).toThrow(/Unexpected end of input/); + }); + + test('improper escaped string', () => { + expect(() => parseArray('{foo,"bar}')).toThrow(/Unexpected end of input/); + }); + + test('illegal escape sequence', () => { + expect(() => parseArray('{foo,"b\\ar"}')).toThrow(/Expected escaped double quote or escaped backslash/); + }); + + test('illegal delimiter in value', () => { + expect(() => parseArray('{foo{}')).toThrow(/illegal char, should require escaping/); + }); + + test('illegal quote in value', () => { + expect(() => parseArray('{foo"}')).toThrow(/illegal char, should require escaping/); + }); + }); + + describe('composite', () => { + const parseComposite = (source: string) => { + const events: any[] = []; + new StructureParser(source).parseComposite((e) => events.push(e)); + return events; + }; + + test('empty composite', () => { + // Both of the following render as '()': + // create type foo as (); select ROW()::foo; create type foo2 as (foo integer); + // SELECT ROW()::foo, ROW(NULL)::foo2; + // Here, we resolve the ambiguity by parsing () as an one-element composite - callers need to be aware of this. + expect(parseComposite('()')).toStrictEqual([null]); + }); + + test('only null entries', () => { + expect(parseComposite('(,)')).toStrictEqual([null, null]); + expect(parseComposite('(,,)')).toStrictEqual([null, null, null]); + }); + + test('null before element', () => { + expect(parseComposite('(,foo)')).toStrictEqual([null, 'foo']); + }); + + test('null after element', () => { + expect(parseComposite('(foo,)')).toStrictEqual(['foo', null]); + }); + + test('nested', () => { + expect(parseComposite('(foo,bar,{baz})')).toStrictEqual(['foo', 'bar', '{baz}']); + }); + + test('escaped strings', () => { + expect(parseComposite('("foo""bar")')).toStrictEqual(['foo"bar']); + expect(parseComposite('("")')).toStrictEqual(['']); + }); + }); + + describe('range', () => { + const parseIntRange = (source: string) => { + return new StructureParser(source).parseRange((source) => Number(source)); + }; + + test('empty', () => { + // select '(3, 3)'::int4range + expect(parseIntRange('empty')).toStrictEqual('empty'); + }); + + test('regular', () => { + expect(parseIntRange('[1,2]')).toStrictEqual({ + lower: 1, + upper: 2, + lower_exclusive: false, + upper_exclusive: false + }); + expect(parseIntRange('[1,2)')).toStrictEqual({ + lower: 1, + upper: 2, + lower_exclusive: false, + upper_exclusive: true + }); + expect(parseIntRange('(1,2]')).toStrictEqual({ + lower: 1, + upper: 2, + lower_exclusive: true, + upper_exclusive: false + }); + expect(parseIntRange('(1,2)')).toStrictEqual({ + lower: 1, + upper: 2, + lower_exclusive: true, + upper_exclusive: true + }); + }); + + test('no lower bound', () => { + expect(parseIntRange('(,3]')).toStrictEqual({ + lower: null, + upper: 3, + lower_exclusive: true, + upper_exclusive: false + }); + }); + + test('no upper bound', () => { + expect(parseIntRange('(3,]')).toStrictEqual({ + lower: 3, + upper: null, + lower_exclusive: true, + upper_exclusive: false + }); + }); + + test('no bounds', () => { + expect(parseIntRange('(,)')).toStrictEqual({ + lower: null, + upper: null, + lower_exclusive: true, + upper_exclusive: true + }); + }); + }); + + describe('multirange', () => { + const parseIntMultiRange = (source: string) => { + return new StructureParser(source).parseMultiRange((source) => Number(source)); + }; + + test('empty', () => { + expect(parseIntMultiRange('{}')).toStrictEqual([]); + }); + + test('single', () => { + expect(parseIntMultiRange('{[3,7)}')).toStrictEqual([ + { + lower: 3, + upper: 7, + lower_exclusive: false, + upper_exclusive: true + } + ]); + }); + + test('multiple', () => { + expect(parseIntMultiRange('{[3,7),[8,9)}')).toStrictEqual([ + { + lower: 3, + upper: 7, + lower_exclusive: false, + upper_exclusive: true + }, + { + lower: 8, + upper: 9, + lower_exclusive: false, + upper_exclusive: true + } + ]); + }); + }); +}); diff --git a/packages/sync-rules/src/ExpressionType.ts b/packages/sync-rules/src/ExpressionType.ts index 3e625c717..9e120cb36 100644 --- a/packages/sync-rules/src/ExpressionType.ts +++ b/packages/sync-rules/src/ExpressionType.ts @@ -77,7 +77,7 @@ export class ExpressionType { } /** - * Here only for backwards-compatibility only. + * @deprecated Here only for backwards-compatibility only. */ export function expressionTypeFromPostgresType(type: string | undefined): ExpressionType { if (type?.endsWith('[]')) { From f67446043fbe83915e6ad60dd6441fcd9a548985 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 27 Aug 2025 16:34:05 +0200 Subject: [PATCH 07/14] Fix building tests --- packages/jpgwire/test/tsconfig.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/jpgwire/test/tsconfig.json diff --git a/packages/jpgwire/test/tsconfig.json b/packages/jpgwire/test/tsconfig.json new file mode 100644 index 000000000..5b5f74483 --- /dev/null +++ b/packages/jpgwire/test/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "noEmit": true, + "baseUrl": "./", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../" + } + ] +} From 14e9c27914e0c1be6a73c5ef734d09d2a61f80f8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 27 Aug 2025 17:35:41 +0200 Subject: [PATCH 08/14] Fix postgres tests --- modules/module-postgres/src/types/cache.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/cache.ts index bb9de1d51..ff689c8a0 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -62,13 +62,16 @@ WHERE t.oid = ANY($1) if (!this.registry.knows(oid)) { // This type is an array of another custom type. const inner = Number(element_type); - requireType(inner); - this.registry.set(oid, { - type: 'array', - innerId: inner, - separatorCharCode: (delim as string).charCodeAt(0), - sqliteType: () => 'text' // Since it's JSON - }); + if (inner != 0) { + // Some array types like macaddr[] don't seem to have their inner type set properly - skip! + requireType(inner); + this.registry.set(oid, { + type: 'array', + innerId: inner, + separatorCharCode: (delim as string).charCodeAt(0), + sqliteType: () => 'text' // Since it's JSON + }); + } } break; case 'c': From 124a5927078513c6b285819d30dd81ad6c2cb688 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 28 Aug 2025 09:12:46 +0200 Subject: [PATCH 09/14] Actually fetch types --- .../src/replication/WalStream.ts | 6 ++--- .../test/src/schema_changes.test.ts | 20 ++++++++++++++++ .../test/src/wal_stream.test.ts | 24 +++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index fc4419812..ee34b6d0f 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -710,6 +710,9 @@ WHERE oid = $1::regclass`, // Drop conflicting tables. This includes for example renamed tables. await batch.drop(result.dropTables); + // Ensure we have a description for custom types referenced in the table. + await this.connections.types.fetchTypes(referencedTypeIds); + // Snapshot if: // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere) // 2. Snapshot is not already done, AND: @@ -720,9 +723,6 @@ WHERE oid = $1::regclass`, // Truncate this table, in case a previous snapshot was interrupted. await batch.truncate([result.table]); - // Ensure we have a description for custom types referenced in the table. - await this.connections.types.fetchTypes(referencedTypeIds); - // Start the snapshot inside a transaction. // We use a dedicated connection for this. const db = await this.connections.snapshotConnection(); diff --git a/modules/module-postgres/test/src/schema_changes.test.ts b/modules/module-postgres/test/src/schema_changes.test.ts index 26a4fa3c8..356545a17 100644 --- a/modules/module-postgres/test/src/schema_changes.test.ts +++ b/modules/module-postgres/test/src/schema_changes.test.ts @@ -590,4 +590,24 @@ function defineTests(factory: storage.TestStorageFactory) { expect(failures).toEqual([]); }); + + test('custom types', async () => { + await using context = await WalStreamTestContext.open(factory); + + await context.updateSyncRules(` +streams: + stream: + query: SELECT id, * FROM "test_data" + +config: + edition: 2 +`); + + const { pool } = context; + await pool.query(`DROP TABLE IF EXISTS test_data`); + await pool.query(`CREATE TABLE test_data(id text primary key, description composite);`); + await pool.query(`INSERT INTO test_data(id, description) VALUES ('t1', ROW(TRUE, 2)::composite)`); + + await context.replicateSnapshot(); + }); } diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index 53f426171..9827c4e16 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -324,4 +324,28 @@ bucket_definitions: // creating a new replication slot. } }); + + test('custom types', async () => { + await using context = await WalStreamTestContext.open(factory); + + await context.updateSyncRules(` +streams: + stream: + query: SELECT id, * FROM "test_data" + +config: + edition: 2 +`); + + const { pool } = context; + await pool.query(`DROP TABLE IF EXISTS test_data`); + await pool.query(`CREATE TYPE composite AS (foo bool, bar int4);`); + await pool.query(`CREATE TABLE test_data(id text primary key, description composite);`); + + await context.initializeReplication(); + await pool.query(`INSERT INTO test_data(id, description) VALUES ('t1', ROW(TRUE, 2)::composite)`); + + const data = await context.getBucketData('1#stream|0[]'); + expect(data).toMatchObject([putOp('test_data', { id: 't1', description: 'test1' })]); + }); } From bbde900cc4a7fb94bda9e59dedd4d37651a08b05 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 28 Aug 2025 10:42:19 +0200 Subject: [PATCH 10/14] Try restoring support for older Postgres versions --- .../src/replication/PgManager.ts | 7 +- .../src/replication/WalStream.ts | 2 +- modules/module-postgres/src/types/cache.ts | 31 +++++++- .../src/utils/postgres_version.ts | 8 ++ .../module-postgres/test/src/pg_test.test.ts | 77 ++++++++++++++++--- .../test/src/schema_changes.test.ts | 4 +- .../test/src/wal_stream.test.ts | 4 +- 7 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 modules/module-postgres/src/utils/postgres_version.ts diff --git a/modules/module-postgres/src/replication/PgManager.ts b/modules/module-postgres/src/replication/PgManager.ts index 6acf9638d..244714a3a 100644 --- a/modules/module-postgres/src/replication/PgManager.ts +++ b/modules/module-postgres/src/replication/PgManager.ts @@ -3,6 +3,7 @@ import semver from 'semver'; import { NormalizedPostgresConnectionConfig } from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; import { PostgresTypeCache } from '../types/cache.js'; +import { getServerVersion } from '../utils/postgres_version.js'; /** * Shorter timeout for snapshot connections than for replication connections. @@ -25,7 +26,7 @@ export class PgManager { ) { // The pool is lazy - no connections are opened until a query is performed. this.pool = pgwire.connectPgWirePool(this.options, poolOptions); - this.types = new PostgresTypeCache(this.pool); + this.types = new PostgresTypeCache(this.pool, () => this.getServerVersion()); } public get connectionTag() { @@ -45,9 +46,7 @@ export class PgManager { * @returns The Postgres server version in a parsed Semver instance */ async getServerVersion(): Promise { - const result = await this.pool.query(`SHOW server_version;`); - // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)" - return semver.coerce(result.rows[0][0].split(' ')[0]); + return await getServerVersion(this.pool); } /** diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index ee34b6d0f..428c46c73 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -711,7 +711,7 @@ WHERE oid = $1::regclass`, await batch.drop(result.dropTables); // Ensure we have a description for custom types referenced in the table. - await this.connections.types.fetchTypes(referencedTypeIds); + //await this.connections.types.fetchTypes(referencedTypeIds); // Snapshot if: // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere) diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/cache.ts index ff689c8a0..a099015bd 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -1,17 +1,39 @@ import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules'; import * as pgwire from '@powersync/service-jpgwire'; import { CustomTypeRegistry } from './registry.js'; +import semver from 'semver'; /** * A cache of custom types for which information can be crawled from the source database. */ export class PostgresTypeCache { readonly registry: CustomTypeRegistry; + private cachedVersion: semver.SemVer | null = null; - constructor(private readonly pool: pgwire.PgClient) { + constructor( + private readonly pool: pgwire.PgClient, + private readonly getVersion: () => Promise + ) { this.registry = new CustomTypeRegistry(); } + private async fetchVersion(): Promise { + if (this.cachedVersion == null) { + this.cachedVersion = (await this.getVersion()) ?? semver.parse('0.0.1'); + } + + return this.cachedVersion!; + } + + /** + * @returns Whether the Postgres instance this type cache is connected to has support for the multirange type (which + * is the case for Postgres 14 and later). + */ + async supportsMultiRanges() { + const version = await this.fetchVersion(); + return version.compare(PostgresTypeCache.minVersionForMultirange) >= 0; + } + /** * Fetches information about indicated types. * @@ -19,8 +41,11 @@ export class PostgresTypeCache { * automatically crawled as well. */ public async fetchTypes(oids: number[]) { + const multiRangeSupport = await this.supportsMultiRanges(); + let pending = oids.filter((id) => !this.registry.knows(id)); // For details on columns, see https://www.postgresql.org/docs/current/catalog-pg-type.html + const multiRangeDesc = `WHEN 'm' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngmultitypid = t.oid))`; const statement = ` SELECT oid, t.typtype, CASE t.typtype @@ -33,7 +58,7 @@ SELECT oid, t.typtype, WHERE a.attrelid = t.typrelid) ) WHEN 'r' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngtypid = t.oid)) - WHEN 'm' THEN json_build_object('inner', (SELECT rngsubtype FROM pg_range WHERE rngmultitypid = t.oid)) + ${multiRangeSupport ? multiRangeDesc : ''} ELSE NULL END AS desc FROM pg_type t @@ -178,4 +203,6 @@ WHERE a.attnum > 0 } return result; } + + private static minVersionForMultirange: semver.SemVer = semver.parse('14.0.0')!; } diff --git a/modules/module-postgres/src/utils/postgres_version.ts b/modules/module-postgres/src/utils/postgres_version.ts new file mode 100644 index 000000000..7e2a7e9ce --- /dev/null +++ b/modules/module-postgres/src/utils/postgres_version.ts @@ -0,0 +1,8 @@ +import * as pgwire from '@powersync/service-jpgwire'; +import semver, { type SemVer } from 'semver'; + +export async function getServerVersion(db: pgwire.PgClient): Promise { + const result = await db.query(`SHOW server_version;`); + // The result is usually of the form "16.2 (Debian 16.2-1.pgdg120+2)" + return semver.coerce(result.rows[0][0].split(' ')[0]); +} diff --git a/modules/module-postgres/test/src/pg_test.test.ts b/modules/module-postgres/test/src/pg_test.test.ts index 0d6367790..782adf7ac 100644 --- a/modules/module-postgres/test/src/pg_test.test.ts +++ b/modules/module-postgres/test/src/pg_test.test.ts @@ -11,6 +11,7 @@ import { describe, expect, test } from 'vitest'; import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js'; import { WalStream } from '@module/replication/WalStream.js'; import { PostgresTypeCache } from '@module/types/cache.js'; +import { getServerVersion } from '@module/utils/postgres_version.js'; describe('pg data types', () => { async function setupTable(db: pgwire.PgClient) { @@ -486,8 +487,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' composite composite, nested_composite nested_composite, boxes box[], - mood mood, - ranges int4multirange[] + mood mood );`); const slotName = 'test_slot'; @@ -504,14 +504,13 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' await db.query(` INSERT INTO test_custom - (rating, composite, nested_composite, boxes, mood, ranges) + (rating, composite, nested_composite, boxes, mood) VALUES ( 1, (ARRAY[2,3], 'bar'), (TRUE, (ARRAY[2,3], 'bar')), ARRAY[box(point '(1,2)', point '(3,4)'), box(point '(5, 6)', point '(7,8)')], - 'happy', - ARRAY[int4multirange(int4range(2, 4), int4range(5, 7, '(]'))]::int4multirange[] + 'happy' ); `); @@ -533,8 +532,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' composite: '("{2,3}",bar)', nested_composite: '(t,"(""{2,3}"",bar)")', boxes: '["(3","4)","(1","2);(7","8)","(5","6)"]', - mood: 'happy', - ranges: '{"{[2,4),[6,8)}"}' + mood: 'happy' }); const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)); @@ -543,7 +541,68 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' composite: '{"foo":[2.0,3.0],"bar":"bar"}', nested_composite: '{"a":1,"b":{"foo":[2.0,3.0],"bar":"bar"}}', boxes: JSON.stringify(['(3,4),(1,2)', '(7,8),(5,6)']), - mood: 'happy', + mood: 'happy' + }); + } finally { + await db.end(); + } + }); + + test('test replication - multiranges', async () => { + const db = await connectPgPool(); + + if (!(await new PostgresTypeCache(db, () => getServerVersion(db)).supportsMultiRanges())) { + // This test requires Postgres 14 or later. + return; + } + + try { + await clearTestDb(db); + + await db.query(`CREATE TABLE test_custom( + id serial primary key, + ranges int4multirange[] + );`); + + const slotName = 'test_slot'; + + await db.query({ + statement: 'SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = $1', + params: [{ type: 'varchar', value: slotName }] + }); + + await db.query({ + statement: `SELECT slot_name, lsn FROM pg_catalog.pg_create_logical_replication_slot($1, 'pgoutput')`, + params: [{ type: 'varchar', value: slotName }] + }); + + await db.query(` + INSERT INTO test_custom + (ranges) + VALUES ( + ARRAY[int4multirange(int4range(2, 4), int4range(5, 7, '(]'))]::int4multirange[] + ); + `); + + const pg: pgwire.PgConnection = await pgwire.pgconnect({ replication: 'database' }, TEST_URI); + const replicationStream = await pg.logicalReplication({ + slot: slotName, + options: { + proto_version: '1', + publication_names: 'powersync' + } + }); + + const [transformed] = await getReplicationTx(db, replicationStream); + await pg.end(); + + const oldFormat = applyRowContext(transformed, CompatibilityContext.FULL_BACKWARDS_COMPATIBILITY); + expect(oldFormat).toMatchObject({ + ranges: '{"{[2,4),[6,8)}"}' + }); + + const newFormat = applyRowContext(transformed, new CompatibilityContext(CompatibilityEdition.SYNC_STREAMS)); + expect(newFormat).toMatchObject({ ranges: JSON.stringify([ [ { lower: 2, upper: 4, lower_exclusive: 0, upper_exclusive: 1 }, @@ -561,7 +620,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' * Return all the inserts from the first transaction in the replication stream. */ async function getReplicationTx(db: pgwire.PgClient, replicationStream: pgwire.ReplicationStream) { - const typeCache = new PostgresTypeCache(db); + const typeCache = new PostgresTypeCache(db, () => getServerVersion(db)); await typeCache.fetchTypesForSchema(); let transformed: SqliteInputRow[] = []; diff --git a/modules/module-postgres/test/src/schema_changes.test.ts b/modules/module-postgres/test/src/schema_changes.test.ts index 356545a17..69a884db6 100644 --- a/modules/module-postgres/test/src/schema_changes.test.ts +++ b/modules/module-postgres/test/src/schema_changes.test.ts @@ -1,6 +1,6 @@ import { compareIds, putOp, reduceBucket, removeOp, test_utils } from '@powersync/service-core-tests'; import * as timers from 'timers/promises'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, it } from 'vitest'; import { storage } from '@powersync/service-core'; import { describeWithStorage } from './util.js'; @@ -591,7 +591,7 @@ function defineTests(factory: storage.TestStorageFactory) { expect(failures).toEqual([]); }); - test('custom types', async () => { + it.skip('custom types', async () => { await using context = await WalStreamTestContext.open(factory); await context.updateSyncRules(` diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index 9827c4e16..68c5dfa56 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -4,7 +4,7 @@ import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import { ReplicationMetric } from '@powersync/service-types'; import * as crypto from 'crypto'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, it } from 'vitest'; import { describeWithStorage } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; @@ -325,7 +325,7 @@ bucket_definitions: } }); - test('custom types', async () => { + it.skip('custom types', async () => { await using context = await WalStreamTestContext.open(factory); await context.updateSyncRules(` From c5e5662f07455506584970a38f4bf9c64dcea73a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 28 Aug 2025 11:20:43 +0200 Subject: [PATCH 11/14] Add test for changing schema --- .../src/replication/WalStream.ts | 2 +- .../test/src/schema_changes.test.ts | 24 ++++++++++++++----- .../test/src/wal_stream.test.ts | 6 ++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 428c46c73..ee34b6d0f 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -711,7 +711,7 @@ WHERE oid = $1::regclass`, await batch.drop(result.dropTables); // Ensure we have a description for custom types referenced in the table. - //await this.connections.types.fetchTypes(referencedTypeIds); + await this.connections.types.fetchTypes(referencedTypeIds); // Snapshot if: // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere) diff --git a/modules/module-postgres/test/src/schema_changes.test.ts b/modules/module-postgres/test/src/schema_changes.test.ts index 69a884db6..c1994e7a8 100644 --- a/modules/module-postgres/test/src/schema_changes.test.ts +++ b/modules/module-postgres/test/src/schema_changes.test.ts @@ -1,6 +1,6 @@ import { compareIds, putOp, reduceBucket, removeOp, test_utils } from '@powersync/service-core-tests'; import * as timers from 'timers/promises'; -import { describe, expect, test, it } from 'vitest'; +import { describe, expect, test } from 'vitest'; import { storage } from '@powersync/service-core'; import { describeWithStorage } from './util.js'; @@ -591,23 +591,35 @@ function defineTests(factory: storage.TestStorageFactory) { expect(failures).toEqual([]); }); - it.skip('custom types', async () => { + test('custom types', async () => { await using context = await WalStreamTestContext.open(factory); await context.updateSyncRules(` streams: stream: - query: SELECT id, * FROM "test_data" + query: SELECT * FROM "test_data" config: edition: 2 `); const { pool } = context; - await pool.query(`DROP TABLE IF EXISTS test_data`); - await pool.query(`CREATE TABLE test_data(id text primary key, description composite);`); - await pool.query(`INSERT INTO test_data(id, description) VALUES ('t1', ROW(TRUE, 2)::composite)`); + await pool.query(`CREATE TABLE test_data(id text primary key);`); + await pool.query(`INSERT INTO test_data(id) VALUES ('t1')`); await context.replicateSnapshot(); + context.startStreaming(); + + await pool.query( + { statement: `CREATE TYPE composite AS (foo bool, bar int4);` }, + { statement: `ALTER TABLE test_data ADD COLUMN other composite;` }, + { statement: `UPDATE test_data SET other = ROW(TRUE, 2)::composite;` } + ); + + const data = await context.getBucketData('1#stream|0[]'); + expect(data).toMatchObject([ + putOp('test_data', { id: 't1' }), + putOp('test_data', { id: 't1', other: '{"foo":1,"bar":2}' }) + ]); }); } diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index 68c5dfa56..80e545773 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -4,7 +4,7 @@ import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import { ReplicationMetric } from '@powersync/service-types'; import * as crypto from 'crypto'; -import { describe, expect, test, it } from 'vitest'; +import { describe, expect, test } from 'vitest'; import { describeWithStorage } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; @@ -325,7 +325,7 @@ bucket_definitions: } }); - it.skip('custom types', async () => { + test('custom types', async () => { await using context = await WalStreamTestContext.open(factory); await context.updateSyncRules(` @@ -346,6 +346,6 @@ config: await pool.query(`INSERT INTO test_data(id, description) VALUES ('t1', ROW(TRUE, 2)::composite)`); const data = await context.getBucketData('1#stream|0[]'); - expect(data).toMatchObject([putOp('test_data', { id: 't1', description: 'test1' })]); + expect(data).toMatchObject([putOp('test_data', { id: 't1', description: '{"foo":1,"bar":2}' })]); }); } From e1cce45cc4a5ecf9af862abc310c5197b9df24d3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 28 Aug 2025 11:56:06 +0200 Subject: [PATCH 12/14] Move cache up --- modules/module-postgres/src/module/PostgresModule.ts | 11 ++++++++--- .../src/replication/ConnectionManagerFactory.ts | 8 ++++++-- modules/module-postgres/src/replication/PgManager.ts | 9 +++++++-- modules/module-postgres/src/types/cache.ts | 2 +- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index d762110c0..244ac75a6 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -19,8 +19,11 @@ import { WalStreamReplicator } from '../replication/WalStreamReplicator.js'; import * as types from '../types/types.js'; import { PostgresConnectionConfig } from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; +import { CustomTypeRegistry } from '../types/registry.js'; export class PostgresModule extends replication.ReplicationModule { + private customTypes: CustomTypeRegistry = new CustomTypeRegistry(); + constructor() { super({ name: 'Postgres', @@ -48,7 +51,7 @@ export class PostgresModule extends replication.ReplicationModule this.getServerVersion()); + this.types = new PostgresTypeCache(poolOptions.registry, this.pool, () => this.getServerVersion()); } public get connectionTag() { diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/cache.ts index a099015bd..de8d3769c 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -7,10 +7,10 @@ import semver from 'semver'; * A cache of custom types for which information can be crawled from the source database. */ export class PostgresTypeCache { - readonly registry: CustomTypeRegistry; private cachedVersion: semver.SemVer | null = null; constructor( + readonly registry: CustomTypeRegistry, private readonly pool: pgwire.PgClient, private readonly getVersion: () => Promise ) { From 4454d2b4ead98cefed67e9a7a042ae91fae18f20 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 28 Aug 2025 13:42:35 +0200 Subject: [PATCH 13/14] Report correct type for schema --- .../src/api/PostgresRouteAPIAdapter.ts | 10 +++- .../src/module/PostgresModule.ts | 3 +- .../src/replication/PgManager.ts | 2 +- modules/module-postgres/src/types/cache.ts | 14 ++--- modules/module-postgres/src/types/registry.ts | 4 ++ modules/module-postgres/src/types/types.ts | 6 +- .../module-postgres/test/src/pg_test.test.ts | 6 +- .../test/src/route_api_adapter.test.ts | 60 +++++++++++++++++++ .../test/src/slow_tests.test.ts | 5 +- .../test/src/wal_stream_utils.ts | 3 +- 10 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 modules/module-postgres/test/src/route_api_adapter.test.ts diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index e35f2e4f2..b5e6061c1 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -9,8 +9,11 @@ import { getDebugTableInfo } from '../replication/replication-utils.js'; import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js'; import * as types from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; +import { PostgresTypeCache } from '../types/cache.js'; +import { CustomTypeRegistry, isKnownType } from '../types/registry.js'; export class PostgresRouteAPIAdapter implements api.RouteAPI { + private typeCache: PostgresTypeCache; connectionTag: string; // TODO this should probably be configurable one day publicationName = PUBLICATION_NAME; @@ -31,6 +34,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { connectionTag?: string, private config?: types.ResolvedConnectionConfig ) { + this.typeCache = new PostgresTypeCache(config?.typeRegistry ?? new CustomTypeRegistry(), pool); this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG; } @@ -297,6 +301,7 @@ LEFT JOIN ( SELECT attrelid, attname, + atttypid, format_type(atttypid, atttypmod) as data_type, (SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type, attnum, @@ -311,6 +316,7 @@ LEFT JOIN ( ) GROUP BY schemaname, tablename, quoted_name` ); + await this.typeCache.fetchTypesForSchema(); const rows = pgwire.pgwireRows(results); let schemas: Record = {}; @@ -332,9 +338,11 @@ GROUP BY schemaname, tablename, quoted_name` if (pg_type.startsWith('_')) { pg_type = `${pg_type.substring(1)}[]`; } + + const knownType = this.typeCache.registry.lookupType(Number(column.atttypid)); table.columns.push({ name: column.attname, - sqlite_type: sync_rules.expressionTypeFromPostgresType(pg_type).typeFlags, + sqlite_type: sync_rules.ExpressionType.fromTypeText(knownType.sqliteType()).typeFlags, type: column.data_type, internal_type: column.data_type, pg_type: pg_type diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index 244ac75a6..48aefd9f5 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -69,7 +69,8 @@ export class PostgresModule extends replication.ReplicationModule this.getServerVersion()); + this.types = new PostgresTypeCache(poolOptions.registry, this.pool); } public get connectionTag() { diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/cache.ts index de8d3769c..6c69c25e2 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/cache.ts @@ -2,6 +2,7 @@ import { DatabaseInputRow, SqliteInputRow, toSyncRulesRow } from '@powersync/ser import * as pgwire from '@powersync/service-jpgwire'; import { CustomTypeRegistry } from './registry.js'; import semver from 'semver'; +import { getServerVersion } from '../utils/postgres_version.js'; /** * A cache of custom types for which information can be crawled from the source database. @@ -11,15 +12,14 @@ export class PostgresTypeCache { constructor( readonly registry: CustomTypeRegistry, - private readonly pool: pgwire.PgClient, - private readonly getVersion: () => Promise + private readonly pool: pgwire.PgClient ) { this.registry = new CustomTypeRegistry(); } private async fetchVersion(): Promise { if (this.cachedVersion == null) { - this.cachedVersion = (await this.getVersion()) ?? semver.parse('0.0.1'); + this.cachedVersion = (await getServerVersion(this.pool)) ?? semver.parse('0.0.1'); } return this.cachedVersion!; @@ -138,9 +138,9 @@ WHERE t.oid = ANY($1) } /** - * Used for testing - fetches all custom types referenced by any column in the schema. + * Used for testing - fetches all custom types referenced by any column in the database. */ - public async fetchTypesForSchema(schema: string = 'public') { + public async fetchTypesForSchema() { const sql = ` SELECT DISTINCT a.atttypid AS type_oid FROM pg_attribute a @@ -150,10 +150,10 @@ JOIN pg_type t ON t.oid = a.atttypid JOIN pg_namespace tn ON tn.oid = t.typnamespace WHERE a.attnum > 0 AND NOT a.attisdropped - AND cn.nspname = $1 + AND cn.nspname not in ('information_schema', 'pg_catalog', 'pg_toast') `; - const query = await this.pool.query({ statement: sql, params: [{ type: 'varchar', value: schema }] }); + const query = await this.pool.query({ statement: sql }); let ids: number[] = []; for (const row of pgwire.pgwireRows(query)) { ids.push(Number(row.type_oid)); diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index 5ab6b9231..fa96a3bd2 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -276,3 +276,7 @@ export class CustomTypeRegistry { } } } + +export function isKnownType(type: MaybeKnownType): type is KnownType { + return type.type != 'unknown'; +} diff --git a/modules/module-postgres/src/types/types.ts b/modules/module-postgres/src/types/types.ts index 4de2ac0ed..3fd3c9fbb 100644 --- a/modules/module-postgres/src/types/types.ts +++ b/modules/module-postgres/src/types/types.ts @@ -1,6 +1,7 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import * as service_types from '@powersync/service-types'; import * as t from 'ts-codec'; +import { CustomTypeRegistry } from './registry.js'; // Maintain backwards compatibility by exporting these export const validatePort = lib_postgres.validatePort; @@ -24,7 +25,10 @@ export type PostgresConnectionConfig = t.Decoded { async function setupTable(db: pgwire.PgClient) { @@ -551,7 +551,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' test('test replication - multiranges', async () => { const db = await connectPgPool(); - if (!(await new PostgresTypeCache(db, () => getServerVersion(db)).supportsMultiRanges())) { + if (!(await new PostgresTypeCache(new CustomTypeRegistry(), db).supportsMultiRanges())) { // This test requires Postgres 14 or later. return; } @@ -620,7 +620,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' * Return all the inserts from the first transaction in the replication stream. */ async function getReplicationTx(db: pgwire.PgClient, replicationStream: pgwire.ReplicationStream) { - const typeCache = new PostgresTypeCache(db, () => getServerVersion(db)); + const typeCache = new PostgresTypeCache(new CustomTypeRegistry(), db); await typeCache.fetchTypesForSchema(); let transformed: SqliteInputRow[] = []; diff --git a/modules/module-postgres/test/src/route_api_adapter.test.ts b/modules/module-postgres/test/src/route_api_adapter.test.ts new file mode 100644 index 000000000..98f16930c --- /dev/null +++ b/modules/module-postgres/test/src/route_api_adapter.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from 'vitest'; +import { clearTestDb, connectPgPool } from './util.js'; +import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js'; +import { TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from '@powersync/service-sync-rules'; + +describe('PostgresRouteAPIAdapter tests', () => { + test('infers connection schema', async () => { + const db = await connectPgPool(); + try { + await clearTestDb(db); + const api = new PostgresRouteAPIAdapter(db); + + await db.query(`CREATE DOMAIN rating_value AS FLOAT CHECK (VALUE BETWEEN 0 AND 5)`); + await db.query(` + CREATE TABLE test_users ( + id TEXT NOT NULL PRIMARY KEY, + is_admin BOOLEAN, + rating RATING_VALUE + ); + `); + + const schema = await api.getConnectionSchema(); + expect(schema).toStrictEqual([ + { + name: 'public', + tables: [ + { + name: 'test_users', + columns: [ + { + internal_type: 'text', + name: 'id', + pg_type: 'text', + sqlite_type: TYPE_TEXT, + type: 'text' + }, + { + internal_type: 'boolean', + name: 'is_admin', + pg_type: 'bool', + sqlite_type: TYPE_INTEGER, + type: 'boolean' + }, + { + internal_type: 'rating_value', + name: 'rating', + pg_type: 'rating_value', + sqlite_type: TYPE_REAL, + type: 'rating_value' + } + ] + } + ] + } + ]); + } finally { + await db.end(); + } + }); +}); diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index ae5294887..49dd23952 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -19,6 +19,7 @@ import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; +import { CustomTypeRegistry } from '@module/types/registry.js'; describe.skipIf(!(env.CI || env.SLOW_TESTS))('slow tests', function () { describeWithStorage({ timeout: 120_000 }, function (factory) { @@ -68,7 +69,7 @@ function defineSlowTests(factory: storage.TestStorageFactory) { }); async function testRepeatedReplication(testOptions: { compact: boolean; maxBatchSize: number; numBatches: number }) { - const connections = new PgManager(TEST_CONNECTION_OPTIONS, {}); + const connections = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() }); const replicationConnection = await connections.replicationConnection(); const pool = connections.pool; await clearTestDb(pool); @@ -329,7 +330,7 @@ bucket_definitions: await pool.query(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE active = FALSE`); i += 1; - const connections = new PgManager(TEST_CONNECTION_OPTIONS, {}); + const connections = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() }); const replicationConnection = await connections.replicationConnection(); abortController = new AbortController(); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index b94c73c5a..40ea5aaba 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -12,6 +12,7 @@ import { import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; +import { CustomTypeRegistry } from '@module/types/registry.js'; export class WalStreamTestContext implements AsyncDisposable { private _walStream?: WalStream; @@ -32,7 +33,7 @@ export class WalStreamTestContext implements AsyncDisposable { options?: { doNotClear?: boolean; walStreamOptions?: Partial } ) { const f = await factory({ doNotClear: options?.doNotClear }); - const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, {}); + const connectionManager = new PgManager(TEST_CONNECTION_OPTIONS, { registry: new CustomTypeRegistry() }); if (!options?.doNotClear) { await clearTestDb(connectionManager.pool); From 0eefce1b63665493be65f54a12993d86b1a58bb9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 28 Aug 2025 14:04:17 +0200 Subject: [PATCH 14/14] Cleanup --- .../module-postgres/src/api/PostgresRouteAPIAdapter.ts | 8 ++++---- modules/module-postgres/src/replication/PgManager.ts | 6 +++--- modules/module-postgres/src/types/registry.ts | 4 ---- .../src/types/{cache.ts => resolver.ts} | 10 ++++++---- modules/module-postgres/test/src/pg_test.test.ts | 6 +++--- packages/jpgwire/src/pgwire_types.ts | 2 +- 6 files changed, 17 insertions(+), 19 deletions(-) rename modules/module-postgres/src/types/{cache.ts => resolver.ts} (94%) diff --git a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts index b5e6061c1..c40eddc7a 100644 --- a/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts +++ b/modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts @@ -9,11 +9,11 @@ import { getDebugTableInfo } from '../replication/replication-utils.js'; import { KEEPALIVE_STATEMENT, PUBLICATION_NAME } from '../replication/WalStream.js'; import * as types from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; -import { PostgresTypeCache } from '../types/cache.js'; -import { CustomTypeRegistry, isKnownType } from '../types/registry.js'; +import { CustomTypeRegistry } from '../types/registry.js'; +import { PostgresTypeResolver } from '../types/resolver.js'; export class PostgresRouteAPIAdapter implements api.RouteAPI { - private typeCache: PostgresTypeCache; + private typeCache: PostgresTypeResolver; connectionTag: string; // TODO this should probably be configurable one day publicationName = PUBLICATION_NAME; @@ -34,7 +34,7 @@ export class PostgresRouteAPIAdapter implements api.RouteAPI { connectionTag?: string, private config?: types.ResolvedConnectionConfig ) { - this.typeCache = new PostgresTypeCache(config?.typeRegistry ?? new CustomTypeRegistry(), pool); + this.typeCache = new PostgresTypeResolver(config?.typeRegistry ?? new CustomTypeRegistry(), pool); this.connectionTag = connectionTag ?? sync_rules.DEFAULT_TAG; } diff --git a/modules/module-postgres/src/replication/PgManager.ts b/modules/module-postgres/src/replication/PgManager.ts index 1e38d0e23..3d034d326 100644 --- a/modules/module-postgres/src/replication/PgManager.ts +++ b/modules/module-postgres/src/replication/PgManager.ts @@ -2,7 +2,7 @@ import * as pgwire from '@powersync/service-jpgwire'; import semver from 'semver'; import { NormalizedPostgresConnectionConfig } from '../types/types.js'; import { getApplicationName } from '../utils/application-name.js'; -import { PostgresTypeCache } from '../types/cache.js'; +import { PostgresTypeResolver } from '../types/resolver.js'; import { getServerVersion } from '../utils/postgres_version.js'; import { CustomTypeRegistry } from '../types/registry.js'; @@ -21,7 +21,7 @@ export class PgManager { */ public readonly pool: pgwire.PgClient; - public readonly types: PostgresTypeCache; + public readonly types: PostgresTypeResolver; private connectionPromises: Promise[] = []; @@ -31,7 +31,7 @@ export class PgManager { ) { // The pool is lazy - no connections are opened until a query is performed. this.pool = pgwire.connectPgWirePool(this.options, poolOptions); - this.types = new PostgresTypeCache(poolOptions.registry, this.pool); + this.types = new PostgresTypeResolver(poolOptions.registry, this.pool); } public get connectionTag() { diff --git a/modules/module-postgres/src/types/registry.ts b/modules/module-postgres/src/types/registry.ts index fa96a3bd2..5ab6b9231 100644 --- a/modules/module-postgres/src/types/registry.ts +++ b/modules/module-postgres/src/types/registry.ts @@ -276,7 +276,3 @@ export class CustomTypeRegistry { } } } - -export function isKnownType(type: MaybeKnownType): type is KnownType { - return type.type != 'unknown'; -} diff --git a/modules/module-postgres/src/types/cache.ts b/modules/module-postgres/src/types/resolver.ts similarity index 94% rename from modules/module-postgres/src/types/cache.ts rename to modules/module-postgres/src/types/resolver.ts index 6c69c25e2..694b53537 100644 --- a/modules/module-postgres/src/types/cache.ts +++ b/modules/module-postgres/src/types/resolver.ts @@ -5,9 +5,11 @@ import semver from 'semver'; import { getServerVersion } from '../utils/postgres_version.js'; /** - * A cache of custom types for which information can be crawled from the source database. + * Resolves descriptions used to decode values for custom postgres types. + * + * Custom types are resolved from the source database, which also involves crawling inner types (e.g. for composites). */ -export class PostgresTypeCache { +export class PostgresTypeResolver { private cachedVersion: semver.SemVer | null = null; constructor( @@ -31,7 +33,7 @@ export class PostgresTypeCache { */ async supportsMultiRanges() { const version = await this.fetchVersion(); - return version.compare(PostgresTypeCache.minVersionForMultirange) >= 0; + return version.compare(PostgresTypeResolver.minVersionForMultirange) >= 0; } /** @@ -138,7 +140,7 @@ WHERE t.oid = ANY($1) } /** - * Used for testing - fetches all custom types referenced by any column in the database. + * Crawls all custom types referenced by table columns in the current database. */ public async fetchTypesForSchema() { const sql = ` diff --git a/modules/module-postgres/test/src/pg_test.test.ts b/modules/module-postgres/test/src/pg_test.test.ts index 111fd5bcd..e41c61f63 100644 --- a/modules/module-postgres/test/src/pg_test.test.ts +++ b/modules/module-postgres/test/src/pg_test.test.ts @@ -10,7 +10,7 @@ import { import { describe, expect, test } from 'vitest'; import { clearTestDb, connectPgPool, connectPgWire, TEST_URI } from './util.js'; import { WalStream } from '@module/replication/WalStream.js'; -import { PostgresTypeCache } from '@module/types/cache.js'; +import { PostgresTypeResolver } from '@module/types/resolver.js'; import { CustomTypeRegistry } from '@module/types/registry.js'; describe('pg data types', () => { @@ -551,7 +551,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' test('test replication - multiranges', async () => { const db = await connectPgPool(); - if (!(await new PostgresTypeCache(new CustomTypeRegistry(), db).supportsMultiRanges())) { + if (!(await new PostgresTypeResolver(new CustomTypeRegistry(), db).supportsMultiRanges())) { // This test requires Postgres 14 or later. return; } @@ -620,7 +620,7 @@ INSERT INTO test_data(id, time, timestamp, timestamptz) VALUES (1, '17:42:01.12' * Return all the inserts from the first transaction in the replication stream. */ async function getReplicationTx(db: pgwire.PgClient, replicationStream: pgwire.ReplicationStream) { - const typeCache = new PostgresTypeCache(new CustomTypeRegistry(), db); + const typeCache = new PostgresTypeResolver(new CustomTypeRegistry(), db); await typeCache.fetchTypesForSchema(); let transformed: SqliteInputRow[] = []; diff --git a/packages/jpgwire/src/pgwire_types.ts b/packages/jpgwire/src/pgwire_types.ts index 7842ea452..555de6e3a 100644 --- a/packages/jpgwire/src/pgwire_types.ts +++ b/packages/jpgwire/src/pgwire_types.ts @@ -156,7 +156,7 @@ export class PgType { static _decodeArray(text: string, elemTypeOid: number): DatabaseInputValue[] { text = text.replace(/^\[.+=/, ''); // skip dimensions - return new StructureParser(text).parseArray((raw) => (raw == null ? null : PgType.decode(raw, elemTypeOid))); + return new StructureParser(text).parseArray((raw) => PgType.decode(raw, elemTypeOid)); } static _decodeBytea(text: string): Uint8Array {