From 1561867919e9290884fd663ac316a6bd3ee08da3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 17 Jun 2025 18:04:46 +0200 Subject: [PATCH 01/21] Start with new unified syntax --- .../sync-rules/src/SqlBucketDescriptor.ts | 21 +- packages/sync-rules/src/SqlSyncRules.ts | 81 ++++++-- packages/sync-rules/src/StreamQuery.ts | 191 ++++++++++++++++++ packages/sync-rules/src/json_schema.ts | 33 ++- packages/sync-rules/src/request_functions.ts | 40 ++-- packages/sync-rules/src/sql_filters.ts | 100 +++++++-- packages/sync-rules/src/types.ts | 4 + packages/sync-rules/test/src/streams.test.ts | 126 ++++++++++++ 8 files changed, 548 insertions(+), 48 deletions(-) create mode 100644 packages/sync-rules/src/StreamQuery.ts create mode 100644 packages/sync-rules/test/src/streams.test.ts diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 95b2b4a10..22714e845 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -6,6 +6,7 @@ import { SqlDataQuery } from './SqlDataQuery.js'; import { SqlParameterQuery } from './SqlParameterQuery.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; +import { StreamQuery } from './StreamQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { SqlRuleError } from './errors.js'; @@ -15,7 +16,8 @@ import { EvaluationResult, QueryParseOptions, RequestParameters, - SqliteRow + SqliteRow, + StreamParseOptions } from './types.js'; export interface QueryParseResult { @@ -81,6 +83,23 @@ export class SqlBucketDescriptor { }; } + addUnifiedStreamQuery(sql: string, options: StreamParseOptions): QueryParseResult { + const [query, errors] = StreamQuery.fromSql(this.name, sql, options); + for (const parameterQuery of query.inferredParameters) { + if (parameterQuery instanceof StaticSqlParameterQuery) { + this.globalParameterQueries.push(parameterQuery); + } else { + this.parameterQueries.push(parameterQuery); + } + } + this.dataQueries.push(query.data); + + return { + parsed: true, + errors + }; + } + evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { let results: EvaluationResult[] = []; for (let query of this.dataQueries) { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index a77933630..e8943b66a 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -22,6 +22,7 @@ import { RequestParameters, SourceSchema, SqliteRow, + StreamParseOptions, SyncRules } from './types.js'; @@ -95,9 +96,23 @@ export class SqlSyncRules implements SyncRules { return rules; } + // Bucket definitions using explicit parameter and data queries. const bucketMap = parsed.get('bucket_definitions') as YAMLMap; - if (bucketMap == null) { - rules.errors.push(new YamlError(new Error(`'bucket_definitions' is required`))); + // Streams (which also map to buckets internally) with a new syntax and options. + const streamMap = parsed.get('streams') as YAMLMap; + const definitionNames = new Set(); + const checkUniqueName = (name: string, literal: Scalar) => { + if (definitionNames.has(name)) { + rules.errors.push(this.tokenError(literal, 'Duplicate stream or bucket definition.')); + return false; + } + + definitionNames.add(name); + return true; + }; + + if (bucketMap == null && streamMap == null) { + rules.errors.push(new YamlError(new Error(`Either 'bucket_definitions' or 'streams' are required`))); if (throwOnError) { rules.throwOnError(); @@ -105,9 +120,12 @@ export class SqlSyncRules implements SyncRules { return rules; } - for (let entry of bucketMap.items) { + for (let entry of bucketMap?.items ?? []) { const { key: keyScalar, value } = entry as { key: Scalar; value: YAMLMap }; const key = keyScalar.toString(); + if (!checkUniqueName(key, keyScalar)) { + continue; + } if (value == null || !(value instanceof YAMLMap)) { rules.errors.push(this.tokenError(keyScalar, `'${key}' bucket definition must be an object`)); @@ -116,17 +134,7 @@ export class SqlSyncRules implements SyncRules { const accept_potentially_dangerous_queries = value.get('accept_potentially_dangerous_queries', true)?.value == true; - let parseOptionPriority: BucketPriority | undefined = undefined; - if (value.has('priority')) { - const priorityValue = value.get('priority', true)!; - if (typeof priorityValue.value != 'number' || !isValidPriority(priorityValue.value)) { - rules.errors.push( - this.tokenError(priorityValue, 'Invalid priority, expected a number between 0 and 3 (inclusive).') - ); - } else { - parseOptionPriority = priorityValue.value; - } - } + const parseOptionPriority = rules.parsePriority(value); const queryOptions: QueryParseOptions = { ...options, @@ -164,6 +172,38 @@ export class SqlSyncRules implements SyncRules { rules.bucketDescriptors.push(descriptor); } + for (const entry of streamMap?.items ?? []) { + const { key: keyScalar, value } = entry as { key: Scalar; value: YAMLMap }; + const key = keyScalar.toString(); + if (!checkUniqueName(key, keyScalar)) { + continue; + } + + const descriptor = new SqlBucketDescriptor(key); + + const accept_potentially_dangerous_queries = + value.get('accept_potentially_dangerous_queries', true)?.value == true; + + const queryOptions: StreamParseOptions = { + ...options, + accept_potentially_dangerous_queries, + priority: rules.parsePriority(value), + default: value.get('default', true)?.value == true + }; + + const data = value.get('query', true) as unknown; + if (data instanceof Scalar) { + rules.withScalar(data, (q) => { + return descriptor.addUnifiedStreamQuery(q, queryOptions); + }); + } else { + rules.errors.push(this.tokenError(data, 'Must be a string.')); + continue; + } + + rules.bucketDescriptors.push(descriptor); + } + const eventMap = parsed.get('event_definitions') as YAMLMap; for (const event of eventMap?.items ?? []) { const { key, value } = event as { key: Scalar; value: YAMLSeq }; @@ -391,4 +431,17 @@ export class SqlSyncRules implements SyncRules { } return result; } + + private parsePriority(value: YAMLMap) { + if (value.has('priority')) { + const priorityValue = value.get('priority', true)!; + if (typeof priorityValue.value != 'number' || !isValidPriority(priorityValue.value)) { + this.errors.push( + SqlSyncRules.tokenError(priorityValue, 'Invalid priority, expected a number between 0 and 3 (inclusive).') + ); + } else { + return priorityValue.value; + } + } + } } diff --git a/packages/sync-rules/src/StreamQuery.ts b/packages/sync-rules/src/StreamQuery.ts new file mode 100644 index 000000000..ca7caac2c --- /dev/null +++ b/packages/sync-rules/src/StreamQuery.ts @@ -0,0 +1,191 @@ +import { parse } from 'pgsql-ast-parser'; +import { ParameterValueClause, QuerySchema, StreamParseOptions } from './types.js'; +import { SqlRuleError } from './errors.js'; +import { isSelectStatement } from './utils.js'; +import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; +import { SqlDataQuery, SqlDataQueryOptions } from './SqlDataQuery.js'; +import { RowValueExtractor } from './BaseSqlDataQuery.js'; +import { TablePattern } from './TablePattern.js'; +import { TableQuerySchema } from './TableQuerySchema.js'; +import { SqlTools } from './sql_filters.js'; +import { ExpressionType } from './ExpressionType.js'; +import { SqlParameterQuery } from './SqlParameterQuery.js'; +import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; +import { DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; + +/** + * Represents a query backing a stream definition. + * + * Streams are a new way to define sync rules that don't require separate data and + * parameter queries. However, since most of the sync service is built around that + * distiction at the moment, stream queries are implemented by desugaring a unified + * query into its individual components. + */ +export class StreamQuery { + inferredParameters: (SqlParameterQuery | StaticSqlParameterQuery)[]; + data: SqlDataQuery; + + static fromSql(descriptorName: string, sql: string, options: StreamParseOptions): [StreamQuery, SqlRuleError[]] { + const [query, ...illegalRest] = parse(sql, { locationTracking: true }); + const schema = options.schema; + const parameters: (SqlParameterQuery | StaticSqlParameterQuery)[] = []; + const errors: SqlRuleError[] = []; + + // TODO: Share more of this code with SqlDataQuery + if (illegalRest.length > 0) { + throw new SqlRuleError('Only a single SELECT statement is supported', sql, illegalRest[0]?._location); + } + + if (!isSelectStatement(query)) { + throw new SqlRuleError('Only SELECT statements are supported', sql, query._location); + } + + if (query.from == null || query.from.length != 1 || query.from[0].type != 'table') { + throw new SqlRuleError('Must SELECT from a single table', sql, query.from?.[0]._location); + } + + errors.push(...checkUnsupportedFeatures(sql, query)); + + const tableRef = query.from?.[0].name; + if (tableRef?.name == null) { + throw new SqlRuleError('Must SELECT from a single table', sql, query.from?.[0]._location); + } + const alias: string = tableRef.alias ?? tableRef.name; + + const sourceTable = new TablePattern(tableRef.schema ?? options.defaultSchema, tableRef.name); + let querySchema: QuerySchema | undefined = undefined; + if (schema) { + const tables = schema.getTables(sourceTable); + if (tables.length == 0) { + const e = new SqlRuleError( + `Table ${sourceTable.schema}.${sourceTable.tablePattern} not found`, + sql, + query.from?.[0]?._location + ); + e.type = 'warning'; + + errors.push(e); + } else { + querySchema = new TableQuerySchema(tables, alias); + } + } + + const where = query.where; + const tools = new SqlTools({ + table: alias, + parameterTables: [], + valueTables: [alias], + sql, + schema: querySchema, + supportsStreamInputs: true, + supportsParameterExpressions: true + }); + tools.checkSpecificNameCase(tableRef); + const filter = tools.compileWhereClause(where); + const inputParameterNames = filter.inputParameters.map((p) => `bucket.${p.key}`); + + // Build parameter queries based on inferred bucket parameters + if (tools.inferredParameters.length) { + const extractors: Record = {}; + for (const inferred of tools.inferredParameters) { + extractors[inferred.name] = inferred.clause; + } + + parameters.push( + new StaticSqlParameterQuery({ + sql, + queryId: 'static', + descriptorName, + parameterExtractors: extractors, + bucketParameters: tools.inferredParameters.map((p) => p.name), + filter: undefined, // TODO + priority: DEFAULT_BUCKET_PRIORITY // Ignored here + }) + ); + } + + let hasId = false; + let hasWildcard = false; + let extractors: RowValueExtractor[] = []; + + for (let column of query.columns ?? []) { + const name = tools.getOutputName(column); + if (name != '*') { + const clause = tools.compileRowValueExtractor(column.expr); + if (isClauseError(clause)) { + // Error logged already + continue; + } + extractors.push({ + extract: (tables, output) => { + output[name] = clause.evaluate(tables); + }, + getTypes(schema, into) { + const def = clause.getColumnDefinition(schema); + + into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType }; + } + }); + } else { + extractors.push({ + extract: (tables, output) => { + const row = tables[alias]; + for (let key in row) { + if (key.startsWith('_')) { + continue; + } + output[key] ??= row[key]; + } + }, + getTypes(schema, into) { + for (let column of schema.getColumns(alias)) { + into[column.name] ??= column; + } + } + }); + } + if (name == 'id') { + hasId = true; + } else if (name == '*') { + hasWildcard = true; + if (querySchema == null) { + // Not performing schema-based validation - assume there is an id + hasId = true; + } else { + const idType = querySchema.getColumn(alias, 'id')?.type ?? ExpressionType.NONE; + if (!idType.isNone()) { + hasId = true; + } + } + } + } + if (!hasId) { + const error = new SqlRuleError(`Query must return an "id" column`, sql, query.columns?.[0]._location); + if (hasWildcard) { + // Schema-based validations are always warnings + error.type = 'warning'; + } + errors.push(error); + } + + errors.push(...tools.errors); + + const data: SqlDataQueryOptions = { + sourceTable, + table: alias, + sql, + filter, + columns: query.columns ?? [], + descriptorName, + bucketParameters: inputParameterNames, + tools, + extractors + }; + return [new StreamQuery(parameters, data), errors]; + } + + private constructor(parameters: (SqlParameterQuery | StaticSqlParameterQuery)[], data: SqlDataQueryOptions) { + this.inferredParameters = parameters; + this.data = new SqlDataQuery(data); + } +} diff --git a/packages/sync-rules/src/json_schema.ts b/packages/sync-rules/src/json_schema.ts index 5d21169d4..ef75b5409 100644 --- a/packages/sync-rules/src/json_schema.ts +++ b/packages/sync-rules/src/json_schema.ts @@ -49,6 +49,37 @@ export const syncRulesSchema: ajvModule.Schema = { } } }, + streams: { + type: 'object', + description: 'Stream definitions', + patternProperties: { + '.*': { + type: 'object', + required: ['query'], + examples: [{ query: ['select * from mytable'] }], + properties: { + accept_potentially_dangerous_queries: { + description: 'If true, disables warnings on potentially dangerous queries', + type: 'boolean' + }, + priority: { + description: + 'Default priority for the stream (lower values indicate higher priority). Clients can override the priority when subscribing.', + type: 'integer' + }, + default: { + type: 'boolean', + description: 'Whether the stream should be subscribed to by default.' + }, + query: { + description: 'The SQL query to sync to clients.', + type: 'string' + } + }, + additionalProperties: false + } + } + }, event_definitions: { type: 'object', description: 'Record of sync replication event definitions', @@ -79,7 +110,7 @@ export const syncRulesSchema: ajvModule.Schema = { } } }, - required: ['bucket_definitions'], + // required: ['bucket_definitions'], additionalProperties: false } as const; diff --git a/packages/sync-rules/src/request_functions.ts b/packages/sync-rules/src/request_functions.ts index fc840875f..00cd89b58 100644 --- a/packages/sync-rules/src/request_functions.ts +++ b/packages/sync-rules/src/request_functions.ts @@ -13,19 +13,21 @@ export interface SqlParameterFunction { documentation: string; } -const request_parameters: SqlParameterFunction = { - debugName: 'request.parameters', - call(parameters: ParameterValueSet) { - return parameters.rawUserParameters; - }, - getReturnType() { - return ExpressionType.TEXT; - }, - detail: 'Unauthenticated request parameters as JSON', - documentation: - 'Returns parameters passed by the client as a JSON string. These parameters are not authenticated - any value can be passed in by the client.', - usesAuthenticatedRequestParameters: false, - usesUnauthenticatedRequestParameters: true +const parametersFunction = (name: string): SqlParameterFunction => { + return { + debugName: name, + call(parameters: ParameterValueSet) { + return parameters.rawUserParameters; + }, + getReturnType() { + return ExpressionType.TEXT; + }, + detail: 'Unauthenticated request parameters as JSON', + documentation: + 'Returns parameters passed by the client as a JSON string. These parameters are not authenticated - any value can be passed in by the client.', + usesAuthenticatedRequestParameters: false, + usesUnauthenticatedRequestParameters: true + }; }; const request_jwt: SqlParameterFunction = { @@ -56,10 +58,16 @@ const request_user_id: SqlParameterFunction = { usesUnauthenticatedRequestParameters: false }; -export const REQUEST_FUNCTIONS_NAMED = { - parameters: request_parameters, +export const REQUEST_FUNCTIONS_WITHOUT_PARAMETERS: Record = { jwt: request_jwt, user_id: request_user_id }; -export const REQUEST_FUNCTIONS: Record = REQUEST_FUNCTIONS_NAMED; +export const REQUEST_FUNCTIONS: Record = { + parameters: parametersFunction('request.parameters'), + ...REQUEST_FUNCTIONS_WITHOUT_PARAMETERS +}; + +export const QUERY_FUNCTIONS: Record = { + params: parametersFunction('query.params') +}; diff --git a/packages/sync-rules/src/sql_filters.ts b/packages/sync-rules/src/sql_filters.ts index c35ce91c6..d9a31c061 100644 --- a/packages/sync-rules/src/sql_filters.ts +++ b/packages/sync-rules/src/sql_filters.ts @@ -4,7 +4,7 @@ import { nil } from 'pgsql-ast-parser/src/utils.js'; import { BucketPriority, isValidPriority } from './BucketDescription.js'; import { ExpressionType } from './ExpressionType.js'; import { SqlRuleError } from './errors.js'; -import { REQUEST_FUNCTIONS } from './request_functions.js'; +import { QUERY_FUNCTIONS, REQUEST_FUNCTIONS } from './request_functions.js'; import { BASIC_OPERATORS, OPERATOR_IN, @@ -94,6 +94,11 @@ export interface SqlToolsOptions { */ supportsParameterExpressions?: boolean; + /** + * true if expressions on stream parameters are supported. + */ + supportsStreamInputs?: boolean; + /** * Schema for validations. */ @@ -113,6 +118,9 @@ export class SqlTools { readonly supportsExpandingParameters: boolean; readonly supportsParameterExpressions: boolean; + readonly supportsStreamInputs: boolean; + + private inferredStaticParameters: Map = new Map(); schema?: QuerySchema; @@ -131,6 +139,7 @@ export class SqlTools { this.sql = options.sql; this.supportsExpandingParameters = options.supportsExpandingParameters ?? false; this.supportsParameterExpressions = options.supportsParameterExpressions ?? false; + this.supportsStreamInputs = options.supportsStreamInputs ?? false; } error(message: string, expr: NodeLocation | Expr | undefined): ClauseError { @@ -271,7 +280,7 @@ export class SqlTools { return compileStaticOperator(op, leftFilter as RowValueClause, rightFilter as RowValueClause); } else if (isParameterValueClause(otherFilter)) { // 2. row value = parameter value - const inputParam = basicInputParameter(otherFilter); + const inputParam = this.basicInputParameter(otherFilter); return { error: false, @@ -318,7 +327,7 @@ export class SqlTools { } else if (isParameterValueClause(leftFilter) && isRowValueClause(rightFilter)) { // token_parameters.value IN table.some_array // bucket.param IN table.some_array - const inputParam = basicInputParameter(leftFilter); + const inputParam = this.basicInputParameter(leftFilter); return { error: false, @@ -418,6 +427,10 @@ export class SqlTools { return this.error(`${schema} schema is not available in data queries`, expr); } + if (fn == 'parameters' && this.supportsStreamInputs) { + return this.error(`'request.parameters()' is unavailable on streams - use 'stream.params()' instead.`, expr); + } + if (expr.args.length > 0) { return this.error(`Function '${schema}.${fn}' does not take arguments`, expr); } @@ -435,6 +448,24 @@ export class SqlTools { } else { return this.error(`Function '${schema}.${fn}' is not defined`, expr); } + } else if (schema == 'stream') { + if (!this.supportsStreamInputs) { + return this.error(`${schema} schema is only available in stream definitions`, expr); + } + + if (fn in QUERY_FUNCTIONS) { + const fnImpl = QUERY_FUNCTIONS[fn]; + return { + key: 'stream.params()', + lookupParameterValue(parameters) { + return fnImpl.call(parameters); + }, + usesAuthenticatedRequestParameters: fnImpl.usesAuthenticatedRequestParameters, + usesUnauthenticatedRequestParameters: fnImpl.usesUnauthenticatedRequestParameters + } satisfies ParameterValueClause; + } else { + return this.error(`Function '${schema}.${fn}' is not defined`, expr); + } } else { // Unknown function with schema return this.error(`Function '${schema}.${fn}' is not defined`, expr); @@ -757,8 +788,58 @@ export class SqlTools { return value as BucketPriority; } + + private basicInputParameter(clause: ParameterValueClause): InputParameter { + if (this.supportsStreamInputs) { + let key = this.inferredStaticParameters.get(clause.key)?.name; + if (key == null) { + key = this.newInferredBucketParameterName(); + this.inferredStaticParameters.set(clause.key, { + name: key, + variant: 'static', + clause + }); + } + + return { + key, + expands: false, + filteredRowToLookupValue: () => { + return SQLITE_FALSE; // Only relevant for parameter queries, but this is a stream query. + }, + parametersToLookupValue: () => { + return SQLITE_FALSE; + } + }; + } + + return { + key: clause.key, + expands: false, + filteredRowToLookupValue: (filterParameters) => { + return filterParameters[clause.key]; + }, + parametersToLookupValue: (parameters) => { + return clause.lookupParameterValue(parameters); + } + }; + } + + public get inferredParameters(): InferredBucketParameter[] { + return [...this.inferredStaticParameters.values()]; + } + + private newInferredBucketParameterName() { + return `p${this.inferredStaticParameters.size}`; + } } +export type InferredBucketParameter = { + name: string; +} & StaticBucketParameter; + +export type StaticBucketParameter = { variant: 'static'; clause: ParameterValueClause }; + function isStatic(expr: Expr) { return ['integer', 'string', 'numeric', 'boolean', 'null'].includes(expr.type); } @@ -795,16 +876,3 @@ function staticValueClause(value: SqliteValue): StaticValueClause { usesUnauthenticatedRequestParameters: false }; } - -function basicInputParameter(clause: ParameterValueClause): InputParameter { - return { - key: clause.key, - expands: false, - filteredRowToLookupValue: (filterParameters) => { - return filterParameters[clause.key]; - }, - parametersToLookupValue: (parameters) => { - return clause.lookupParameterValue(parameters); - } - }; -} diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index ac6d9035a..b03b07c06 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -18,6 +18,10 @@ export interface QueryParseOptions extends SyncRulesOptions { priority?: BucketPriority; } +export interface StreamParseOptions extends QueryParseOptions { + default?: boolean; +} + export interface EvaluatedParameters { lookup: ParameterLookup; diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts new file mode 100644 index 000000000..2cd5f17ea --- /dev/null +++ b/packages/sync-rules/test/src/streams.test.ts @@ -0,0 +1,126 @@ +import { assert, describe, expect, test } from 'vitest'; +import { SqlSyncRules } from '../../src/index.js'; +import { ASSETS, BASIC_SCHEMA, PARSE_OPTIONS } from './util.js'; + +describe('streams', () => { + test('without parameters', () => { + const desc = parseSingleBucketDescription(` +streams: + lists: + query: SELECT * FROM assets +`); + expect(desc.bucketParameters).toBeFalsy(); + }); + + test('static filter', () => { + const desc = parseSingleBucketDescription(` +streams: + lists: + query: SELECT * FROM assets WHERE count > 5 +`); + expect(desc.bucketParameters).toBeFalsy(); + assert.isEmpty(desc.parameterQueries); + + assert.isEmpty( + desc.evaluateRow({ + sourceTable: ASSETS, + record: { + count: 4 + } + }) + ); + + expect( + desc.evaluateRow({ + sourceTable: ASSETS, + record: { + count: 6 + } + }) + ).toHaveLength(1); + }); + + test('stream param', () => { + const desc = parseSingleBucketDescription(` +streams: + lists: + query: SELECT * FROM assets WHERE id = stream.params() ->> 'id'; +`); + + // This should be desuraged to + // params: SELECT request.params() ->> 'id' AS p0 + // data: SELECT * FROM assets WHERE id = bucket.p0 + + expect(desc.globalParameterQueries).toHaveLength(1); + const [parameter] = desc.globalParameterQueries; + expect(parameter.bucketParameters).toEqual(['p0']); + + const [data] = desc.dataQueries; + expect(data.bucketParameters).toEqual(['bucket.p0']); + }); + + test('user filter', () => { + const desc = parseSingleBucketDescription(` +streams: + lists: + query: SELECT * FROM assets WHERE request.jwt() ->> 'isAdmin' +`); + + // This should be desuraged to + // params: SELECT request.params() ->> 'id' AS p0 + // data: SELECT * FROM assets WHERE id = bucket.p0 + + const [data] = desc.dataQueries; + expect(data.bucketParameters).toEqual(['bucket.p0']); + }); +}); + +/** + +SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) + parameter: SELECT id AS p0 FROM asset_groups WHERE owner = request.user_id() + data: SELECT * FROM assets WHERE id = bucket.p0 + +SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) + OR count > 10 + parameter: SELECT id AS p0 FROM asset_groups WHERE owner = request.user_id() + data: SELECT * FROM assets WHERE id = bucket.p0 OR count > 10 + +SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) + OR request.jwt() ->> 'isAdmin' + parameter: SELECT id AS p0, request.jwt() ->> 'isAdmin' AS p1 FROM asset_groups WHERE owner = request.user_id() + parameter: SELECT NULL as p0, request.jwt() ->> 'isAdmin' AS p1 + data: SELECT * FROM assets WHERE id = bucket.p0 OR bucket.p1 + +SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) + AND request.jwt() ->> 'isAdmin' + parameter: SELECT id AS p0, request.jwt() ->> 'isAdmin' AS p1 FROM asset_groups WHERE owner = request.user_id() + AND request.jwt() ->> 'isAdmin' + data: SELECT * FROM assets WHERE id = bucket.p0 + +SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) + OR name IN (SELECT name FROM test2 WHERE owner = request.user_id()) + parameter: SELECT id AS p0, NULL AS p1 FROM asset_groups WHERE owner = request.user_id() + parameter: SELECT NULL AS p0, NULL AS p1 FROM test2 WHERE owner = request.user_id() + data: SELECT * FROM assets WHERE id = bucket.p0 OR name = bucket.p1 + +SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) + AND name IN (SELECT name FROM test2 WHERE owner = request.user_id()) + parameter: SELECT id AS p0, NULL AS p1 FROM asset_groups WHERE owner = request.user_id() + parameter: SELECT NULL AS p0, NULL AS p1 FROM test2 WHERE owner = request.user_id() + data: SELECT * FROM assets WHERE id = bucket.p0 AND name = bucket.p1 + */ + +const options = { schema: BASIC_SCHEMA, ...PARSE_OPTIONS }; + +function parseSyncRules(yaml: string) { + const rules = SqlSyncRules.fromYaml(yaml, options); + assert.isEmpty(rules.errors); + return rules; +} + +function parseSingleBucketDescription(yaml: string) { + const rules = parseSyncRules(yaml); + expect(rules.bucketDescriptors).toHaveLength(1); + return rules.bucketDescriptors[0]; +} From 626358aaca7620beaa2e8db3fe4842ed554603ee Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 18 Jun 2025 12:52:32 +0200 Subject: [PATCH 02/21] Support customizing subscriptions --- .../src/sync/BucketChecksumState.ts | 53 ++++++++++++++++--- packages/service-core/src/sync/sync.ts | 1 + .../service-core/src/util/protocol-types.ts | 51 +++++++++++++++++- packages/sync-rules/src/BucketDescription.ts | 4 ++ .../sync-rules/src/SqlBucketDescriptor.ts | 14 ++++- packages/sync-rules/src/SqlParameterQuery.ts | 1 + packages/sync-rules/src/SqlSyncRules.ts | 43 +++++++++++++-- .../sync-rules/src/StaticSqlParameterQuery.ts | 1 + .../TableValuedFunctionSqlParameterQuery.ts | 1 + packages/sync-rules/src/types.ts | 8 +++ .../sync-rules/test/src/sync_rules.test.ts | 38 +++++++------ packages/sync-rules/test/src/util.ts | 15 ++++++ 12 files changed, 201 insertions(+), 29 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index f94720279..a0d4088dc 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -1,4 +1,4 @@ -import { BucketDescription, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { BucketDescription, isValidPriority, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; import * as storage from '../storage/storage-index.js'; import * as util from '../util/util-index.js'; @@ -20,6 +20,7 @@ export interface BucketChecksumStateOptions { bucketStorage: BucketChecksumStateStorage; syncRules: SqlSyncRules; syncParams: RequestParameters; + syncRequest: util.StreamingSyncRequest; logger?: Logger; initialBucketPositions?: { name: string; after: util.InternalOpId }[]; } @@ -70,6 +71,7 @@ export class BucketChecksumState { options.bucketStorage, options.syncRules, options.syncParams, + options.syncRequest, this.logger ); this.bucketDataPositions = new Map(); @@ -182,12 +184,12 @@ export class BucketChecksumState { const updatedBucketDescriptions = diff.updatedBuckets.map((e) => ({ ...e, - priority: bucketDescriptionMap.get(e.bucket)!.priority + ...bucketDescriptionMap.get(e.bucket)! })); bucketsToFetch = [...generateBucketsToFetch].map((b) => { return { - bucket: b, - priority: bucketDescriptionMap.get(b)!.priority + ...bucketDescriptionMap.get(b)!, + bucket: b }; }); @@ -227,7 +229,7 @@ export class BucketChecksumState { write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined, buckets: [...checksumMap.values()].map((e) => ({ ...e, - priority: bucketDescriptionMap.get(e.bucket)!.priority + ...bucketDescriptionMap.get(e.bucket)! })) } } satisfies util.StreamingSyncCheckpoint; @@ -336,6 +338,7 @@ export class BucketParameterState { public readonly syncParams: RequestParameters; private readonly querier: BucketParameterQuerier; private readonly staticBuckets: Map; + private readonly explicitStreamSubscriptions: Record; private readonly logger: Logger; private cachedDynamicBuckets: BucketDescription[] | null = null; private cachedDynamicBucketSet: Set | null = null; @@ -347,6 +350,7 @@ export class BucketParameterState { bucketStorage: BucketChecksumStateStorage, syncRules: SqlSyncRules, syncParams: RequestParameters, + request: util.StreamingSyncRequest, logger: Logger ) { this.context = context; @@ -355,11 +359,48 @@ export class BucketParameterState { this.syncParams = syncParams; this.logger = logger; - this.querier = syncRules.getBucketParameterQuerier(this.syncParams); + const explicitStreamSubscriptions: Record = {}; + const subscriptions = request.subscriptions; + if (subscriptions) { + for (const subscription of subscriptions.subscriptions) { + explicitStreamSubscriptions[subscription.stream] = subscription; + } + } + this.explicitStreamSubscriptions = explicitStreamSubscriptions; + + this.querier = syncRules.getBucketParameterQuerier({ + globalParameters: this.syncParams, + hasDefaultSubscriptions: subscriptions?.include_defaults ?? true, + resolveSubscription(name) { + const subscription = explicitStreamSubscriptions[name]; + if (subscription) { + return subscription.parameters ?? {}; + } else { + return null; + } + } + }); this.staticBuckets = new Map(this.querier.staticBuckets.map((b) => [b.bucket, b])); this.lookups = new Set(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values))); } + /** + * Overrides the `description` based on subscriptions from the client. + * + * In partiuclar, this can override the priority assigned to a bucket. + */ + overrideBucketDescription(description: BucketDescription): BucketDescription { + const changedPriority = this.explicitStreamSubscriptions[description.definition]?.override_priority; + if (changedPriority != null && isValidPriority(changedPriority)) { + return { + ...description, + priority: changedPriority + }; + } else { + return description; + } + } + async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise { const querier = this.querier; let update: CheckpointUpdate; diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index cc8bfea30..91aa2bda0 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -100,6 +100,7 @@ async function* streamResponseInner( bucketStorage, syncRules, syncParams, + syncRequest: params, initialBucketPositions: params.buckets?.map((bucket) => ({ name: bucket.name, after: BigInt(bucket.after) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index d061e16d1..5be87a07a 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -13,9 +13,51 @@ export const BucketRequest = t.object({ export type BucketRequest = t.Decoded; +/** + * An explicit subscription to a defined sync stream made by the client. + */ +export const StreamSubscription = t.object({ + /** + * The defined name of the stream as it appears in sync rules. + */ + stream: t.string, + /** + * An optional dictionary of parameters to pass to this specific stream. + */ + parameters: t.record(t.any).optional(), + /** + * Set when the client wishes to re-assign a different priority to this subscription. + * + * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be + * useful when the priority for partial syncs depends on e.g. the current page opened in a client. + */ + override_priority: t.number.optional() +}); + +export type StreamSubscription = t.Decoded; + +/** + * An overview of all subscriptions as part of a streaming sync request. + */ +export const StreamSubscriptions = t.object({ + /** + * Whether to sync default streams. + * + * When disabled,only + */ + include_defaults: t.boolean.optional(), + + /** + * An array of streams the client has subscribed to. + */ + subscriptions: t.array(StreamSubscription) +}); + +export type StreamSubscriptions = t.Decoded; + export const StreamingSyncRequest = t.object({ /** - * Existing bucket states. + * Existing client-side bucket states. */ buckets: t.array(BucketRequest).optional(), @@ -47,7 +89,12 @@ export const StreamingSyncRequest = t.object({ /** * Unique client id. */ - client_id: t.string.optional() + client_id: t.string.optional(), + + /** + * If the client is aware of stream subscriptions, an array of streams the client is subscribing to. + */ + subscriptions: StreamSubscriptions.optional() }); export type StreamingSyncRequest = t.Decoded; diff --git a/packages/sync-rules/src/BucketDescription.ts b/packages/sync-rules/src/BucketDescription.ts index dbaec0da6..1d54faafa 100644 --- a/packages/sync-rules/src/BucketDescription.ts +++ b/packages/sync-rules/src/BucketDescription.ts @@ -20,6 +20,10 @@ export const isValidPriority = (i: number): i is BucketPriority => { }; export interface BucketDescription { + /** + * The name of the sync rule or stream definition from which the bucket is derived. + */ + definition: string; /** * The id of the bucket, which is derived from the name of the bucket's definition * in the sync rules as well as the values returned by the parameter queries. diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 22714e845..7116984de 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -29,12 +29,23 @@ export interface QueryParseResult { errors: SqlRuleError[]; } +export enum SqlBucketDescriptorType { + SYNC_RULE, + STREAM +} + export class SqlBucketDescriptor { name: string; bucketParameters?: string[]; + type: SqlBucketDescriptorType; + subscribedToByDefault: boolean; - constructor(name: string) { + constructor(name: string, type: SqlBucketDescriptorType) { this.name = name; + this.type = type; + + // Sync-rule style buckets are subscribed to by default, streams are opt-in unless their definition says otherwise. + this.subscribedToByDefault = type == SqlBucketDescriptorType.SYNC_RULE; } /** @@ -93,6 +104,7 @@ export class SqlBucketDescriptor { } } this.dataQueries.push(query.data); + this.subscribedToByDefault = options.default ?? false; return { parsed: true, diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index 85b6dfefc..be509bf7e 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -367,6 +367,7 @@ export class SqlParameterQuery { } return { + definition: this.descriptorName, bucket: getBucketId(this.descriptorName, this.bucketParameters, result), priority: this.priority }; diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index e8943b66a..aeee85916 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -6,7 +6,7 @@ import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; import { IdSequence } from './IdSequence.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; -import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; +import { QueryParseResult, SqlBucketDescriptor, SqlBucketDescriptorType } from './SqlBucketDescriptor.js'; import { TablePattern } from './TablePattern.js'; import { EvaluatedParameters, @@ -40,6 +40,22 @@ export interface SyncRulesOptions { throwOnError?: boolean; } +export interface GetQuerierOptions { + globalParameters: RequestParameters; + /** + * Whether the client is subscribing to default subscriptions (the default). + */ + hasDefaultSubscriptions: boolean; + /** + * For streams, this is invoked to check whether the client has requested a subscription to + * the stream. + * + * @param name The name of the stream as it appears in the sync rule definitions. + * @returns If a subscription is active, the stream parameters for that particular stream. Otherwise null. + */ + resolveSubscription: (name: string) => Record | null; +} + export class SqlSyncRules implements SyncRules { bucketDescriptors: SqlBucketDescriptor[] = []; eventDescriptors: SqlEventDescriptor[] = []; @@ -144,7 +160,7 @@ export class SqlSyncRules implements SyncRules { const parameters = value.get('parameters', true) as unknown; const dataQueries = value.get('data', true) as unknown; - const descriptor = new SqlBucketDescriptor(key); + const descriptor = new SqlBucketDescriptor(key, SqlBucketDescriptorType.SYNC_RULE); if (parameters instanceof Scalar) { rules.withScalar(parameters, (q) => { @@ -179,7 +195,7 @@ export class SqlSyncRules implements SyncRules { continue; } - const descriptor = new SqlBucketDescriptor(key); + const descriptor = new SqlBucketDescriptor(key, SqlBucketDescriptorType.STREAM); const accept_potentially_dangerous_queries = value.get('accept_potentially_dangerous_queries', true)?.value == true; @@ -355,8 +371,25 @@ export class SqlSyncRules implements SyncRules { return { results, errors }; } - getBucketParameterQuerier(parameters: RequestParameters): BucketParameterQuerier { - const queriers = this.bucketDescriptors.map((query) => query.getBucketParameterQuerier(parameters)); + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + for (const descriptor of this.bucketDescriptors) { + let params = options.globalParameters; + const subscription = + descriptor.type == SqlBucketDescriptorType.STREAM ? options.resolveSubscription(descriptor.name) : null; + + if (!descriptor.subscribedToByDefault && subscription == null) { + // The client is not subscribing to this stream, so don't query buckets related to it. + continue; + } + + if (subscription != null) { + params = params.withAddedParameters(subscription); + } + + queriers.push(descriptor.getBucketParameterQuerier(params)); + } + return mergeBucketParameterQueriers(queriers); } diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index 414ee2bdd..ed461401e 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -177,6 +177,7 @@ export class StaticSqlParameterQuery { return [ { + definition: this.descriptorName, bucket: getBucketId(this.descriptorName, this.bucketParameters, result), priority: this.priority } diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index 57a5451a9..b6d15b2cd 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -249,6 +249,7 @@ export class TableValuedFunctionSqlParameterQuery { } return { + definition: this.descriptorName, bucket: getBucketId(this.descriptorName, this.bucketParameters, result), priority: this.priority }; diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index b03b07c06..da80d68b3 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -135,6 +135,14 @@ export class RequestParameters implements ParameterValueSet { } throw new Error(`Unknown table: ${table}`); } + + withAddedParameters(params: Record): RequestParameters { + const clone = structuredClone(this); + clone.rawUserParameters = JSONBig.stringify(params); + clone.userParameters = toSyncRulesParameters(params); + + return clone; + } } /** diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 91f3357ac..26cf7e42f 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -1,7 +1,15 @@ import { describe, expect, test } from 'vitest'; import { ParameterLookup, SqlSyncRules } from '../../src/index.js'; -import { ASSETS, BASIC_SCHEMA, PARSE_OPTIONS, TestSourceTable, USERS, normalizeTokenParameters } from './util.js'; +import { + ASSETS, + BASIC_SCHEMA, + PARSE_OPTIONS, + TestSourceTable, + USERS, + normalizeQuerierOptions, + normalizeTokenParameters +} from './util.js'; describe('sync rules', () => { test('parse empty sync rules', () => { @@ -37,7 +45,7 @@ bucket_definitions: } ]); expect(rules.hasDynamicBucketQueries()).toBe(false); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); @@ -61,15 +69,15 @@ bucket_definitions: expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ is_admin: true }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true }))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ is_admin: false }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false }))).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); @@ -114,7 +122,7 @@ bucket_definitions: const param_query = bucket.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - rules.getBucketParameterQuerier(normalizeTokenParameters({ user_id: 'user1' }, { device_id: 'device1' })) + rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) .staticBuckets ).toEqual([{ bucket: 'mybucket["user1","device1"]', priority: 3 }]); @@ -159,7 +167,7 @@ bucket_definitions: expect(bucket.bucketParameters).toEqual(['user_id']); const param_query = bucket.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ user_id: 'user1' })).staticBuckets).toEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).staticBuckets).toEqual([ { bucket: 'mybucket["user1"]', priority: 3 } ]); @@ -301,7 +309,7 @@ bucket_definitions: ); const bucket = rules.bucketDescriptors[0]; expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ user_id: 'user1' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); @@ -338,7 +346,7 @@ bucket_definitions: ); const bucket = rules.bucketDescriptors[0]; expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ user_id: 'user1' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); @@ -503,7 +511,7 @@ bucket_definitions: } ]); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ is_admin: true })).staticBuckets).toEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).staticBuckets).toEqual([ { bucket: 'mybucket[1]', priority: 3 } ]); }); @@ -546,7 +554,7 @@ bucket_definitions: PARSE_OPTIONS ); expect( - rules.getBucketParameterQuerier(normalizeTokenParameters({ int1: 314, float1: 3.14, float2: 314 })) + rules.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })) ).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[314,3.14,314]', priority: 3 }] }); expect( @@ -574,7 +582,7 @@ bucket_definitions: PARSE_OPTIONS ); expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ user_id: 'test' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' }))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["TEST"]', priority: 3 }], hasDynamicBuckets: false }); @@ -827,7 +835,7 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -852,7 +860,7 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -917,7 +925,7 @@ bucket_definitions: expect(bucket.bucketParameters).toEqual(['user_id']); expect(rules.hasDynamicBucketQueries()).toBe(true); - expect(rules.getBucketParameterQuerier(normalizeTokenParameters({ user_id: 'user1' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ ParameterLookup.normalized('mybucket', '2', ['user1']), diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index b780cd0c2..bb0066851 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -1,5 +1,6 @@ import { DEFAULT_TAG, + GetQuerierOptions, RequestJwtPayload, RequestParameters, SourceTableInterface, @@ -60,3 +61,17 @@ export function normalizeTokenParameters( delete tokenPayload.parameters.user_id; return new RequestParameters(tokenPayload, user_parameters ?? {}); } + +export function normalizeQuerierOptions( + token_parameters: Record, + user_parameters?: Record +): GetQuerierOptions { + const globalParameters = normalizeTokenParameters(token_parameters, user_parameters); + return { + globalParameters, + hasDefaultSubscriptions: true, + resolveSubscription(name) { + return null; + } + }; +} From 0d575b4ada37e50b4680d10702e2186087a4a5ff Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 18 Jun 2025 14:56:06 +0200 Subject: [PATCH 03/21] Include query streams in checkpoint message --- .../src/sync/BucketChecksumState.ts | 28 +++++++++++++++++-- .../service-core/src/util/protocol-types.ts | 6 ++++ packages/sync-rules/src/SqlSyncRules.ts | 23 ++++++++------- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index a0d4088dc..f2375cb1b 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -14,6 +14,7 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js'; import { SyncContext } from './SyncContext.js'; import { getIntersection, hasIntersection } from './util.js'; +import { SqlBucketDescriptor, SqlBucketDescriptorType } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js'; export interface BucketChecksumStateOptions { syncContext: SyncContext; @@ -102,7 +103,9 @@ export class BucketChecksumState { const { buckets: allBuckets, updatedBuckets } = update; /** Set of all buckets in this checkpoint. */ - const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b])); + const bucketDescriptionMap = new Map( + allBuckets.map((b) => [b.bucket, this.parameterState.overrideBucketDescription(b)]) + ); if (bucketDescriptionMap.size > this.context.maxBuckets) { throw new ServiceError( @@ -223,6 +226,18 @@ export class BucketChecksumState { this.logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length }); }; bucketsToFetch = allBuckets; + this.parameterState.syncRules.bucketDescriptors; + + const subscriptions: util.SubscribedStream[] = []; + for (const desc of this.parameterState.syncRules.bucketDescriptors) { + if (desc.type == SqlBucketDescriptorType.STREAM && this.parameterState.isSubscribedToStream(desc)) { + subscriptions.push({ + name: desc.name, + is_default: desc.subscribedToByDefault + }); + } + } + checkpointLine = { checkpoint: { last_op_id: util.internalToExternalOpId(base.checkpoint), @@ -230,7 +245,8 @@ export class BucketChecksumState { buckets: [...checksumMap.values()].map((e) => ({ ...e, ...bucketDescriptionMap.get(e.bucket)! - })) + })), + included_subscriptions: subscriptions } } satisfies util.StreamingSyncCheckpoint; } @@ -338,6 +354,7 @@ export class BucketParameterState { public readonly syncParams: RequestParameters; private readonly querier: BucketParameterQuerier; private readonly staticBuckets: Map; + private readonly includeDefaultStreams: boolean; private readonly explicitStreamSubscriptions: Record; private readonly logger: Logger; private cachedDynamicBuckets: BucketDescription[] | null = null; @@ -366,11 +383,12 @@ export class BucketParameterState { explicitStreamSubscriptions[subscription.stream] = subscription; } } + this.includeDefaultStreams = subscriptions?.include_defaults ?? true; this.explicitStreamSubscriptions = explicitStreamSubscriptions; this.querier = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, - hasDefaultSubscriptions: subscriptions?.include_defaults ?? true, + hasDefaultSubscriptions: this.includeDefaultStreams, resolveSubscription(name) { const subscription = explicitStreamSubscriptions[name]; if (subscription) { @@ -401,6 +419,10 @@ export class BucketParameterState { } } + isSubscribedToStream(desc: SqlBucketDescriptor): boolean { + return (desc.subscribedToByDefault && this.includeDefaultStreams) || desc.name in this.explicitStreamSubscriptions; + } + async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise { const querier = this.querier; let update: CheckpointUpdate; diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 5be87a07a..9118fe499 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -146,10 +146,16 @@ export type StreamingSyncLine = */ export type ProtocolOpId = string; +export interface SubscribedStream { + name: string; + is_default: boolean; +} + export interface Checkpoint { last_op_id: ProtocolOpId; write_checkpoint?: ProtocolOpId; buckets: BucketChecksumWithDescription[]; + included_subscriptions: SubscribedStream[]; } export interface BucketState { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index aeee85916..ef40a70ed 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,9 +1,8 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; -import { BucketPriority, isValidPriority } from './BucketDescription.js'; +import { isValidPriority } from './BucketDescription.js'; import { BucketParameterQuerier, mergeBucketParameterQueriers } from './BucketParameterQuerier.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; -import { IdSequence } from './IdSequence.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { QueryParseResult, SqlBucketDescriptor, SqlBucketDescriptorType } from './SqlBucketDescriptor.js'; @@ -375,16 +374,20 @@ export class SqlSyncRules implements SyncRules { const queriers: BucketParameterQuerier[] = []; for (const descriptor of this.bucketDescriptors) { let params = options.globalParameters; - const subscription = - descriptor.type == SqlBucketDescriptorType.STREAM ? options.resolveSubscription(descriptor.name) : null; - if (!descriptor.subscribedToByDefault && subscription == null) { - // The client is not subscribing to this stream, so don't query buckets related to it. - continue; - } + if (descriptor.type == SqlBucketDescriptorType.STREAM) { + const subscription = options.resolveSubscription(descriptor.name); + + if (!descriptor.subscribedToByDefault && subscription == null) { + // The client is not subscribing to this stream, so don't query buckets related to it. + continue; + } + + if (subscription != null) { + params = params.withAddedParameters(subscription); + } - if (subscription != null) { - params = params.withAddedParameters(subscription); + queriers.push(descriptor.getBucketParameterQuerier(params)); } queriers.push(descriptor.getBucketParameterQuerier(params)); From 8af7af4138812621aac9e7ca775ff301f6e27f87 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Jun 2025 11:40:38 +0200 Subject: [PATCH 04/21] Avoid subscription terminology --- .../src/sync/BucketChecksumState.ts | 20 ++++++------- .../service-core/src/util/protocol-types.ts | 28 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index f2375cb1b..bbd32830d 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -228,7 +228,7 @@ export class BucketChecksumState { bucketsToFetch = allBuckets; this.parameterState.syncRules.bucketDescriptors; - const subscriptions: util.SubscribedStream[] = []; + const subscriptions: util.StreamDescription[] = []; for (const desc of this.parameterState.syncRules.bucketDescriptors) { if (desc.type == SqlBucketDescriptorType.STREAM && this.parameterState.isSubscribedToStream(desc)) { subscriptions.push({ @@ -246,7 +246,7 @@ export class BucketChecksumState { ...e, ...bucketDescriptionMap.get(e.bucket)! })), - included_subscriptions: subscriptions + streams: subscriptions } } satisfies util.StreamingSyncCheckpoint; } @@ -355,7 +355,7 @@ export class BucketParameterState { private readonly querier: BucketParameterQuerier; private readonly staticBuckets: Map; private readonly includeDefaultStreams: boolean; - private readonly explicitStreamSubscriptions: Record; + private readonly explicitlyOpenedStreams: Record; private readonly logger: Logger; private cachedDynamicBuckets: BucketDescription[] | null = null; private cachedDynamicBucketSet: Set | null = null; @@ -376,21 +376,21 @@ export class BucketParameterState { this.syncParams = syncParams; this.logger = logger; - const explicitStreamSubscriptions: Record = {}; + const explicitlyOpenedStreams: Record = {}; const subscriptions = request.subscriptions; if (subscriptions) { - for (const subscription of subscriptions.subscriptions) { - explicitStreamSubscriptions[subscription.stream] = subscription; + for (const subscription of subscriptions.opened) { + explicitlyOpenedStreams[subscription.stream] = subscription; } } this.includeDefaultStreams = subscriptions?.include_defaults ?? true; - this.explicitStreamSubscriptions = explicitStreamSubscriptions; + this.explicitlyOpenedStreams = explicitlyOpenedStreams; this.querier = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, hasDefaultSubscriptions: this.includeDefaultStreams, resolveSubscription(name) { - const subscription = explicitStreamSubscriptions[name]; + const subscription = explicitlyOpenedStreams[name]; if (subscription) { return subscription.parameters ?? {}; } else { @@ -408,7 +408,7 @@ export class BucketParameterState { * In partiuclar, this can override the priority assigned to a bucket. */ overrideBucketDescription(description: BucketDescription): BucketDescription { - const changedPriority = this.explicitStreamSubscriptions[description.definition]?.override_priority; + const changedPriority = this.explicitlyOpenedStreams[description.definition]?.override_priority; if (changedPriority != null && isValidPriority(changedPriority)) { return { ...description, @@ -420,7 +420,7 @@ export class BucketParameterState { } isSubscribedToStream(desc: SqlBucketDescriptor): boolean { - return (desc.subscribedToByDefault && this.includeDefaultStreams) || desc.name in this.explicitStreamSubscriptions; + return (desc.subscribedToByDefault && this.includeDefaultStreams) || desc.name in this.explicitlyOpenedStreams; } async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise { diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 9118fe499..252d73ceb 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -14,11 +14,11 @@ export const BucketRequest = t.object({ export type BucketRequest = t.Decoded; /** - * An explicit subscription to a defined sync stream made by the client. + * A sync steam that a client has expressed interest in by explicitly opening it on the client side. */ -export const StreamSubscription = t.object({ +export const OpenedStream = t.object({ /** - * The defined name of the stream as it appears in sync rules. + * The defined name of the stream as it appears in sync stream definitions. */ stream: t.string, /** @@ -26,7 +26,7 @@ export const StreamSubscription = t.object({ */ parameters: t.record(t.any).optional(), /** - * Set when the client wishes to re-assign a different priority to this subscription. + * Set when the client wishes to re-assign a different priority to this stream. * * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be * useful when the priority for partial syncs depends on e.g. the current page opened in a client. @@ -34,12 +34,12 @@ export const StreamSubscription = t.object({ override_priority: t.number.optional() }); -export type StreamSubscription = t.Decoded; +export type OpenedStream = t.Decoded; /** - * An overview of all subscriptions as part of a streaming sync request. + * An overview of all opened streams as part of a streaming sync request. */ -export const StreamSubscriptions = t.object({ +export const OpenedStreams = t.object({ /** * Whether to sync default streams. * @@ -48,12 +48,12 @@ export const StreamSubscriptions = t.object({ include_defaults: t.boolean.optional(), /** - * An array of streams the client has subscribed to. + * An array of sync streams the client has opened explicitly. */ - subscriptions: t.array(StreamSubscription) + opened: t.array(OpenedStream) }); -export type StreamSubscriptions = t.Decoded; +export type StreamSubscriptions = t.Decoded; export const StreamingSyncRequest = t.object({ /** @@ -92,9 +92,9 @@ export const StreamingSyncRequest = t.object({ client_id: t.string.optional(), /** - * If the client is aware of stream subscriptions, an array of streams the client is subscribing to. + * If the client is aware of streams, an array of streams the client has opened. */ - subscriptions: StreamSubscriptions.optional() + subscriptions: OpenedStreams.optional() }); export type StreamingSyncRequest = t.Decoded; @@ -146,7 +146,7 @@ export type StreamingSyncLine = */ export type ProtocolOpId = string; -export interface SubscribedStream { +export interface StreamDescription { name: string; is_default: boolean; } @@ -155,7 +155,7 @@ export interface Checkpoint { last_op_id: ProtocolOpId; write_checkpoint?: ProtocolOpId; buckets: BucketChecksumWithDescription[]; - included_subscriptions: SubscribedStream[]; + streams: StreamDescription[]; } export interface BucketState { From bd2962cd3f23717401c562512932fb315bc1ad3c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 19 Jun 2025 14:19:11 +0200 Subject: [PATCH 05/21] Rename more stream subscription references --- .../src/sync/BucketChecksumState.ts | 4 +- packages/sync-rules/src/SqlSyncRules.ts | 22 ++++---- .../TableValuedFunctionSqlParameterQuery.ts | 4 +- packages/sync-rules/src/request_functions.ts | 51 +++++++++++-------- packages/sync-rules/src/sql_filters.ts | 8 +-- packages/sync-rules/src/types.ts | 13 +++-- 6 files changed, 56 insertions(+), 46 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index bbd32830d..f4b9a4db3 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -388,8 +388,8 @@ export class BucketParameterState { this.querier = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, - hasDefaultSubscriptions: this.includeDefaultStreams, - resolveSubscription(name) { + hasDefaultStreams: this.includeDefaultStreams, + resolveOpenedStream(name) { const subscription = explicitlyOpenedStreams[name]; if (subscription) { return subscription.parameters ?? {}; diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index ef40a70ed..a9bd63ccf 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -42,17 +42,19 @@ export interface SyncRulesOptions { export interface GetQuerierOptions { globalParameters: RequestParameters; /** - * Whether the client is subscribing to default subscriptions (the default). + * Whether the client is subscribing to default query streams. + * + * Client do this by default, but can disable the behavior if needed. */ - hasDefaultSubscriptions: boolean; + hasDefaultStreams: boolean; /** - * For streams, this is invoked to check whether the client has requested a subscription to - * the stream. + * For streams, this is invoked to check whether the client has opened the relevant stream. * * @param name The name of the stream as it appears in the sync rule definitions. - * @returns If a subscription is active, the stream parameters for that particular stream. Otherwise null. + * @returns If the strema has been opened by the client, the stream parameters for that particular stream. Otherwise + * null. */ - resolveSubscription: (name: string) => Record | null; + resolveOpenedStream: (name: string) => Record | null; } export class SqlSyncRules implements SyncRules { @@ -376,15 +378,15 @@ export class SqlSyncRules implements SyncRules { let params = options.globalParameters; if (descriptor.type == SqlBucketDescriptorType.STREAM) { - const subscription = options.resolveSubscription(descriptor.name); + const opened = options.resolveOpenedStream(descriptor.name); - if (!descriptor.subscribedToByDefault && subscription == null) { + if (!descriptor.subscribedToByDefault && opened == null) { // The client is not subscribing to this stream, so don't query buckets related to it. continue; } - if (subscription != null) { - params = params.withAddedParameters(subscription); + if (opened != null) { + params = params.withAddedStreamParameters(opened); } queriers.push(descriptor.getBucketParameterQuerier(params)); diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index b6d15b2cd..c56ef4b14 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -222,9 +222,7 @@ export class TableValuedFunctionSqlParameterQuery { private getIndividualBucketDescription(row: SqliteRow, parameters: RequestParameters): BucketDescription | null { const mergedParams: ParameterValueSet = { - rawTokenPayload: parameters.rawTokenPayload, - rawUserParameters: parameters.rawUserParameters, - userId: parameters.userId, + ...parameters, lookup: (table, column) => { if (table == this.callTableName) { return row[column]!; diff --git a/packages/sync-rules/src/request_functions.ts b/packages/sync-rules/src/request_functions.ts index 00cd89b58..caa659adb 100644 --- a/packages/sync-rules/src/request_functions.ts +++ b/packages/sync-rules/src/request_functions.ts @@ -13,21 +13,19 @@ export interface SqlParameterFunction { documentation: string; } -const parametersFunction = (name: string): SqlParameterFunction => { - return { - debugName: name, - call(parameters: ParameterValueSet) { - return parameters.rawUserParameters; - }, - getReturnType() { - return ExpressionType.TEXT; - }, - detail: 'Unauthenticated request parameters as JSON', - documentation: - 'Returns parameters passed by the client as a JSON string. These parameters are not authenticated - any value can be passed in by the client.', - usesAuthenticatedRequestParameters: false, - usesUnauthenticatedRequestParameters: true - }; +const request_parameters: SqlParameterFunction = { + debugName: 'request.parameters', + call(parameters: ParameterValueSet) { + return parameters.rawUserParameters; + }, + getReturnType() { + return ExpressionType.TEXT; + }, + detail: 'Unauthenticated request parameters as JSON', + documentation: + 'Returns parameters passed by the client as a JSON string. These parameters are not authenticated - any value can be passed in by the client.', + usesAuthenticatedRequestParameters: false, + usesUnauthenticatedRequestParameters: true }; const request_jwt: SqlParameterFunction = { @@ -58,16 +56,25 @@ const request_user_id: SqlParameterFunction = { usesUnauthenticatedRequestParameters: false }; -export const REQUEST_FUNCTIONS_WITHOUT_PARAMETERS: Record = { +export const REQUEST_FUNCTIONS: Record = { + parameters: request_parameters, jwt: request_jwt, user_id: request_user_id }; -export const REQUEST_FUNCTIONS: Record = { - parameters: parametersFunction('request.parameters'), - ...REQUEST_FUNCTIONS_WITHOUT_PARAMETERS -}; - export const QUERY_FUNCTIONS: Record = { - params: parametersFunction('query.params') + params: { + debugName: 'stream.params', + call(parameters: ParameterValueSet) { + return parameters.rawUserParameters; + }, + getReturnType() { + return ExpressionType.TEXT; + }, + detail: 'Unauthenticated stream parameters as JSON', + documentation: + 'Returns stream passed by the client when opening the stream. These parameters are not authenticated - any value can be passed in by the client.', + usesAuthenticatedRequestParameters: false, + usesUnauthenticatedRequestParameters: true + } }; diff --git a/packages/sync-rules/src/sql_filters.ts b/packages/sync-rules/src/sql_filters.ts index d9a31c061..cde93d56b 100644 --- a/packages/sync-rules/src/sql_filters.ts +++ b/packages/sync-rules/src/sql_filters.ts @@ -427,10 +427,6 @@ export class SqlTools { return this.error(`${schema} schema is not available in data queries`, expr); } - if (fn == 'parameters' && this.supportsStreamInputs) { - return this.error(`'request.parameters()' is unavailable on streams - use 'stream.params()' instead.`, expr); - } - if (expr.args.length > 0) { return this.error(`Function '${schema}.${fn}' does not take arguments`, expr); } @@ -438,7 +434,7 @@ export class SqlTools { if (fn in REQUEST_FUNCTIONS) { const fnImpl = REQUEST_FUNCTIONS[fn]; return { - key: 'request.parameters()', + key: `stream.${fn}()`, lookupParameterValue(parameters) { return fnImpl.call(parameters); }, @@ -456,7 +452,7 @@ export class SqlTools { if (fn in QUERY_FUNCTIONS) { const fnImpl = QUERY_FUNCTIONS[fn]; return { - key: 'stream.params()', + key: `stream.${fn}()`, lookupParameterValue(parameters) { return fnImpl.call(parameters); }, diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index da80d68b3..91acd1afd 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -85,6 +85,11 @@ export interface ParameterValueSet { */ rawUserParameters: string; + /** + * For streams, the raw JSON string of stream parameters. + */ + rawStreamParameters: string | null; + /** * JSON string of raw request parameters. */ @@ -102,6 +107,8 @@ export class RequestParameters implements ParameterValueSet { */ rawUserParameters: string; + rawStreamParameters: string | null; + /** * JSON string of raw request parameters. */ @@ -125,6 +132,7 @@ export class RequestParameters implements ParameterValueSet { this.rawUserParameters = JSONBig.stringify(clientParameters); this.userParameters = toSyncRulesParameters(clientParameters); + this.rawStreamParameters = null; } lookup(table: string, column: string): SqliteJsonValue { @@ -136,10 +144,9 @@ export class RequestParameters implements ParameterValueSet { throw new Error(`Unknown table: ${table}`); } - withAddedParameters(params: Record): RequestParameters { + withAddedStreamParameters(params: Record): RequestParameters { const clone = structuredClone(this); - clone.rawUserParameters = JSONBig.stringify(params); - clone.userParameters = toSyncRulesParameters(params); + clone.rawStreamParameters = JSONBig.stringify(params); return clone; } From 55340335f8f9eb5d1acb4e484b05573c2be1f75f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 21 Jul 2025 12:29:41 +0200 Subject: [PATCH 06/21] Translate resolved buckets --- .../src/sync/BucketChecksumState.ts | 84 +++++++++++++------ .../service-core/src/util/protocol-types.ts | 55 +++++++++--- packages/sync-rules/src/BucketDescription.ts | 13 +++ packages/sync-rules/src/SqlSyncRules.ts | 43 ++++++++-- 4 files changed, 151 insertions(+), 44 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index f4b9a4db3..a7af600b6 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -1,4 +1,12 @@ -import { BucketDescription, isValidPriority, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { + BucketDescription, + BucketPriority, + isValidPriority, + RequestedStream, + RequestParameters, + ResolvedBucket, + SqlSyncRules +} from '@powersync/service-sync-rules'; import * as storage from '../storage/storage-index.js'; import * as util from '../util/util-index.js'; @@ -355,7 +363,9 @@ export class BucketParameterState { private readonly querier: BucketParameterQuerier; private readonly staticBuckets: Map; private readonly includeDefaultStreams: boolean; - private readonly explicitlyOpenedStreams: Record; + // Indexed by the client-side id + private readonly explicitStreamSubscriptions: Record; + private readonly subscribedStreamNames: Set; private readonly logger: Logger; private cachedDynamicBuckets: BucketDescription[] | null = null; private cachedDynamicBucketSet: Set | null = null; @@ -376,51 +386,73 @@ export class BucketParameterState { this.syncParams = syncParams; this.logger = logger; - const explicitlyOpenedStreams: Record = {}; + const idToStreamSubscription: Record = {}; + const streamsByName: Record = {}; const subscriptions = request.subscriptions; if (subscriptions) { for (const subscription of subscriptions.opened) { - explicitlyOpenedStreams[subscription.stream] = subscription; + idToStreamSubscription[subscription.stream] = subscription; + + const syncRuleStream: RequestedStream = { + parameters: subscription.parameters ?? {}, + opaque_id: subscription.client_id + }; + if (Object.hasOwn(streamsByName, subscription.stream)) { + streamsByName[subscription.stream].push(syncRuleStream); + } else { + streamsByName[subscription.stream] = [syncRuleStream]; + } } } this.includeDefaultStreams = subscriptions?.include_defaults ?? true; - this.explicitlyOpenedStreams = explicitlyOpenedStreams; + this.explicitStreamSubscriptions = idToStreamSubscription; this.querier = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, hasDefaultStreams: this.includeDefaultStreams, - resolveOpenedStream(name) { - const subscription = explicitlyOpenedStreams[name]; - if (subscription) { - return subscription.parameters ?? {}; - } else { - return null; - } - } + streams: streamsByName }); this.staticBuckets = new Map(this.querier.staticBuckets.map((b) => [b.bucket, b])); this.lookups = new Set(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values))); + this.subscribedStreamNames = new Set(Object.keys(streamsByName)); } /** - * Overrides the `description` based on subscriptions from the client. - * - * In partiuclar, this can override the priority assigned to a bucket. + * Translates an internal sync-rules {@link ResolvedBucket} instance to the public + * {@link util.ClientBucketDescription}. */ - overrideBucketDescription(description: BucketDescription): BucketDescription { - const changedPriority = this.explicitlyOpenedStreams[description.definition]?.override_priority; - if (changedPriority != null && isValidPriority(changedPriority)) { - return { - ...description, - priority: changedPriority - }; - } else { - return description; + translateResolvedBucket(description: ResolvedBucket): util.ClientBucketDescription { + // Assign + let priorityOverride: BucketPriority | null = null; + for (const reason of description.inclusion_reasons) { + if (reason != 'default') { + const requestedPriority = this.explicitStreamSubscriptions[reason.subscription]?.override_priority; + if (requestedPriority != null) { + if (priorityOverride == null) { + priorityOverride = requestedPriority as BucketPriority; + } else { + priorityOverride = Math.min(requestedPriority, priorityOverride) as BucketPriority; + } + } + } } + + return { + definition: description.definition, + bucket: description.bucket, + priority: priorityOverride ?? description.priority, + subscriptions: description.inclusion_reasons.map((reason) => { + if (reason == 'default') { + return { def: description.definition }; + } else { + return { sub: reason.subscription }; + } + }) + }; } isSubscribedToStream(desc: SqlBucketDescriptor): boolean { - return (desc.subscribedToByDefault && this.includeDefaultStreams) || desc.name in this.explicitlyOpenedStreams; + return (desc.subscribedToByDefault && this.includeDefaultStreams) || this.subscribedStreamNames.has(desc.name); } async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise { diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 252d73ceb..c287878c1 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -16,11 +16,17 @@ export type BucketRequest = t.Decoded; /** * A sync steam that a client has expressed interest in by explicitly opening it on the client side. */ -export const OpenedStream = t.object({ +export const RequestedStreamSubscription = t.object({ /** * The defined name of the stream as it appears in sync stream definitions. */ stream: t.string, + /** + * An opaque textual identifier assigned to this request by the client. + * + * Wh + */ + client_id: t.string, /** * An optional dictionary of parameters to pass to this specific stream. */ @@ -34,26 +40,26 @@ export const OpenedStream = t.object({ override_priority: t.number.optional() }); -export type OpenedStream = t.Decoded; +export type RequestedStreamSubscription = t.Decoded; /** - * An overview of all opened streams as part of a streaming sync request. + * An overview of all subscribed streams as part of a streaming sync request. */ -export const OpenedStreams = t.object({ +export const StreamSubscriptionRequest = t.object({ /** * Whether to sync default streams. * - * When disabled,only + * When disabled, only explicitly-opened subscriptions are included. */ include_defaults: t.boolean.optional(), /** * An array of sync streams the client has opened explicitly. */ - opened: t.array(OpenedStream) + opened: t.array(RequestedStreamSubscription) }); -export type StreamSubscriptions = t.Decoded; +export type StreamSubscriptionRequest = t.Decoded; export const StreamingSyncRequest = t.object({ /** @@ -94,7 +100,7 @@ export const StreamingSyncRequest = t.object({ /** * If the client is aware of streams, an array of streams the client has opened. */ - subscriptions: OpenedStreams.optional() + subscriptions: StreamSubscriptionRequest.optional() }); export type StreamingSyncRequest = t.Decoded; @@ -107,7 +113,7 @@ export interface StreamingSyncCheckpointDiff { checkpoint_diff: { last_op_id: ProtocolOpId; write_checkpoint?: ProtocolOpId; - updated_buckets: BucketChecksumWithDescription[]; + updated_buckets: CheckpointBucket[]; removed_buckets: string[]; }; } @@ -154,7 +160,7 @@ export interface StreamDescription { export interface Checkpoint { last_op_id: ProtocolOpId; write_checkpoint?: ProtocolOpId; - buckets: BucketChecksumWithDescription[]; + buckets: CheckpointBucket[]; streams: StreamDescription[]; } @@ -211,4 +217,31 @@ export interface BucketChecksum { count: number; } -export interface BucketChecksumWithDescription extends BucketChecksum, BucketDescription {} +/** + * The reason a particular bucket is included in a checkpoint. + * + * This information allows clients to associate individual buckets with sync streams they're subscribed to. Having that + * association is useful because it enables clients to track progress for individual sync streams. + */ +export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDerivedFromExplicitSubscription; + +/** + * A bucket has been included in a checkpoint because it's part of a default stream. + * + * The string is the name of the stream definition. + */ +export type BucketDerivedFromDefaultStream = { def: string }; + +/** + * The bucket has been included in a checkpoint because it's part of a stream that a client has explicitly subscribed + * to. + * + * The string is the client id associated with the subscription in {@link RequestedStreamSubscription}. + */ +export type BucketDerivedFromExplicitSubscription = { sub: string }; + +export interface ClientBucketDescription extends BucketDescription { + subscriptions: BucketSubscriptionReason[]; +} + +export interface CheckpointBucket extends BucketChecksum, ClientBucketDescription {} diff --git a/packages/sync-rules/src/BucketDescription.ts b/packages/sync-rules/src/BucketDescription.ts index 1d54faafa..399f1300d 100644 --- a/packages/sync-rules/src/BucketDescription.ts +++ b/packages/sync-rules/src/BucketDescription.ts @@ -24,6 +24,7 @@ export interface BucketDescription { * The name of the sync rule or stream definition from which the bucket is derived. */ definition: string; + /** * The id of the bucket, which is derived from the name of the bucket's definition * in the sync rules as well as the values returned by the parameter queries. @@ -34,3 +35,15 @@ export interface BucketDescription { */ priority: BucketPriority; } + +/** + * A bucket that was resolved to a specific request including stream subscriptions. + * + * This includes information on why the bucket has been included in a checkpoint subset + * shown to clients. + */ +export interface ResolvedBucket extends BucketDescription { + inclusion_reasons: BucketInclusionReason[]; +} + +export type BucketInclusionReason = 'default' | { subscription: string }; diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index a9bd63ccf..1498df495 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -20,6 +20,7 @@ import { QueryParseOptions, RequestParameters, SourceSchema, + SqliteJsonRow, SqliteRow, StreamParseOptions, SyncRules @@ -39,6 +40,21 @@ export interface SyncRulesOptions { throwOnError?: boolean; } +export interface RequestedStream { + /** + * The parameters for the explicit stream subscription. + * + * Unlike {@link GetQuerierOptions.globalParameters}, these parameters are only applied to the particular stream. + */ + parameters: SqliteJsonRow | null; + + /** + * An opaque id of the stream subscription, used to associate buckets with the stream subscriptions that have caused + * them to be included. + */ + opaque_id: string; +} + export interface GetQuerierOptions { globalParameters: RequestParameters; /** @@ -48,13 +64,14 @@ export interface GetQuerierOptions { */ hasDefaultStreams: boolean; /** + * * For streams, this is invoked to check whether the client has opened the relevant stream. * * @param name The name of the stream as it appears in the sync rule definitions. * @returns If the strema has been opened by the client, the stream parameters for that particular stream. Otherwise * null. */ - resolveOpenedStream: (name: string) => Record | null; + streams: Record; } export class SqlSyncRules implements SyncRules { @@ -378,21 +395,33 @@ export class SqlSyncRules implements SyncRules { let params = options.globalParameters; if (descriptor.type == SqlBucketDescriptorType.STREAM) { - const opened = options.resolveOpenedStream(descriptor.name); + const subscriptions = options.streams[descriptor.name] ?? []; - if (!descriptor.subscribedToByDefault && opened == null) { + if (!descriptor.subscribedToByDefault && subscriptions.length) { // The client is not subscribing to this stream, so don't query buckets related to it. continue; } - if (opened != null) { - params = params.withAddedStreamParameters(opened); + let hasExplicitDefaultSubscription = false; + for (const subscription of subscriptions) { + let subscriptionParams = params; + if (subscription.parameters != null) { + subscriptionParams = params.withAddedStreamParameters(subscription.parameters); + } else { + hasExplicitDefaultSubscription = true; + } + + queriers.push(descriptor.getBucketParameterQuerier(subscriptionParams)); } + // If the stream is subscribed to by default and there is no explicit subscription that would match the default + // subscription, also include the default querier. + if (descriptor.subscribedToByDefault && !hasExplicitDefaultSubscription) { + queriers.push(descriptor.getBucketParameterQuerier(params)); + } + } else { queriers.push(descriptor.getBucketParameterQuerier(params)); } - - queriers.push(descriptor.getBucketParameterQuerier(params)); } return mergeBucketParameterQueriers(queriers); From bc77408b0811d1e407bf5edbbcd1757ea866b681 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 21 Jul 2025 15:48:00 +0200 Subject: [PATCH 07/21] Deduplicate buckets --- .../src/sync/BucketChecksumState.ts | 45 ++++++++++++++--- .../sync-rules/src/BucketParameterQuerier.ts | 8 +-- .../sync-rules/src/SqlBucketDescriptor.ts | 49 +++++++++++++++---- packages/sync-rules/src/SqlParameterQuery.ts | 17 +++++-- packages/sync-rules/src/SqlSyncRules.ts | 6 +-- 5 files changed, 99 insertions(+), 26 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index a7af600b6..8a0ce976f 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -112,7 +112,7 @@ export class BucketChecksumState { /** Set of all buckets in this checkpoint. */ const bucketDescriptionMap = new Map( - allBuckets.map((b) => [b.bucket, this.parameterState.overrideBucketDescription(b)]) + allBuckets.map((b) => [b.bucket, this.parameterState.translateResolvedBucket(b)]) ); if (bucketDescriptionMap.size > this.context.maxBuckets) { @@ -345,7 +345,7 @@ export interface CheckpointUpdate { /** * All buckets forming part of the checkpoint. */ - buckets: BucketDescription[]; + buckets: ResolvedBucket[]; /** * If present, a set of buckets that have been updated since the last checkpoint. @@ -367,7 +367,7 @@ export class BucketParameterState { private readonly explicitStreamSubscriptions: Record; private readonly subscribedStreamNames: Set; private readonly logger: Logger; - private cachedDynamicBuckets: BucketDescription[] | null = null; + private cachedDynamicBuckets: ResolvedBucket[] | null = null; private cachedDynamicBucketSet: Set | null = null; private readonly lookups: Set; @@ -412,7 +412,10 @@ export class BucketParameterState { hasDefaultStreams: this.includeDefaultStreams, streams: streamsByName }); - this.staticBuckets = new Map(this.querier.staticBuckets.map((b) => [b.bucket, b])); + + this.staticBuckets = new Map( + mergeBuckets(this.querier.staticBuckets).map((b) => [b.bucket, b]) + ); this.lookups = new Set(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values))); this.subscribedStreamNames = new Set(Object.keys(streamsByName)); } @@ -422,7 +425,8 @@ export class BucketParameterState { * {@link util.ClientBucketDescription}. */ translateResolvedBucket(description: ResolvedBucket): util.ClientBucketDescription { - // Assign + // If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that + // priority. let priorityOverride: BucketPriority | null = null; for (const reason of description.inclusion_reasons) { if (reason != 'default') { @@ -531,7 +535,7 @@ export class BucketParameterState { } } - let dynamicBuckets: BucketDescription[]; + let dynamicBuckets: ResolvedBucket[]; if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) { dynamicBuckets = await querier.queryDynamicBucketDescriptions({ getParameterSets(lookups) { @@ -612,3 +616,32 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) const limited = buckets.slice(0, limit); return `${JSON.stringify(limited)}...`; } + +/** + * Resolves duplicate buckets in the given array, merging the inclusion reasons for duplicate. + * + * It's possible for duplicates to occur when a stream has multiple subscriptions, consider e.g. + * + * ``` + * sync_streams: + * assets_by_category: + * query: select * from assets where category in (request.parameters() -> 'categories') + * ``` + * + * Here, a client might subscribe once with `{"categories": [1]}` and once with `{"categories": [1, 2]}`. Since each + * subscription is evaluated independently, this would lead to three buckets, with a duplicate `assets_by_category[1]` + * bucket. + */ +function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] { + const byDefinition: Record = {}; + + for (const bucket of buckets) { + if (Object.hasOwn(byDefinition, bucket.definition)) { + byDefinition[bucket.definition].inclusion_reasons.push(...bucket.inclusion_reasons); + } else { + byDefinition[bucket.definition] = bucket; + } + } + + return Object.values(byDefinition); +} diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index 8a21ac4be..08a7fe115 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -1,4 +1,4 @@ -import { BucketDescription } from './BucketDescription.js'; +import { BucketDescription, ResolvedBucket } from './BucketDescription.js'; import { RequestParameters, SqliteJsonRow, SqliteJsonValue } from './types.js'; import { normalizeParameterValue } from './utils.js'; @@ -14,7 +14,7 @@ export interface BucketParameterQuerier { * select request.user_id() as user_id() * select value as project_id from json_each(request.jwt() -> 'project_ids') */ - readonly staticBuckets: BucketDescription[]; + readonly staticBuckets: ResolvedBucket[]; /** * True if there are dynamic buckets, meaning queryDynamicBucketDescriptions() should be used. @@ -36,7 +36,7 @@ export interface BucketParameterQuerier { * * select id as user_id from users where users.id = request.user_id() */ - queryDynamicBucketDescriptions(source: ParameterLookupSource): Promise; + queryDynamicBucketDescriptions(source: ParameterLookupSource): Promise; } export interface ParameterLookupSource { @@ -54,7 +54,7 @@ export function mergeBucketParameterQueriers(queriers: BucketParameterQuerier[]) hasDynamicBuckets: parameterQueryLookups.length > 0, parameterQueryLookups: parameterQueryLookups, async queryDynamicBucketDescriptions(source: ParameterLookupSource) { - let results: BucketDescription[] = []; + let results: ResolvedBucket[] = []; for (let q of queriers) { if (q.hasDynamicBuckets) { results.push(...(await q.queryDynamicBucketDescriptions(source))); diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 7116984de..c67c669b3 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,10 +1,10 @@ -import { BucketDescription } from './BucketDescription.js'; +import { BucketDescription, BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; import { BucketParameterQuerier, mergeBucketParameterQueriers } from './BucketParameterQuerier.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; import { SqlParameterQuery } from './SqlParameterQuery.js'; -import { SyncRulesOptions } from './SqlSyncRules.js'; +import { GetQuerierOptions, SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; import { StreamQuery } from './StreamQuery.js'; import { TablePattern } from './TablePattern.js'; @@ -134,27 +134,48 @@ export class SqlBucketDescriptor { return results; } - getBucketParameterQuerier(parameters: RequestParameters): BucketParameterQuerier { - const staticBuckets = this.getStaticBucketDescriptions(parameters); + /** + * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. + */ + getBucketParameterQuerier(options: GetQuerierOptions, parameters: RequestParameters): BucketParameterQuerier { + const queriers: BucketParameterQuerier[] = []; + this.pushBucketParameterQueriers(queriers, options, parameters); + + return mergeBucketParameterQueriers(queriers); + } + + pushBucketParameterQueriers( + result: BucketParameterQuerier[], + options: GetQuerierOptions, + parameters: RequestParameters + ) { + const reasons = [this.bucketInclusionReason(options)]; + const staticBuckets = this.getStaticBucketDescriptions(parameters, reasons); const staticQuerier = { staticBuckets, hasDynamicBuckets: false, parameterQueryLookups: [], queryDynamicBucketDescriptions: async () => [] } satisfies BucketParameterQuerier; + result.push(staticQuerier); if (this.parameterQueries.length == 0) { - return staticQuerier; + return; } - const dynamicQueriers = this.parameterQueries.map((query) => query.getBucketParameterQuerier(parameters)); - return mergeBucketParameterQueriers([staticQuerier, ...dynamicQueriers]); + const dynamicQueriers = this.parameterQueries.map((query) => query.getBucketParameterQuerier(parameters, reasons)); + result.push(...dynamicQueriers); } - getStaticBucketDescriptions(parameters: RequestParameters): BucketDescription[] { - let results: BucketDescription[] = []; + getStaticBucketDescriptions(parameters: RequestParameters, reasons: BucketInclusionReason[]): ResolvedBucket[] { + let results: ResolvedBucket[] = []; for (let query of this.globalParameterQueries) { - results.push(...query.getStaticBucketDescriptions(parameters)); + for (const desc of query.getStaticBucketDescriptions(parameters)) { + results.push({ + ...desc, + inclusion_reasons: reasons + }); + } } return results; } @@ -177,6 +198,14 @@ export class SqlBucketDescriptor { return result; } + private bucketInclusionReason(parameters: GetQuerierOptions): BucketInclusionReason { + if (this.type == SqlBucketDescriptorType.STREAM && !this.subscribedToByDefault) { + return { subscription: this.name }; + } else { + return 'default'; + } + } + tableSyncsData(table: SourceTableInterface): boolean { for (let query of this.dataQueries) { if (query.applies(table)) { diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index be509bf7e..a49421762 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -1,5 +1,10 @@ import { parse, SelectedColumn } from 'pgsql-ast-parser'; -import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; +import { + BucketDescription, + BucketInclusionReason, + BucketPriority, + DEFAULT_BUCKET_PRIORITY +} from './BucketDescription.js'; import { BucketParameterQuerier, ParameterLookup, ParameterLookupSource } from './BucketParameterQuerier.js'; import { SqlRuleError } from './errors.js'; import { SourceTableInterface } from './SourceTableInterface.js'; @@ -451,7 +456,10 @@ export class SqlParameterQuery { } } - getBucketParameterQuerier(requestParameters: RequestParameters): BucketParameterQuerier { + getBucketParameterQuerier( + requestParameters: RequestParameters, + reasons: BucketInclusionReason[] + ): BucketParameterQuerier { const lookups = this.getLookups(requestParameters); if (lookups.length == 0) { // This typically happens when the query is pre-filtered using a where clause @@ -470,7 +478,10 @@ export class SqlParameterQuery { parameterQueryLookups: lookups, queryDynamicBucketDescriptions: async (source: ParameterLookupSource) => { const bucketParameters = await source.getParameterSets(lookups); - return this.resolveBucketDescriptions(bucketParameters, requestParameters); + return this.resolveBucketDescriptions(bucketParameters, requestParameters).map((bucket) => ({ + ...bucket, + inclusion_reasons: reasons + })); } }; } diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index 1498df495..e6769de4b 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -411,16 +411,16 @@ export class SqlSyncRules implements SyncRules { hasExplicitDefaultSubscription = true; } - queriers.push(descriptor.getBucketParameterQuerier(subscriptionParams)); + descriptor.pushBucketParameterQueriers(queriers, options, subscriptionParams); } // If the stream is subscribed to by default and there is no explicit subscription that would match the default // subscription, also include the default querier. if (descriptor.subscribedToByDefault && !hasExplicitDefaultSubscription) { - queriers.push(descriptor.getBucketParameterQuerier(params)); + descriptor.pushBucketParameterQueriers(queriers, options, params); } } else { - queriers.push(descriptor.getBucketParameterQuerier(params)); + descriptor.pushBucketParameterQueriers(queriers, options, params); } } From 5823e24f3e79cf8cc834c80c904d7697aa04a074 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 30 Jul 2025 17:22:06 +0200 Subject: [PATCH 08/21] Introduce BucketSource interface --- .../src/routes/endpoints/sync-rules.ts | 29 +-- .../src/sync/BucketChecksumState.ts | 15 +- packages/sync-rules/src/BucketSource.ts | 81 ++++++++ .../sync-rules/src/SqlBucketDescriptor.ts | 128 +++++++----- packages/sync-rules/src/SqlSyncRules.ts | 118 ++--------- packages/sync-rules/src/StreamQuery.ts | 191 ------------------ packages/sync-rules/src/index.ts | 1 + .../src/schema-generators/SchemaGenerator.ts | 14 +- packages/sync-rules/src/types.ts | 4 - 9 files changed, 189 insertions(+), 392 deletions(-) create mode 100644 packages/sync-rules/src/BucketSource.ts delete mode 100644 packages/sync-rules/src/StreamQuery.ts diff --git a/packages/service-core/src/routes/endpoints/sync-rules.ts b/packages/service-core/src/routes/endpoints/sync-rules.ts index 69d167c03..ebac1c23e 100644 --- a/packages/service-core/src/routes/endpoints/sync-rules.ts +++ b/packages/service-core/src/routes/endpoints/sync-rules.ts @@ -202,34 +202,7 @@ async function debugSyncRules(apiHandler: RouteAPI, sync_rules: string) { return { valid: true, - bucket_definitions: rules.bucketDescriptors.map((d) => { - let all_parameter_queries = [...d.parameterQueries.values()].flat(); - let all_data_queries = [...d.dataQueries.values()].flat(); - return { - name: d.name, - bucket_parameters: d.bucketParameters, - global_parameter_queries: d.globalParameterQueries.map((q) => { - return { - sql: q.sql - }; - }), - parameter_queries: all_parameter_queries.map((q) => { - return { - sql: q.sql, - table: q.sourceTable, - input_parameters: q.inputParameters - }; - }), - - data_queries: all_data_queries.map((q) => { - return { - sql: q.sql, - table: q.sourceTable, - columns: q.columnOutputNames() - }; - }) - }; - }), + bucket_definitions: rules.bucketSources.map((source) => source.debugRepresentation()), source_tables: resolved_tables, data_tables: rules.debugGetOutputTables() }; diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 8a0ce976f..99cc2d1a6 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -1,7 +1,8 @@ import { BucketDescription, BucketPriority, - isValidPriority, + BucketSource, + BucketSourceType, RequestedStream, RequestParameters, ResolvedBucket, @@ -22,7 +23,6 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js'; import { SyncContext } from './SyncContext.js'; import { getIntersection, hasIntersection } from './util.js'; -import { SqlBucketDescriptor, SqlBucketDescriptorType } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js'; export interface BucketChecksumStateOptions { syncContext: SyncContext; @@ -234,14 +234,13 @@ export class BucketChecksumState { this.logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length }); }; bucketsToFetch = allBuckets; - this.parameterState.syncRules.bucketDescriptors; const subscriptions: util.StreamDescription[] = []; - for (const desc of this.parameterState.syncRules.bucketDescriptors) { - if (desc.type == SqlBucketDescriptorType.STREAM && this.parameterState.isSubscribedToStream(desc)) { + for (const source of this.parameterState.syncRules.bucketSources) { + if (source.type == BucketSourceType.SYNC_STREAM && this.parameterState.isSubscribedToStream(source)) { subscriptions.push({ - name: desc.name, - is_default: desc.subscribedToByDefault + name: source.name, + is_default: source.subscribedToByDefault }); } } @@ -455,7 +454,7 @@ export class BucketParameterState { }; } - isSubscribedToStream(desc: SqlBucketDescriptor): boolean { + isSubscribedToStream(desc: BucketSource): boolean { return (desc.subscribedToByDefault && this.includeDefaultStreams) || this.subscribedStreamNames.has(desc.name); } diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts new file mode 100644 index 000000000..5e2c8a8fe --- /dev/null +++ b/packages/sync-rules/src/BucketSource.ts @@ -0,0 +1,81 @@ +import { BucketParameterQuerier, ParameterLookup } from './BucketParameterQuerier.js'; +import { ColumnDefinition } from './ExpressionType.js'; +import { SourceTableInterface } from './SourceTableInterface.js'; +import { GetQuerierOptions } from './SqlSyncRules.js'; +import { TablePattern } from './TablePattern.js'; +import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, SourceSchema, SqliteRow } from './types.js'; + +/** + * An interface declaring + * + * - which buckets the sync service should create when processing change streams from the database. + * - how data in source tables maps to data in buckets (e.g. when we're not selecting all columns). + * - which buckets a given connection has access to. + * + * There are two ways to define bucket sources: Via sync rules made up of parameter and data queries, and via stream + * definitions that only consist of a single query. + */ +export interface BucketSource { + name: string; + type: BucketSourceType; + + subscribedToByDefault: boolean; + + /** + * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should + * be associated with. + * + * The returned {@link ParameterLookup} can be referenced by {@link pushBucketParameterQueriers} to allow the storage + * system to find buckets. + */ + evaluateParameterRow(sourceTable: SourceTableInterface, row: SqliteRow): EvaluatedParametersResult[]; + + /** + * Given a row as it appears in a table that affects sync data, return buckets, logical table names and transformed + * data for rows to add to buckets. + */ + evaluateRow(options: EvaluateRowOptions): EvaluationResult[]; + + /** + * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. + * + * @param result The target array to insert queriers into. + * @param options Options, including parameters that may affect the buckets loaded by this source. + */ + pushBucketParameterQueriers(result: BucketParameterQuerier[], options: GetQuerierOptions): void; + + /** + * Whether {@link pushBucketParameterQueriers} may include a querier where + * {@link BucketParameterQuerier.hasDynamicBuckets} is true. + * + * This is mostly used for testing. + */ + hasDynamicBucketQueries(): boolean; + + getSourceTables(): Set; + + /** Whether the table possibly affects the buckets resolved by this source. */ + tableSyncsParameters(table: SourceTableInterface): boolean; + + /** Whether the table possibly affects the contents of buckets resolved by this source. */ + tableSyncsData(table: SourceTableInterface): boolean; + + /** + * Given a static schema, infer all logical tables and associated columns that appear in buckets defined by this + * source. + * + * This is use to generate the client-side schema. + */ + resolveResultSets(schema: SourceSchema, tables: Record>): void; + + debugWriteOutputTables(result: Record): void; + + debugRepresentation(): any; +} + +export enum BucketSourceType { + SYNC_RULE, + SYNC_STREAM +} + +export type ResultSetDescription = { name: string; columns: ColumnDefinition[] }; diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index c67c669b3..f5448da0f 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,12 +1,13 @@ -import { BucketDescription, BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; +import { BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; import { BucketParameterQuerier, mergeBucketParameterQueriers } from './BucketParameterQuerier.js'; +import { BucketSource, BucketSourceType, ResultSetDescription } from './BucketSource.js'; +import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { SqlDataQuery } from './SqlDataQuery.js'; import { SqlParameterQuery } from './SqlParameterQuery.js'; import { GetQuerierOptions, SyncRulesOptions } from './SqlSyncRules.js'; import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; -import { StreamQuery } from './StreamQuery.js'; import { TablePattern } from './TablePattern.js'; import { TableValuedFunctionSqlParameterQuery } from './TableValuedFunctionSqlParameterQuery.js'; import { SqlRuleError } from './errors.js'; @@ -16,8 +17,8 @@ import { EvaluationResult, QueryParseOptions, RequestParameters, - SqliteRow, - StreamParseOptions + SourceSchema, + SqliteRow } from './types.js'; export interface QueryParseResult { @@ -29,23 +30,20 @@ export interface QueryParseResult { errors: SqlRuleError[]; } -export enum SqlBucketDescriptorType { - SYNC_RULE, - STREAM -} - -export class SqlBucketDescriptor { +export class SqlBucketDescriptor implements BucketSource { name: string; bucketParameters?: string[]; - type: SqlBucketDescriptorType; - subscribedToByDefault: boolean; - constructor(name: string, type: SqlBucketDescriptorType) { + constructor(name: string) { this.name = name; - this.type = type; + } - // Sync-rule style buckets are subscribed to by default, streams are opt-in unless their definition says otherwise. - this.subscribedToByDefault = type == SqlBucketDescriptorType.SYNC_RULE; + get type(): BucketSourceType { + return BucketSourceType.SYNC_RULE; + } + + public get subscribedToByDefault(): boolean { + return true; } /** @@ -94,24 +92,6 @@ export class SqlBucketDescriptor { }; } - addUnifiedStreamQuery(sql: string, options: StreamParseOptions): QueryParseResult { - const [query, errors] = StreamQuery.fromSql(this.name, sql, options); - for (const parameterQuery of query.inferredParameters) { - if (parameterQuery instanceof StaticSqlParameterQuery) { - this.globalParameterQueries.push(parameterQuery); - } else { - this.parameterQueries.push(parameterQuery); - } - } - this.dataQueries.push(query.data); - this.subscribedToByDefault = options.default ?? false; - - return { - parsed: true, - errors - }; - } - evaluateRow(options: EvaluateRowOptions): EvaluationResult[] { let results: EvaluationResult[] = []; for (let query of this.dataQueries) { @@ -137,20 +117,16 @@ export class SqlBucketDescriptor { /** * @deprecated Use `pushBucketParameterQueriers` instead and merge at the top-level. */ - getBucketParameterQuerier(options: GetQuerierOptions, parameters: RequestParameters): BucketParameterQuerier { + getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers(queriers, options, parameters); + this.pushBucketParameterQueriers(queriers, options); return mergeBucketParameterQueriers(queriers); } - pushBucketParameterQueriers( - result: BucketParameterQuerier[], - options: GetQuerierOptions, - parameters: RequestParameters - ) { - const reasons = [this.bucketInclusionReason(options)]; - const staticBuckets = this.getStaticBucketDescriptions(parameters, reasons); + pushBucketParameterQueriers(result: BucketParameterQuerier[], options: GetQuerierOptions) { + const reasons = [this.bucketInclusionReason()]; + const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, reasons); const staticQuerier = { staticBuckets, hasDynamicBuckets: false, @@ -163,7 +139,9 @@ export class SqlBucketDescriptor { return; } - const dynamicQueriers = this.parameterQueries.map((query) => query.getBucketParameterQuerier(parameters, reasons)); + const dynamicQueriers = this.parameterQueries.map((query) => + query.getBucketParameterQuerier(options.globalParameters, reasons) + ); result.push(...dynamicQueriers); } @@ -198,12 +176,8 @@ export class SqlBucketDescriptor { return result; } - private bucketInclusionReason(parameters: GetQuerierOptions): BucketInclusionReason { - if (this.type == SqlBucketDescriptorType.STREAM && !this.subscribedToByDefault) { - return { subscription: this.name }; - } else { - return 'default'; - } + private bucketInclusionReason(): BucketInclusionReason { + return 'default'; } tableSyncsData(table: SourceTableInterface): boolean { @@ -223,4 +197,58 @@ export class SqlBucketDescriptor { } return false; } + + resolveResultSets(schema: SourceSchema, tables: Record>) { + for (let query of this.dataQueries) { + const outTables = query.getColumnOutputs(schema); + for (let table of outTables) { + tables[table.name] ??= {}; + for (let column of table.columns) { + if (column.name != 'id') { + tables[table.name][column.name] ??= column; + } + } + } + } + } + + debugWriteOutputTables(result: Record): void { + for (let q of this.dataQueries) { + result[q.table!] ??= []; + const r = { + query: q.sql + }; + + result[q.table!].push(r); + } + } + + debugRepresentation() { + let all_parameter_queries = [...this.parameterQueries.values()].flat(); + let all_data_queries = [...this.dataQueries.values()].flat(); + return { + name: this.name, + type: this.type.toString(), + bucket_parameters: this.bucketParameters, + global_parameter_queries: this.globalParameterQueries.map((q) => { + return { + sql: q.sql + }; + }), + parameter_queries: all_parameter_queries.map((q) => { + return { + sql: q.sql, + table: q.sourceTable, + input_parameters: q.inputParameters + }; + }), + data_queries: all_data_queries.map((q) => { + return { + sql: q.sql, + table: q.sourceTable, + columns: q.columnOutputNames() + }; + }) + }; + } } diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index e6769de4b..b3b5a046b 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -5,7 +5,7 @@ import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; import { validateSyncRulesSchema } from './json_schema.js'; import { SourceTableInterface } from './SourceTableInterface.js'; -import { QueryParseResult, SqlBucketDescriptor, SqlBucketDescriptorType } from './SqlBucketDescriptor.js'; +import { QueryParseResult, SqlBucketDescriptor } from './SqlBucketDescriptor.js'; import { TablePattern } from './TablePattern.js'; import { EvaluatedParameters, @@ -22,9 +22,9 @@ import { SourceSchema, SqliteJsonRow, SqliteRow, - StreamParseOptions, SyncRules } from './types.js'; +import { BucketSource } from './BucketSource.js'; const ACCEPT_POTENTIALLY_DANGEROUS_QUERIES = Symbol('ACCEPT_POTENTIALLY_DANGEROUS_QUERIES'); @@ -75,7 +75,7 @@ export interface GetQuerierOptions { } export class SqlSyncRules implements SyncRules { - bucketDescriptors: SqlBucketDescriptor[] = []; + bucketSources: BucketSource[] = []; eventDescriptors: SqlEventDescriptor[] = []; content: string; @@ -132,8 +132,6 @@ export class SqlSyncRules implements SyncRules { // Bucket definitions using explicit parameter and data queries. const bucketMap = parsed.get('bucket_definitions') as YAMLMap; - // Streams (which also map to buckets internally) with a new syntax and options. - const streamMap = parsed.get('streams') as YAMLMap; const definitionNames = new Set(); const checkUniqueName = (name: string, literal: Scalar) => { if (definitionNames.has(name)) { @@ -145,8 +143,8 @@ export class SqlSyncRules implements SyncRules { return true; }; - if (bucketMap == null && streamMap == null) { - rules.errors.push(new YamlError(new Error(`Either 'bucket_definitions' or 'streams' are required`))); + if (bucketMap == null) { + rules.errors.push(new YamlError(new Error(`'bucket_definitions' is required`))); if (throwOnError) { rules.throwOnError(); @@ -178,7 +176,7 @@ export class SqlSyncRules implements SyncRules { const parameters = value.get('parameters', true) as unknown; const dataQueries = value.get('data', true) as unknown; - const descriptor = new SqlBucketDescriptor(key, SqlBucketDescriptorType.SYNC_RULE); + const descriptor = new SqlBucketDescriptor(key); if (parameters instanceof Scalar) { rules.withScalar(parameters, (q) => { @@ -203,39 +201,7 @@ export class SqlSyncRules implements SyncRules { return descriptor.addDataQuery(q, queryOptions); }); } - rules.bucketDescriptors.push(descriptor); - } - - for (const entry of streamMap?.items ?? []) { - const { key: keyScalar, value } = entry as { key: Scalar; value: YAMLMap }; - const key = keyScalar.toString(); - if (!checkUniqueName(key, keyScalar)) { - continue; - } - - const descriptor = new SqlBucketDescriptor(key, SqlBucketDescriptorType.STREAM); - - const accept_potentially_dangerous_queries = - value.get('accept_potentially_dangerous_queries', true)?.value == true; - - const queryOptions: StreamParseOptions = { - ...options, - accept_potentially_dangerous_queries, - priority: rules.parsePriority(value), - default: value.get('default', true)?.value == true - }; - - const data = value.get('query', true) as unknown; - if (data instanceof Scalar) { - rules.withScalar(data, (q) => { - return descriptor.addUnifiedStreamQuery(q, queryOptions); - }); - } else { - rules.errors.push(this.tokenError(data, 'Must be a string.')); - continue; - } - - rules.bucketDescriptors.push(descriptor); + rules.bucketSources.push(descriptor); } const eventMap = parsed.get('event_definitions') as YAMLMap; @@ -354,8 +320,8 @@ export class SqlSyncRules implements SyncRules { evaluateRowWithErrors(options: EvaluateRowOptions): { results: EvaluatedRow[]; errors: EvaluationError[] } { let rawResults: EvaluationResult[] = []; - for (let query of this.bucketDescriptors) { - rawResults.push(...query.evaluateRow(options)); + for (let source of this.bucketSources) { + rawResults.push(...source.evaluateRow(options)); } const results = rawResults.filter(isEvaluatedRow) as EvaluatedRow[]; @@ -380,8 +346,8 @@ export class SqlSyncRules implements SyncRules { row: SqliteRow ): { results: EvaluatedParameters[]; errors: EvaluationError[] } { let rawResults: EvaluatedParametersResult[] = []; - for (let query of this.bucketDescriptors) { - rawResults.push(...query.evaluateParameterRow(table, row)); + for (let source of this.bucketSources) { + rawResults.push(...source.evaluateParameterRow(table, row)); } const results = rawResults.filter(isEvaluatedParameters) as EvaluatedParameters[]; @@ -391,49 +357,20 @@ export class SqlSyncRules implements SyncRules { getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { const queriers: BucketParameterQuerier[] = []; - for (const descriptor of this.bucketDescriptors) { - let params = options.globalParameters; - - if (descriptor.type == SqlBucketDescriptorType.STREAM) { - const subscriptions = options.streams[descriptor.name] ?? []; - - if (!descriptor.subscribedToByDefault && subscriptions.length) { - // The client is not subscribing to this stream, so don't query buckets related to it. - continue; - } - - let hasExplicitDefaultSubscription = false; - for (const subscription of subscriptions) { - let subscriptionParams = params; - if (subscription.parameters != null) { - subscriptionParams = params.withAddedStreamParameters(subscription.parameters); - } else { - hasExplicitDefaultSubscription = true; - } - - descriptor.pushBucketParameterQueriers(queriers, options, subscriptionParams); - } - - // If the stream is subscribed to by default and there is no explicit subscription that would match the default - // subscription, also include the default querier. - if (descriptor.subscribedToByDefault && !hasExplicitDefaultSubscription) { - descriptor.pushBucketParameterQueriers(queriers, options, params); - } - } else { - descriptor.pushBucketParameterQueriers(queriers, options, params); - } + for (const source of this.bucketSources) { + source.pushBucketParameterQueriers(queriers, options); } return mergeBucketParameterQueriers(queriers); } hasDynamicBucketQueries() { - return this.bucketDescriptors.some((query) => query.hasDynamicBucketQueries()); + return this.bucketSources.some((s) => s.hasDynamicBucketQueries()); } getSourceTables(): TablePattern[] { const sourceTables = new Map(); - for (const bucket of this.bucketDescriptors) { + for (const bucket of this.bucketSources) { for (const r of bucket.getSourceTables()) { const key = `${r.connectionTag}.${r.schema}.${r.tablePattern}`; sourceTables.set(key, r); @@ -467,34 +404,17 @@ export class SqlSyncRules implements SyncRules { } tableSyncsData(table: SourceTableInterface): boolean { - for (const bucket of this.bucketDescriptors) { - if (bucket.tableSyncsData(table)) { - return true; - } - } - return false; + return this.bucketSources.some((b) => b.tableSyncsData(table)); } tableSyncsParameters(table: SourceTableInterface): boolean { - for (let bucket of this.bucketDescriptors) { - if (bucket.tableSyncsParameters(table)) { - return true; - } - } - return false; + return this.bucketSources.some((b) => b.tableSyncsParameters(table)); } debugGetOutputTables() { let result: Record = {}; - for (let bucket of this.bucketDescriptors) { - for (let q of bucket.dataQueries) { - result[q.table!] ??= []; - const r = { - query: q.sql - }; - - result[q.table!].push(r); - } + for (let bucket of this.bucketSources) { + bucket.debugWriteOutputTables(result); } return result; } diff --git a/packages/sync-rules/src/StreamQuery.ts b/packages/sync-rules/src/StreamQuery.ts deleted file mode 100644 index ca7caac2c..000000000 --- a/packages/sync-rules/src/StreamQuery.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { parse } from 'pgsql-ast-parser'; -import { ParameterValueClause, QuerySchema, StreamParseOptions } from './types.js'; -import { SqlRuleError } from './errors.js'; -import { isSelectStatement } from './utils.js'; -import { checkUnsupportedFeatures, isClauseError } from './sql_support.js'; -import { SqlDataQuery, SqlDataQueryOptions } from './SqlDataQuery.js'; -import { RowValueExtractor } from './BaseSqlDataQuery.js'; -import { TablePattern } from './TablePattern.js'; -import { TableQuerySchema } from './TableQuerySchema.js'; -import { SqlTools } from './sql_filters.js'; -import { ExpressionType } from './ExpressionType.js'; -import { SqlParameterQuery } from './SqlParameterQuery.js'; -import { StaticSqlParameterQuery } from './StaticSqlParameterQuery.js'; -import { DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js'; - -/** - * Represents a query backing a stream definition. - * - * Streams are a new way to define sync rules that don't require separate data and - * parameter queries. However, since most of the sync service is built around that - * distiction at the moment, stream queries are implemented by desugaring a unified - * query into its individual components. - */ -export class StreamQuery { - inferredParameters: (SqlParameterQuery | StaticSqlParameterQuery)[]; - data: SqlDataQuery; - - static fromSql(descriptorName: string, sql: string, options: StreamParseOptions): [StreamQuery, SqlRuleError[]] { - const [query, ...illegalRest] = parse(sql, { locationTracking: true }); - const schema = options.schema; - const parameters: (SqlParameterQuery | StaticSqlParameterQuery)[] = []; - const errors: SqlRuleError[] = []; - - // TODO: Share more of this code with SqlDataQuery - if (illegalRest.length > 0) { - throw new SqlRuleError('Only a single SELECT statement is supported', sql, illegalRest[0]?._location); - } - - if (!isSelectStatement(query)) { - throw new SqlRuleError('Only SELECT statements are supported', sql, query._location); - } - - if (query.from == null || query.from.length != 1 || query.from[0].type != 'table') { - throw new SqlRuleError('Must SELECT from a single table', sql, query.from?.[0]._location); - } - - errors.push(...checkUnsupportedFeatures(sql, query)); - - const tableRef = query.from?.[0].name; - if (tableRef?.name == null) { - throw new SqlRuleError('Must SELECT from a single table', sql, query.from?.[0]._location); - } - const alias: string = tableRef.alias ?? tableRef.name; - - const sourceTable = new TablePattern(tableRef.schema ?? options.defaultSchema, tableRef.name); - let querySchema: QuerySchema | undefined = undefined; - if (schema) { - const tables = schema.getTables(sourceTable); - if (tables.length == 0) { - const e = new SqlRuleError( - `Table ${sourceTable.schema}.${sourceTable.tablePattern} not found`, - sql, - query.from?.[0]?._location - ); - e.type = 'warning'; - - errors.push(e); - } else { - querySchema = new TableQuerySchema(tables, alias); - } - } - - const where = query.where; - const tools = new SqlTools({ - table: alias, - parameterTables: [], - valueTables: [alias], - sql, - schema: querySchema, - supportsStreamInputs: true, - supportsParameterExpressions: true - }); - tools.checkSpecificNameCase(tableRef); - const filter = tools.compileWhereClause(where); - const inputParameterNames = filter.inputParameters.map((p) => `bucket.${p.key}`); - - // Build parameter queries based on inferred bucket parameters - if (tools.inferredParameters.length) { - const extractors: Record = {}; - for (const inferred of tools.inferredParameters) { - extractors[inferred.name] = inferred.clause; - } - - parameters.push( - new StaticSqlParameterQuery({ - sql, - queryId: 'static', - descriptorName, - parameterExtractors: extractors, - bucketParameters: tools.inferredParameters.map((p) => p.name), - filter: undefined, // TODO - priority: DEFAULT_BUCKET_PRIORITY // Ignored here - }) - ); - } - - let hasId = false; - let hasWildcard = false; - let extractors: RowValueExtractor[] = []; - - for (let column of query.columns ?? []) { - const name = tools.getOutputName(column); - if (name != '*') { - const clause = tools.compileRowValueExtractor(column.expr); - if (isClauseError(clause)) { - // Error logged already - continue; - } - extractors.push({ - extract: (tables, output) => { - output[name] = clause.evaluate(tables); - }, - getTypes(schema, into) { - const def = clause.getColumnDefinition(schema); - - into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType }; - } - }); - } else { - extractors.push({ - extract: (tables, output) => { - const row = tables[alias]; - for (let key in row) { - if (key.startsWith('_')) { - continue; - } - output[key] ??= row[key]; - } - }, - getTypes(schema, into) { - for (let column of schema.getColumns(alias)) { - into[column.name] ??= column; - } - } - }); - } - if (name == 'id') { - hasId = true; - } else if (name == '*') { - hasWildcard = true; - if (querySchema == null) { - // Not performing schema-based validation - assume there is an id - hasId = true; - } else { - const idType = querySchema.getColumn(alias, 'id')?.type ?? ExpressionType.NONE; - if (!idType.isNone()) { - hasId = true; - } - } - } - } - if (!hasId) { - const error = new SqlRuleError(`Query must return an "id" column`, sql, query.columns?.[0]._location); - if (hasWildcard) { - // Schema-based validations are always warnings - error.type = 'warning'; - } - errors.push(error); - } - - errors.push(...tools.errors); - - const data: SqlDataQueryOptions = { - sourceTable, - table: alias, - sql, - filter, - columns: query.columns ?? [], - descriptorName, - bucketParameters: inputParameterNames, - tools, - extractors - }; - return [new StreamQuery(parameters, data), errors]; - } - - private constructor(parameters: (SqlParameterQuery | StaticSqlParameterQuery)[], data: SqlDataQueryOptions) { - this.inferredParameters = parameters; - this.data = new SqlDataQuery(data); - } -} diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index 79e3220d9..cac81ace5 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -1,5 +1,6 @@ export * from './BucketDescription.js'; export * from './BucketParameterQuerier.js'; +export * from './BucketSource.js'; export * from './errors.js'; export * from './events/SqlEventDescriptor.js'; export * from './events/SqlEventSourceQuery.js'; diff --git a/packages/sync-rules/src/schema-generators/SchemaGenerator.ts b/packages/sync-rules/src/schema-generators/SchemaGenerator.ts index 4b6541ba9..cde9585b2 100644 --- a/packages/sync-rules/src/schema-generators/SchemaGenerator.ts +++ b/packages/sync-rules/src/schema-generators/SchemaGenerator.ts @@ -10,18 +10,8 @@ export abstract class SchemaGenerator { protected getAllTables(source: SqlSyncRules, schema: SourceSchema) { let tables: Record> = {}; - for (let descriptor of source.bucketDescriptors) { - for (let query of descriptor.dataQueries) { - const outTables = query.getColumnOutputs(schema); - for (let table of outTables) { - tables[table.name] ??= {}; - for (let column of table.columns) { - if (column.name != 'id') { - tables[table.name][column.name] ??= column; - } - } - } - } + for (let descriptor of source.bucketSources) { + descriptor.resolveResultSets(schema, tables); } return Object.entries(tables).map(([name, columns]) => { diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 91acd1afd..91b23a265 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -18,10 +18,6 @@ export interface QueryParseOptions extends SyncRulesOptions { priority?: BucketPriority; } -export interface StreamParseOptions extends QueryParseOptions { - default?: boolean; -} - export interface EvaluatedParameters { lookup: ParameterLookup; From 62f80275a1f296dcd0cb7ea0b2706fe68635a19f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 30 Jul 2025 17:58:10 +0200 Subject: [PATCH 09/21] Fix sync rule tests --- .../test/src/parameter_queries.test.ts | 6 +- .../test/src/static_parameter_queries.test.ts | 32 ++--- packages/sync-rules/test/src/streams.test.ts | 126 ------------------ .../sync-rules/test/src/sync_rules.test.ts | 27 ++-- .../src/table_valued_function_queries.test.ts | 36 ++--- packages/sync-rules/test/src/util.ts | 6 +- 6 files changed, 54 insertions(+), 179 deletions(-) delete mode 100644 packages/sync-rules/test/src/streams.test.ts diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 12835dbec..93125eadd 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -84,11 +84,11 @@ describe('parameter queries', () => { // We _do_ need to care about the bucket string representation. expect( query.resolveBucketDescriptions([{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({})) - ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[314,3.14,314]', definition: 'mybucket', priority: 3 }]); expect( query.resolveBucketDescriptions([{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({})) - ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[314,3.14,314]', definition: 'mybucket', priority: 3 }]); }); test('plain token_parameter (baseline)', () => { @@ -365,7 +365,7 @@ describe('parameter queries', () => { [{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true }) ) - ).toEqual([{ bucket: 'mybucket["user1",1]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket["user1",1]', definition: 'mybucket', priority: 3 }]); }); test('case-sensitive parameter queries (1)', () => { diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts index 611afd7b1..bfc28676f 100644 --- a/packages/sync-rules/test/src/static_parameter_queries.test.ts +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -10,7 +10,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["user1"]', priority: 3 } + { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } ]); }); @@ -20,7 +20,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); }); @@ -29,7 +29,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - { bucket: 'mybucket["user1"]', priority: 3 } + { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } ]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual( [] @@ -41,7 +41,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["USER1"]', priority: 3 } + { bucket: 'mybucket["USER1"]', definition: 'mybucket', priority: 3 } ]); expect(query.bucketParameters!).toEqual(['upper_id']); }); @@ -51,7 +51,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }))).toEqual([]); }); @@ -61,7 +61,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }))).toEqual([]); }); @@ -80,7 +80,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }))).toEqual([ - { bucket: 'mybucket["test"]', priority: 3 } + { bucket: 'mybucket["test"]', definition: 'mybucket', priority: 3 } ]); }); @@ -91,7 +91,7 @@ describe('static parameter queries', () => { expect(query.bucketParameters).toEqual(['user_id']); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["user1"]', priority: 3 } + { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } ]); }); @@ -102,7 +102,7 @@ describe('static parameter queries', () => { expect(query.bucketParameters).toEqual(['user_id']); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["user1"]', priority: 3 } + { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } ]); }); @@ -111,7 +111,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); }); @@ -120,7 +120,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); }); @@ -136,7 +136,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); }); @@ -147,10 +147,10 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {})) - ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {})) - ).toEqual([{ bucket: 'mybucket[0]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[0]', definition: 'mybucket', priority: 3 }]); }); test('IN for permissions in request.jwt() (2)', function () { @@ -160,7 +160,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {})) - ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[]', definition: 'mybucket', priority: 3 }]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {})) ).toEqual([]); @@ -171,7 +171,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superuser' }, {}))).toEqual([ - { bucket: 'mybucket[]', priority: 3 } + { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } ]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superadmin' }, {}))).toEqual([]); }); diff --git a/packages/sync-rules/test/src/streams.test.ts b/packages/sync-rules/test/src/streams.test.ts deleted file mode 100644 index 2cd5f17ea..000000000 --- a/packages/sync-rules/test/src/streams.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { assert, describe, expect, test } from 'vitest'; -import { SqlSyncRules } from '../../src/index.js'; -import { ASSETS, BASIC_SCHEMA, PARSE_OPTIONS } from './util.js'; - -describe('streams', () => { - test('without parameters', () => { - const desc = parseSingleBucketDescription(` -streams: - lists: - query: SELECT * FROM assets -`); - expect(desc.bucketParameters).toBeFalsy(); - }); - - test('static filter', () => { - const desc = parseSingleBucketDescription(` -streams: - lists: - query: SELECT * FROM assets WHERE count > 5 -`); - expect(desc.bucketParameters).toBeFalsy(); - assert.isEmpty(desc.parameterQueries); - - assert.isEmpty( - desc.evaluateRow({ - sourceTable: ASSETS, - record: { - count: 4 - } - }) - ); - - expect( - desc.evaluateRow({ - sourceTable: ASSETS, - record: { - count: 6 - } - }) - ).toHaveLength(1); - }); - - test('stream param', () => { - const desc = parseSingleBucketDescription(` -streams: - lists: - query: SELECT * FROM assets WHERE id = stream.params() ->> 'id'; -`); - - // This should be desuraged to - // params: SELECT request.params() ->> 'id' AS p0 - // data: SELECT * FROM assets WHERE id = bucket.p0 - - expect(desc.globalParameterQueries).toHaveLength(1); - const [parameter] = desc.globalParameterQueries; - expect(parameter.bucketParameters).toEqual(['p0']); - - const [data] = desc.dataQueries; - expect(data.bucketParameters).toEqual(['bucket.p0']); - }); - - test('user filter', () => { - const desc = parseSingleBucketDescription(` -streams: - lists: - query: SELECT * FROM assets WHERE request.jwt() ->> 'isAdmin' -`); - - // This should be desuraged to - // params: SELECT request.params() ->> 'id' AS p0 - // data: SELECT * FROM assets WHERE id = bucket.p0 - - const [data] = desc.dataQueries; - expect(data.bucketParameters).toEqual(['bucket.p0']); - }); -}); - -/** - -SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) - parameter: SELECT id AS p0 FROM asset_groups WHERE owner = request.user_id() - data: SELECT * FROM assets WHERE id = bucket.p0 - -SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) - OR count > 10 - parameter: SELECT id AS p0 FROM asset_groups WHERE owner = request.user_id() - data: SELECT * FROM assets WHERE id = bucket.p0 OR count > 10 - -SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) - OR request.jwt() ->> 'isAdmin' - parameter: SELECT id AS p0, request.jwt() ->> 'isAdmin' AS p1 FROM asset_groups WHERE owner = request.user_id() - parameter: SELECT NULL as p0, request.jwt() ->> 'isAdmin' AS p1 - data: SELECT * FROM assets WHERE id = bucket.p0 OR bucket.p1 - -SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) - AND request.jwt() ->> 'isAdmin' - parameter: SELECT id AS p0, request.jwt() ->> 'isAdmin' AS p1 FROM asset_groups WHERE owner = request.user_id() - AND request.jwt() ->> 'isAdmin' - data: SELECT * FROM assets WHERE id = bucket.p0 - -SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) - OR name IN (SELECT name FROM test2 WHERE owner = request.user_id()) - parameter: SELECT id AS p0, NULL AS p1 FROM asset_groups WHERE owner = request.user_id() - parameter: SELECT NULL AS p0, NULL AS p1 FROM test2 WHERE owner = request.user_id() - data: SELECT * FROM assets WHERE id = bucket.p0 OR name = bucket.p1 - -SELECT * FROM assets WHERE id IN (SELECT id FROM asset_groups WHERE owner = request.user_id()) - AND name IN (SELECT name FROM test2 WHERE owner = request.user_id()) - parameter: SELECT id AS p0, NULL AS p1 FROM asset_groups WHERE owner = request.user_id() - parameter: SELECT NULL AS p0, NULL AS p1 FROM test2 WHERE owner = request.user_id() - data: SELECT * FROM assets WHERE id = bucket.p0 AND name = bucket.p1 - */ - -const options = { schema: BASIC_SCHEMA, ...PARSE_OPTIONS }; - -function parseSyncRules(yaml: string) { - const rules = SqlSyncRules.fromYaml(yaml, options); - assert.isEmpty(rules.errors); - return rules; -} - -function parseSingleBucketDescription(yaml: string) { - const rules = parseSyncRules(yaml); - expect(rules.bucketDescriptors).toHaveLength(1); - return rules.bucketDescriptors[0]; -} diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 26cf7e42f..2456866ad 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -10,11 +10,12 @@ import { normalizeQuerierOptions, normalizeTokenParameters } from './util.js'; +import { SqlBucketDescriptor } from '../../src/SqlBucketDescriptor.js'; describe('sync rules', () => { test('parse empty sync rules', () => { const rules = SqlSyncRules.fromYaml('bucket_definitions: {}', PARSE_OPTIONS); - expect(rules.bucketDescriptors).toEqual([]); + expect(rules.bucketSources).toEqual([]); }); test('parse global sync rules', () => { @@ -27,7 +28,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.name).toEqual('mybucket'); expect(bucket.bucketParameters).toEqual([]); const dataQuery = bucket.dataQueries[0]; @@ -61,7 +62,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.globalParameterQueries[0]; @@ -93,7 +94,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual([]); const param_query = bucket.parameterQueries[0]; expect(param_query.bucketParameters).toEqual([]); @@ -117,14 +118,16 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id', 'device_id']); const param_query = bucket.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) .staticBuckets - ).toEqual([{ bucket: 'mybucket["user1","device1"]', priority: 3 }]); + ).toEqual([ + { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } + ]); const data_query = bucket.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id', 'device_id']); @@ -163,12 +166,12 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id']); const param_query = bucket.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id']); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).staticBuckets).toEqual([ - { bucket: 'mybucket["user1"]', priority: 3 } + { bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); const data_query = bucket.dataQueries[0]; @@ -307,7 +310,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id']); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], @@ -344,7 +347,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id']); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], @@ -512,7 +515,7 @@ bucket_definitions: ]); expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).staticBuckets).toEqual([ - { bucket: 'mybucket[1]', priority: 3 } + { bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); }); @@ -921,7 +924,7 @@ bucket_definitions: `, PARSE_OPTIONS ); - const bucket = rules.bucketDescriptors[0]; + const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id']); expect(rules.hasDynamicBucketQueries()).toBe(true); diff --git a/packages/sync-rules/test/src/table_valued_function_queries.test.ts b/packages/sync-rules/test/src/table_valued_function_queries.test.ts index 596ade82b..5454f81e1 100644 --- a/packages/sync-rules/test/src/table_valued_function_queries.test.ts +++ b/packages/sync-rules/test/src/table_valued_function_queries.test.ts @@ -19,9 +19,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[1]', priority: 3 }, - { bucket: 'mybucket[2]', priority: 3 }, - { bucket: 'mybucket[3]', priority: 3 } + { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } ]); }); @@ -32,9 +32,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[1]', priority: 3 }, - { bucket: 'mybucket[2]', priority: 3 }, - { bucket: 'mybucket[3]', priority: 3 } + { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } ]); }); @@ -88,9 +88,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket["a"]', priority: 3 }, - { bucket: 'mybucket["b"]', priority: 3 }, - { bucket: 'mybucket["c"]', priority: 3 } + { bucket: 'mybucket["a"]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket["b"]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket["c"]', definition: 'mybucket', priority: 3 } ]); }); @@ -109,9 +109,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[1]', priority: 3 }, - { bucket: 'mybucket[2]', priority: 3 }, - { bucket: 'mybucket[3]', priority: 3 } + { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } ]); }); @@ -130,9 +130,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[1]', priority: 3 }, - { bucket: 'mybucket[2]', priority: 3 }, - { bucket: 'mybucket[3]', priority: 3 } + { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } ]); }); @@ -151,8 +151,8 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[2]', priority: 3 }, - { bucket: 'mybucket[3]', priority: 3 } + { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, + { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } ]); }); @@ -184,7 +184,7 @@ describe('table-valued function queries', () => { {} ) ) - ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }]); }); describe('dangerous queries', function () { diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index bb0066851..3c098a9b0 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -69,9 +69,7 @@ export function normalizeQuerierOptions( const globalParameters = normalizeTokenParameters(token_parameters, user_parameters); return { globalParameters, - hasDefaultSubscriptions: true, - resolveSubscription(name) { - return null; - } + hasDefaultStreams: true, + streams: {} }; } From 4926033003137dd7610d03e74d64daf0d6002314 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 31 Jul 2025 11:13:39 +0200 Subject: [PATCH 10/21] Revert more sync-rule changes --- packages/sync-rules/src/json_schema.ts | 33 +------ packages/sync-rules/src/request_functions.ts | 19 +--- packages/sync-rules/src/sql_filters.ts | 98 ++++---------------- 3 files changed, 20 insertions(+), 130 deletions(-) diff --git a/packages/sync-rules/src/json_schema.ts b/packages/sync-rules/src/json_schema.ts index ef75b5409..5d21169d4 100644 --- a/packages/sync-rules/src/json_schema.ts +++ b/packages/sync-rules/src/json_schema.ts @@ -49,37 +49,6 @@ export const syncRulesSchema: ajvModule.Schema = { } } }, - streams: { - type: 'object', - description: 'Stream definitions', - patternProperties: { - '.*': { - type: 'object', - required: ['query'], - examples: [{ query: ['select * from mytable'] }], - properties: { - accept_potentially_dangerous_queries: { - description: 'If true, disables warnings on potentially dangerous queries', - type: 'boolean' - }, - priority: { - description: - 'Default priority for the stream (lower values indicate higher priority). Clients can override the priority when subscribing.', - type: 'integer' - }, - default: { - type: 'boolean', - description: 'Whether the stream should be subscribed to by default.' - }, - query: { - description: 'The SQL query to sync to clients.', - type: 'string' - } - }, - additionalProperties: false - } - } - }, event_definitions: { type: 'object', description: 'Record of sync replication event definitions', @@ -110,7 +79,7 @@ export const syncRulesSchema: ajvModule.Schema = { } } }, - // required: ['bucket_definitions'], + required: ['bucket_definitions'], additionalProperties: false } as const; diff --git a/packages/sync-rules/src/request_functions.ts b/packages/sync-rules/src/request_functions.ts index caa659adb..fc840875f 100644 --- a/packages/sync-rules/src/request_functions.ts +++ b/packages/sync-rules/src/request_functions.ts @@ -56,25 +56,10 @@ const request_user_id: SqlParameterFunction = { usesUnauthenticatedRequestParameters: false }; -export const REQUEST_FUNCTIONS: Record = { +export const REQUEST_FUNCTIONS_NAMED = { parameters: request_parameters, jwt: request_jwt, user_id: request_user_id }; -export const QUERY_FUNCTIONS: Record = { - params: { - debugName: 'stream.params', - call(parameters: ParameterValueSet) { - return parameters.rawUserParameters; - }, - getReturnType() { - return ExpressionType.TEXT; - }, - detail: 'Unauthenticated stream parameters as JSON', - documentation: - 'Returns stream passed by the client when opening the stream. These parameters are not authenticated - any value can be passed in by the client.', - usesAuthenticatedRequestParameters: false, - usesUnauthenticatedRequestParameters: true - } -}; +export const REQUEST_FUNCTIONS: Record = REQUEST_FUNCTIONS_NAMED; diff --git a/packages/sync-rules/src/sql_filters.ts b/packages/sync-rules/src/sql_filters.ts index cde93d56b..c35ce91c6 100644 --- a/packages/sync-rules/src/sql_filters.ts +++ b/packages/sync-rules/src/sql_filters.ts @@ -4,7 +4,7 @@ import { nil } from 'pgsql-ast-parser/src/utils.js'; import { BucketPriority, isValidPriority } from './BucketDescription.js'; import { ExpressionType } from './ExpressionType.js'; import { SqlRuleError } from './errors.js'; -import { QUERY_FUNCTIONS, REQUEST_FUNCTIONS } from './request_functions.js'; +import { REQUEST_FUNCTIONS } from './request_functions.js'; import { BASIC_OPERATORS, OPERATOR_IN, @@ -94,11 +94,6 @@ export interface SqlToolsOptions { */ supportsParameterExpressions?: boolean; - /** - * true if expressions on stream parameters are supported. - */ - supportsStreamInputs?: boolean; - /** * Schema for validations. */ @@ -118,9 +113,6 @@ export class SqlTools { readonly supportsExpandingParameters: boolean; readonly supportsParameterExpressions: boolean; - readonly supportsStreamInputs: boolean; - - private inferredStaticParameters: Map = new Map(); schema?: QuerySchema; @@ -139,7 +131,6 @@ export class SqlTools { this.sql = options.sql; this.supportsExpandingParameters = options.supportsExpandingParameters ?? false; this.supportsParameterExpressions = options.supportsParameterExpressions ?? false; - this.supportsStreamInputs = options.supportsStreamInputs ?? false; } error(message: string, expr: NodeLocation | Expr | undefined): ClauseError { @@ -280,7 +271,7 @@ export class SqlTools { return compileStaticOperator(op, leftFilter as RowValueClause, rightFilter as RowValueClause); } else if (isParameterValueClause(otherFilter)) { // 2. row value = parameter value - const inputParam = this.basicInputParameter(otherFilter); + const inputParam = basicInputParameter(otherFilter); return { error: false, @@ -327,7 +318,7 @@ export class SqlTools { } else if (isParameterValueClause(leftFilter) && isRowValueClause(rightFilter)) { // token_parameters.value IN table.some_array // bucket.param IN table.some_array - const inputParam = this.basicInputParameter(leftFilter); + const inputParam = basicInputParameter(leftFilter); return { error: false, @@ -434,25 +425,7 @@ export class SqlTools { if (fn in REQUEST_FUNCTIONS) { const fnImpl = REQUEST_FUNCTIONS[fn]; return { - key: `stream.${fn}()`, - lookupParameterValue(parameters) { - return fnImpl.call(parameters); - }, - usesAuthenticatedRequestParameters: fnImpl.usesAuthenticatedRequestParameters, - usesUnauthenticatedRequestParameters: fnImpl.usesUnauthenticatedRequestParameters - } satisfies ParameterValueClause; - } else { - return this.error(`Function '${schema}.${fn}' is not defined`, expr); - } - } else if (schema == 'stream') { - if (!this.supportsStreamInputs) { - return this.error(`${schema} schema is only available in stream definitions`, expr); - } - - if (fn in QUERY_FUNCTIONS) { - const fnImpl = QUERY_FUNCTIONS[fn]; - return { - key: `stream.${fn}()`, + key: 'request.parameters()', lookupParameterValue(parameters) { return fnImpl.call(parameters); }, @@ -784,58 +757,8 @@ export class SqlTools { return value as BucketPriority; } - - private basicInputParameter(clause: ParameterValueClause): InputParameter { - if (this.supportsStreamInputs) { - let key = this.inferredStaticParameters.get(clause.key)?.name; - if (key == null) { - key = this.newInferredBucketParameterName(); - this.inferredStaticParameters.set(clause.key, { - name: key, - variant: 'static', - clause - }); - } - - return { - key, - expands: false, - filteredRowToLookupValue: () => { - return SQLITE_FALSE; // Only relevant for parameter queries, but this is a stream query. - }, - parametersToLookupValue: () => { - return SQLITE_FALSE; - } - }; - } - - return { - key: clause.key, - expands: false, - filteredRowToLookupValue: (filterParameters) => { - return filterParameters[clause.key]; - }, - parametersToLookupValue: (parameters) => { - return clause.lookupParameterValue(parameters); - } - }; - } - - public get inferredParameters(): InferredBucketParameter[] { - return [...this.inferredStaticParameters.values()]; - } - - private newInferredBucketParameterName() { - return `p${this.inferredStaticParameters.size}`; - } } -export type InferredBucketParameter = { - name: string; -} & StaticBucketParameter; - -export type StaticBucketParameter = { variant: 'static'; clause: ParameterValueClause }; - function isStatic(expr: Expr) { return ['integer', 'string', 'numeric', 'boolean', 'null'].includes(expr.type); } @@ -872,3 +795,16 @@ function staticValueClause(value: SqliteValue): StaticValueClause { usesUnauthenticatedRequestParameters: false }; } + +function basicInputParameter(clause: ParameterValueClause): InputParameter { + return { + key: clause.key, + expands: false, + filteredRowToLookupValue: (filterParameters) => { + return filterParameters[clause.key]; + }, + parametersToLookupValue: (parameters) => { + return clause.lookupParameterValue(parameters); + } + }; +} From 55df061fcc6db01821d6bd131ded3244aa21ada5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 31 Jul 2025 15:58:58 +0200 Subject: [PATCH 11/21] Unit tests for bucket checksum state --- .../src/routes/endpoints/socket-route.ts | 3 - .../src/routes/endpoints/sync-stream.ts | 6 +- .../src/sync/BucketChecksumState.ts | 59 +-- packages/service-core/src/sync/sync.ts | 27 +- .../service-core/src/util/protocol-types.ts | 28 +- .../test/src/sync/BucketChecksumState.test.ts | 336 ++++++++++++++++-- packages/sync-rules/src/BucketDescription.ts | 9 +- packages/sync-rules/src/BucketSource.ts | 6 +- .../sync-rules/src/SqlBucketDescriptor.ts | 1 + packages/sync-rules/src/SqlParameterQuery.ts | 2 +- packages/sync-rules/src/SqlSyncRules.ts | 4 +- .../sync-rules/src/StaticSqlParameterQuery.ts | 1 - .../TableValuedFunctionSqlParameterQuery.ts | 1 - 13 files changed, 379 insertions(+), 104 deletions(-) diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 7f6a86ddf..b3badfb0e 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -58,8 +58,6 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => return; } - const syncParams = new RequestParameters(context.token_payload!, params.parameters ?? {}); - const { storageEngine: { activeBucketStorage } } = service_context; @@ -95,7 +93,6 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => ...params, binary_data: true // always true for web sockets }, - syncParams, token: context!.token_payload!, tokenStreamOptions: { // RSocket handles keepalive events by default diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 8ca6306ca..8ce2aadf4 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -52,9 +52,6 @@ export const syncStreamed = routeDefinition({ }); } - const params: util.StreamingSyncRequest = payload.params; - const syncParams = new RequestParameters(payload.context.token_payload!, payload.params.parameters ?? {}); - const bucketStorage = await storageEngine.activeBucketStorage.getActiveStorage(); if (bucketStorage == null) { @@ -75,8 +72,7 @@ export const syncStreamed = routeDefinition({ syncContext: syncContext, bucketStorage, syncRules: syncRules, - params, - syncParams, + params: payload.params, token: payload.context.token_payload!, tracker, signal: controller.signal, diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 99cc2d1a6..961658d31 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -4,6 +4,7 @@ import { BucketSource, BucketSourceType, RequestedStream, + RequestJwtPayload, RequestParameters, ResolvedBucket, SqlSyncRules @@ -28,10 +29,9 @@ export interface BucketChecksumStateOptions { syncContext: SyncContext; bucketStorage: BucketChecksumStateStorage; syncRules: SqlSyncRules; - syncParams: RequestParameters; + tokenPayload: RequestJwtPayload; syncRequest: util.StreamingSyncRequest; logger?: Logger; - initialBucketPositions?: { name: string; after: util.InternalOpId }[]; } type BucketSyncState = { @@ -79,14 +79,14 @@ export class BucketChecksumState { options.syncContext, options.bucketStorage, options.syncRules, - options.syncParams, + options.tokenPayload, options.syncRequest, this.logger ); this.bucketDataPositions = new Map(); - for (let { name, after: start } of options.initialBucketPositions ?? []) { - this.bucketDataPositions.set(name, { start_op_id: start }); + for (let { name, after: start } of options.syncRequest.buckets ?? []) { + this.bucketDataPositions.set(name, { start_op_id: BigInt(start) }); } } @@ -199,7 +199,7 @@ export class BucketChecksumState { })); bucketsToFetch = [...generateBucketsToFetch].map((b) => { return { - ...bucketDescriptionMap.get(b)!, + priority: bucketDescriptionMap.get(b)!.priority, bucket: b }; }); @@ -233,11 +233,11 @@ export class BucketChecksumState { message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`; this.logger.info(message, { checkpoint: base.checkpoint, user_id: user_id, buckets: allBuckets.length }); }; - bucketsToFetch = allBuckets; + bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority })); const subscriptions: util.StreamDescription[] = []; for (const source of this.parameterState.syncRules.bucketSources) { - if (source.type == BucketSourceType.SYNC_STREAM && this.parameterState.isSubscribedToStream(source)) { + if (this.parameterState.isSubscribedToStream(source)) { subscriptions.push({ name: source.name, is_default: source.subscribedToByDefault @@ -360,7 +360,11 @@ export class BucketParameterState { public readonly syncRules: SqlSyncRules; public readonly syncParams: RequestParameters; private readonly querier: BucketParameterQuerier; - private readonly staticBuckets: Map; + /** + * Static buckets. This map is guaranteed not to change during a request, since resolving static buckets can only + * take request parameters into account, + */ + private readonly staticBuckets: Map; private readonly includeDefaultStreams: boolean; // Indexed by the client-side id private readonly explicitStreamSubscriptions: Record; @@ -375,22 +379,22 @@ export class BucketParameterState { context: SyncContext, bucketStorage: BucketChecksumStateStorage, syncRules: SqlSyncRules, - syncParams: RequestParameters, + tokenPayload: RequestJwtPayload, request: util.StreamingSyncRequest, logger: Logger ) { this.context = context; this.bucketStorage = bucketStorage; this.syncRules = syncRules; - this.syncParams = syncParams; + this.syncParams = new RequestParameters(tokenPayload, request.parameters ?? {}); this.logger = logger; const idToStreamSubscription: Record = {}; const streamsByName: Record = {}; - const subscriptions = request.subscriptions; + const subscriptions = request.streams; if (subscriptions) { - for (const subscription of subscriptions.opened) { - idToStreamSubscription[subscription.stream] = subscription; + for (const subscription of subscriptions.subscriptions) { + idToStreamSubscription[subscription.client_id] = subscription; const syncRuleStream: RequestedStream = { parameters: subscription.parameters ?? {}, @@ -412,7 +416,7 @@ export class BucketParameterState { streams: streamsByName }); - this.staticBuckets = new Map( + this.staticBuckets = new Map( mergeBuckets(this.querier.staticBuckets).map((b) => [b.bucket, b]) ); this.lookups = new Set(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values))); @@ -441,14 +445,13 @@ export class BucketParameterState { } return { - definition: description.definition, bucket: description.bucket, priority: priorityOverride ?? description.priority, subscriptions: description.inclusion_reasons.map((reason) => { if (reason == 'default') { - return { def: description.definition }; + return { default: 0 }; // TODO } else { - return { sub: reason.subscription }; + return reason.subscription; } }) }; @@ -489,19 +492,19 @@ export class BucketParameterState { * For static buckets, we can keep track of which buckets have been updated. */ private async getCheckpointUpdateStatic(checkpoint: storage.StorageCheckpointUpdate): Promise { - const querier = this.querier; + const staticBuckets = [...this.staticBuckets.values()]; const update = checkpoint.update; if (update.invalidateDataBuckets) { return { - buckets: querier.staticBuckets, + buckets: staticBuckets, updatedBuckets: INVALIDATE_ALL_BUCKETS }; } const updatedBuckets = new Set(getIntersection(this.staticBuckets, update.updatedDataBuckets)); return { - buckets: querier.staticBuckets, + buckets: staticBuckets, updatedBuckets }; } @@ -512,7 +515,7 @@ export class BucketParameterState { private async getCheckpointUpdateDynamic(checkpoint: storage.StorageCheckpointUpdate): Promise { const querier = this.querier; const storage = this.bucketStorage; - const staticBuckets = querier.staticBuckets; + const staticBuckets = this.staticBuckets.values(); const update = checkpoint.update; let hasParameterChange = false; @@ -556,7 +559,7 @@ export class BucketParameterState { } } } - const allBuckets = [...staticBuckets, ...dynamicBuckets]; + const allBuckets = [...staticBuckets, ...mergeBuckets(dynamicBuckets)]; if (invalidateDataBuckets) { return { @@ -632,15 +635,15 @@ function limitedBuckets(buckets: string[] | { bucket: string }[], limit: number) * bucket. */ function mergeBuckets(buckets: ResolvedBucket[]): ResolvedBucket[] { - const byDefinition: Record = {}; + const byBucketId: Record = {}; for (const bucket of buckets) { - if (Object.hasOwn(byDefinition, bucket.definition)) { - byDefinition[bucket.definition].inclusion_reasons.push(...bucket.inclusion_reasons); + if (Object.hasOwn(byBucketId, bucket.bucket)) { + byBucketId[bucket.bucket].inclusion_reasons.push(...bucket.inclusion_reasons); } else { - byDefinition[bucket.definition] = bucket; + byBucketId[bucket.bucket] = structuredClone(bucket); } } - return Object.values(byDefinition); + return Object.values(byBucketId); } diff --git a/packages/service-core/src/sync/sync.ts b/packages/service-core/src/sync/sync.ts index 91aa2bda0..fec216a2e 100644 --- a/packages/service-core/src/sync/sync.ts +++ b/packages/service-core/src/sync/sync.ts @@ -1,5 +1,11 @@ import { JSONBig, JsonContainer } from '@powersync/service-jsonbig'; -import { BucketDescription, BucketPriority, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { + BucketDescription, + BucketPriority, + RequestJwtPayload, + RequestParameters, + SqlSyncRules +} from '@powersync/service-sync-rules'; import { AbortError } from 'ix/aborterror.js'; @@ -19,7 +25,6 @@ export interface SyncStreamParameters { bucketStorage: storage.SyncRulesBucketStorage; syncRules: SqlSyncRules; params: util.StreamingSyncRequest; - syncParams: RequestParameters; token: auth.JwtPayload; logger?: Logger; /** @@ -34,8 +39,7 @@ export interface SyncStreamParameters { export async function* streamResponse( options: SyncStreamParameters ): AsyncIterable { - const { syncContext, bucketStorage, syncRules, params, syncParams, token, tokenStreamOptions, tracker, signal } = - options; + const { syncContext, bucketStorage, syncRules, params, token, tokenStreamOptions, tracker, signal } = options; const logger = options.logger ?? defaultLogger; // We also need to be able to abort, so we create our own controller. @@ -58,7 +62,7 @@ export async function* streamResponse( bucketStorage, syncRules, params, - syncParams, + token, tracker, controller.signal, logger @@ -86,25 +90,22 @@ async function* streamResponseInner( bucketStorage: storage.SyncRulesBucketStorage, syncRules: SqlSyncRules, params: util.StreamingSyncRequest, - syncParams: RequestParameters, + tokenPayload: RequestJwtPayload, tracker: RequestTracker, signal: AbortSignal, logger: Logger ): AsyncGenerator { const { raw_data, binary_data } = params; - const checkpointUserId = util.checkpointUserId(syncParams.tokenParameters.user_id as string, params.client_id); + const userId = tokenPayload.sub; + const checkpointUserId = util.checkpointUserId(userId as string, params.client_id); const checksumState = new BucketChecksumState({ syncContext, bucketStorage, syncRules, - syncParams, + tokenPayload, syncRequest: params, - initialBucketPositions: params.buckets?.map((bucket) => ({ - name: bucket.name, - after: BigInt(bucket.after) - })), logger: logger }); const stream = bucketStorage.watchCheckpointChanges({ @@ -229,7 +230,7 @@ async function* streamResponseInner( onRowsSent: markOperationsSent, abort_connection: signal, abort_batch: abortCheckpointSignal, - user_id: syncParams.userId, + user_id: userId, // Passing null here will emit a full sync complete message at the end. If we pass a priority, we'll emit a partial // sync complete message instead. forPriority: !isLast ? priority : null, diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index c287878c1..dbf408fda 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -23,8 +23,6 @@ export const RequestedStreamSubscription = t.object({ stream: t.string, /** * An opaque textual identifier assigned to this request by the client. - * - * Wh */ client_id: t.string, /** @@ -56,7 +54,7 @@ export const StreamSubscriptionRequest = t.object({ /** * An array of sync streams the client has opened explicitly. */ - opened: t.array(RequestedStreamSubscription) + subscriptions: t.array(RequestedStreamSubscription) }); export type StreamSubscriptionRequest = t.Decoded; @@ -100,7 +98,7 @@ export const StreamingSyncRequest = t.object({ /** * If the client is aware of streams, an array of streams the client has opened. */ - subscriptions: StreamSubscriptionRequest.optional() + streams: StreamSubscriptionRequest.optional() }); export type StreamingSyncRequest = t.Decoded; @@ -227,10 +225,13 @@ export type BucketSubscriptionReason = BucketDerivedFromDefaultStream | BucketDe /** * A bucket has been included in a checkpoint because it's part of a default stream. - * - * The string is the name of the stream definition. */ -export type BucketDerivedFromDefaultStream = { def: string }; +export type BucketDerivedFromDefaultStream = { + /** + * The index (into {@link Checkpoint.streams}) of the stream defining the bucket. + */ + default: number; +}; /** * The bucket has been included in a checkpoint because it's part of a stream that a client has explicitly subscribed @@ -238,9 +239,18 @@ export type BucketDerivedFromDefaultStream = { def: string }; * * The string is the client id associated with the subscription in {@link RequestedStreamSubscription}. */ -export type BucketDerivedFromExplicitSubscription = { sub: string }; +export type BucketDerivedFromExplicitSubscription = string; -export interface ClientBucketDescription extends BucketDescription { +export interface ClientBucketDescription { + /** + * An opaque id of the bucket. + */ + bucket: string; + /** + * The priority used to synchronize this bucket, derived from its definition and an optional priority override from + * the stream subscription. + */ + priority: BucketPriority; subscriptions: BucketSubscriptionReason[]; } diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index 312a7dd6c..40a3f98c3 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -1,17 +1,27 @@ import { BucketChecksum, BucketChecksumState, + BucketChecksumStateOptions, BucketChecksumStateStorage, CHECKPOINT_INVALIDATE_ALL, ChecksumMap, InternalOpId, ReplicationCheckpoint, + StreamingSyncRequest, SyncContext, WatchFilterEvent } from '@/index.js'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookup, RequestParameters, SqliteJsonRow, SqlSyncRules } from '@powersync/service-sync-rules'; -import { describe, expect, test } from 'vitest'; +import { + SqliteJsonRow, + ParameterLookup, + SqlSyncRules, + RequestJwtPayload, + BucketSource, + BucketSourceType, + BucketParameterQuerier +} from '@powersync/service-sync-rules'; +import { describe, expect, test, beforeEach } from 'vitest'; describe('BucketChecksumState', () => { // Single global[] bucket. @@ -55,6 +65,9 @@ bucket_definitions: maxDataFetchConcurrency: 10 }); + const syncRequest: StreamingSyncRequest = {}; + const tokenPayload: RequestJwtPayload = { sub: '' }; + test('global bucket with update', async () => { const storage = new MockBucketChecksumStateStorage(); // Set intial state @@ -62,7 +75,8 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, - syncParams: new RequestParameters({ sub: '' }, {}), + syncRequest, + tokenPayload, syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -75,9 +89,10 @@ bucket_definitions: line.advance(); expect(line.checkpointLine).toEqual({ checkpoint: { - buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }], + buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '1', - write_checkpoint: undefined + write_checkpoint: undefined, + streams: [{ name: 'global', is_default: true }] } }); expect(line.bucketsToFetch).toEqual([ @@ -111,7 +126,7 @@ bucket_definitions: expect(line2.checkpointLine).toEqual({ checkpoint_diff: { removed_buckets: [], - updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3 }], + updated_buckets: [{ bucket: 'global[]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '2', write_checkpoint: undefined } @@ -129,9 +144,9 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, + tokenPayload, // Client sets the initial state here - initialBucketPositions: [{ name: 'global[]', after: 1n }], - syncParams: new RequestParameters({ sub: '' }, {}), + syncRequest: { buckets: [{ name: 'global[]', after: '1' }] }, syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -144,9 +159,10 @@ bucket_definitions: line.advance(); expect(line.checkpointLine).toEqual({ checkpoint: { - buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }], + buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '1', - write_checkpoint: undefined + write_checkpoint: undefined, + streams: [{ name: 'global', is_default: true }] } }); expect(line.bucketsToFetch).toEqual([ @@ -167,7 +183,8 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, - syncParams: new RequestParameters({ sub: '' }, {}), + tokenPayload, + syncRequest, syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -180,11 +197,12 @@ bucket_definitions: expect(line.checkpointLine).toEqual({ checkpoint: { buckets: [ - { bucket: 'global[1]', checksum: 1, count: 1, priority: 3 }, - { bucket: 'global[2]', checksum: 1, count: 1, priority: 3 } + { bucket: 'global[1]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }, + { bucket: 'global[2]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] } ], last_op_id: '1', - write_checkpoint: undefined + write_checkpoint: undefined, + streams: [{ name: 'global', is_default: true }] } }); expect(line.bucketsToFetch).toEqual([ @@ -215,8 +233,8 @@ bucket_definitions: checkpoint_diff: { removed_buckets: [], updated_buckets: [ - { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 }, - { bucket: 'global[2]', checksum: 2, count: 2, priority: 3 } + { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }, + { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] } ], last_op_id: '2', write_checkpoint: undefined @@ -232,9 +250,9 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, + tokenPayload, // Client sets the initial state here - initialBucketPositions: [{ name: 'something_here[]', after: 1n }], - syncParams: new RequestParameters({ sub: '' }, {}), + syncRequest: { buckets: [{ name: 'something_here[]', after: '1' }] }, syncRules: SYNC_RULES_GLOBAL, bucketStorage: storage }); @@ -249,9 +267,10 @@ bucket_definitions: line.advance(); expect(line.checkpointLine).toEqual({ checkpoint: { - buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3 }], + buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '1', - write_checkpoint: undefined + write_checkpoint: undefined, + streams: [{ name: 'global', is_default: true }] } }); expect(line.bucketsToFetch).toEqual([ @@ -273,7 +292,8 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, - syncParams: new RequestParameters({ sub: '' }, {}), + tokenPayload, + syncRequest, syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -310,7 +330,7 @@ bucket_definitions: removed_buckets: [], updated_buckets: [ // This does not include global[2], since it was not invalidated. - { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 } + { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] } ], last_op_id: '2', write_checkpoint: undefined @@ -325,7 +345,8 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, - syncParams: new RequestParameters({ sub: '' }, {}), + tokenPayload, + syncRequest, syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -358,8 +379,8 @@ bucket_definitions: checkpoint_diff: { removed_buckets: [], updated_buckets: [ - { bucket: 'global[1]', checksum: 2, count: 2, priority: 3 }, - { bucket: 'global[2]', checksum: 2, count: 2, priority: 3 } + { bucket: 'global[1]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] }, + { bucket: 'global[2]', checksum: 2, count: 2, priority: 3, subscriptions: [{ default: 0 }] } ], last_op_id: '2', write_checkpoint: undefined @@ -379,7 +400,8 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, - syncParams: new RequestParameters({ sub: '' }, {}), + tokenPayload, + syncRequest, syncRules: SYNC_RULES_GLOBAL_TWO, bucketStorage: storage }); @@ -393,11 +415,12 @@ bucket_definitions: expect(line.checkpointLine).toEqual({ checkpoint: { buckets: [ - { bucket: 'global[1]', checksum: 3, count: 3, priority: 3 }, - { bucket: 'global[2]', checksum: 3, count: 3, priority: 3 } + { bucket: 'global[1]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] }, + { bucket: 'global[2]', checksum: 3, count: 3, priority: 3, subscriptions: [{ default: 0 }] } ], last_op_id: '3', - write_checkpoint: undefined + write_checkpoint: undefined, + streams: [{ name: 'global', is_default: true }] } }); expect(line.bucketsToFetch).toEqual([ @@ -444,7 +467,8 @@ bucket_definitions: bucket: 'global[1]', checksum: 4, count: 4, - priority: 3 + priority: 3, + subscriptions: [{ default: 0 }] } ], last_op_id: '4', @@ -480,7 +504,8 @@ bucket_definitions: const state = new BucketChecksumState({ syncContext, - syncParams: new RequestParameters({ sub: 'u1' }, {}), + tokenPayload: { sub: 'u1' }, + syncRequest, syncRules: SYNC_RULES_DYNAMIC, bucketStorage: storage }); @@ -496,10 +521,28 @@ bucket_definitions: expect(line.checkpointLine).toEqual({ checkpoint: { buckets: [ - { bucket: 'by_project[1]', checksum: 1, count: 1, priority: 3 }, - { bucket: 'by_project[2]', checksum: 1, count: 1, priority: 3 } + { + bucket: 'by_project[1]', + checksum: 1, + count: 1, + priority: 3, + subscriptions: [{ default: 0 }] + }, + { + bucket: 'by_project[2]', + checksum: 1, + count: 1, + priority: 3, + subscriptions: [{ default: 0 }] + } ], last_op_id: '1', + streams: [ + { + is_default: true, + name: 'by_project' + } + ], write_checkpoint: undefined } }); @@ -544,13 +587,238 @@ bucket_definitions: expect(line2.checkpointLine).toEqual({ checkpoint_diff: { removed_buckets: [], - updated_buckets: [{ bucket: 'by_project[3]', checksum: 1, count: 1, priority: 3 }], + updated_buckets: [ + { + bucket: 'by_project[3]', + checksum: 1, + count: 1, + priority: 3, + subscriptions: [{ default: 0 }] + } + ], last_op_id: '2', write_checkpoint: undefined } }); expect(line2.getFilteredBucketPositions()).toEqual(new Map([['by_project[3]', 0n]])); }); + + describe('streams', () => { + let source: { -readonly [P in keyof BucketSource]: BucketSource[P] }; + let storage: MockBucketChecksumStateStorage; + let staticBucketIds = ['stream|0[]']; + + function checksumState(options?: Partial) { + const rules = new SqlSyncRules(''); + rules.bucketSources.push(source); + + return new BucketChecksumState({ + syncContext, + syncRequest, + tokenPayload, + syncRules: rules, + bucketStorage: storage, + ...options + }); + } + + function createQuerier(ids: string[], subscription: string | null): BucketParameterQuerier { + return { + staticBuckets: ids.map((bucket) => ({ + definition: 'stream', + inclusion_reasons: subscription == null ? ['default'] : [{ subscription }], + bucket, + priority: 3 + })), + hasDynamicBuckets: false, + parameterQueryLookups: [], + queryDynamicBucketDescriptions: function (): never { + throw new Error('no dynamic buckets.'); + } + }; + } + + beforeEach(() => { + // Currently using mocked streams before streams are actually implemented as parsable rules. + source = { + name: 'stream', + type: BucketSourceType.SYNC_STREAM, + subscribedToByDefault: false, + pushBucketParameterQueriers(result, options) { + // Create a fake querier that resolves teh global stream["default"] bucket by default and allows extracting + // additional buckets from parameters. + const subscriptions = options.streams['stream'] ?? []; + if (!this.subscribedToByDefault && !subscriptions.length) { + return; + } + + let hasExplicitDefaultSubscription = false; + for (const subscription of subscriptions) { + let subscriptionParameters = []; + + if (subscription.parameters != null) { + subscriptionParameters = JSON.parse(subscription.parameters['ids'] as string).map( + (e: string) => `stream["${e}"]` + ); + } else { + hasExplicitDefaultSubscription = true; + } + + result.push(createQuerier([...subscriptionParameters], subscription.opaque_id)); + } + + // If the stream is subscribed to by default and there is no explicit subscription that would match the default + // subscription, also include the default querier. + if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { + result.push(createQuerier(['stream["default"]'], null)); + } + } + } satisfies Partial as any; + + storage = new MockBucketChecksumStateStorage(); + storage.updateTestChecksum({ bucket: 'stream["default"]', checksum: 1, count: 1 }); + storage.updateTestChecksum({ bucket: 'stream["a"]', checksum: 1, count: 1 }); + storage.updateTestChecksum({ bucket: 'stream["b"]', checksum: 1, count: 1 }); + }); + + test('includes defaults', async () => { + source.subscribedToByDefault = true; + const state = checksumState(); + + const line = await state.buildNextCheckpointLine({ + base: { checkpoint: 1n, lsn: '1' }, + writeCheckpoint: null, + update: CHECKPOINT_INVALIDATE_ALL + })!; + line?.advance(); + expect(line?.checkpointLine).toEqual({ + checkpoint: { + buckets: [ + { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] } + ], + last_op_id: '1', + write_checkpoint: undefined, + streams: [{ name: 'stream', is_default: true }] + } + }); + }); + + test('can exclude defaults', async () => { + source.subscribedToByDefault = true; + const state = checksumState({ syncRequest: { streams: { include_defaults: false, subscriptions: [] } } }); + + const line = await state.buildNextCheckpointLine({ + base: { checkpoint: 1n, lsn: '1' }, + writeCheckpoint: null, + update: CHECKPOINT_INVALIDATE_ALL + })!; + line?.advance(); + expect(line?.checkpointLine).toEqual({ + checkpoint: { + buckets: [], + last_op_id: '1', + write_checkpoint: undefined, + streams: [] + } + }); + }); + + test('custom subscriptions', async () => { + source.subscribedToByDefault = true; + + const state = checksumState({ + syncRequest: { + streams: { + subscriptions: [ + { stream: 'stream', client_id: '1', parameters: { ids: '["a"]' } }, + { stream: 'stream', client_id: '2', parameters: { ids: '["b"]' }, override_priority: 1 } + ] + } + } + }); + + const line = await state.buildNextCheckpointLine({ + base: { checkpoint: 1n, lsn: '1' }, + writeCheckpoint: null, + update: CHECKPOINT_INVALIDATE_ALL + })!; + line?.advance(); + expect(line?.checkpointLine).toEqual({ + checkpoint: { + buckets: [ + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: ['1'] }, + { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: ['2'] }, + { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] } + ], + last_op_id: '1', + write_checkpoint: undefined, + streams: [{ name: 'stream', is_default: true }] + } + }); + }); + + test('overlap between custom subscriptions', async () => { + const state = checksumState({ + syncRequest: { + streams: { + subscriptions: [ + { stream: 'stream', client_id: '1', parameters: { ids: '["a", "b"]' } }, + { stream: 'stream', client_id: '2', parameters: { ids: '["b"]' }, override_priority: 1 } + ] + } + } + }); + + const line = await state.buildNextCheckpointLine({ + base: { checkpoint: 1n, lsn: '1' }, + writeCheckpoint: null, + update: CHECKPOINT_INVALIDATE_ALL + })!; + line?.advance(); + expect(line?.checkpointLine).toEqual({ + checkpoint: { + buckets: [ + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: ['1'] }, + { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: ['1', '2'] } + ], + last_op_id: '1', + write_checkpoint: undefined, + streams: [{ name: 'stream', is_default: false }] + } + }); + }); + + test('overlap between default and custom subscription', async () => { + source.subscribedToByDefault = true; + const state = checksumState({ + syncRequest: { + streams: { + subscriptions: [ + { stream: 'stream', client_id: '1', parameters: { ids: '["a", "default"]' }, override_priority: 1 } + ] + } + } + }); + + const line = await state.buildNextCheckpointLine({ + base: { checkpoint: 1n, lsn: '1' }, + writeCheckpoint: null, + update: CHECKPOINT_INVALIDATE_ALL + })!; + line?.advance(); + expect(line?.checkpointLine).toEqual({ + checkpoint: { + buckets: [ + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: ['1'] }, + { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 1, subscriptions: ['1', { default: 0 }] } + ], + last_op_id: '1', + write_checkpoint: undefined, + streams: [{ name: 'stream', is_default: true }] + } + }); + }); + }); }); class MockBucketChecksumStateStorage implements BucketChecksumStateStorage { diff --git a/packages/sync-rules/src/BucketDescription.ts b/packages/sync-rules/src/BucketDescription.ts index 399f1300d..eb9a3a628 100644 --- a/packages/sync-rules/src/BucketDescription.ts +++ b/packages/sync-rules/src/BucketDescription.ts @@ -20,11 +20,6 @@ export const isValidPriority = (i: number): i is BucketPriority => { }; export interface BucketDescription { - /** - * The name of the sync rule or stream definition from which the bucket is derived. - */ - definition: string; - /** * The id of the bucket, which is derived from the name of the bucket's definition * in the sync rules as well as the values returned by the parameter queries. @@ -43,6 +38,10 @@ export interface BucketDescription { * shown to clients. */ export interface ResolvedBucket extends BucketDescription { + /** + * The name of the sync rule or stream definition from which the bucket is derived. + */ + definition: string; inclusion_reasons: BucketInclusionReason[]; } diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 5e2c8a8fe..586f86d2e 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -16,10 +16,10 @@ import { EvaluatedParametersResult, EvaluateRowOptions, EvaluationResult, Source * definitions that only consist of a single query. */ export interface BucketSource { - name: string; - type: BucketSourceType; + readonly name: string; + readonly type: BucketSourceType; - subscribedToByDefault: boolean; + readonly subscribedToByDefault: boolean; /** * Given a row in a source table that affects sync parameters, returns a structure to index which buckets rows should diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index f5448da0f..52fd9709f 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -151,6 +151,7 @@ export class SqlBucketDescriptor implements BucketSource { for (const desc of query.getStaticBucketDescriptions(parameters)) { results.push({ ...desc, + definition: this.name, inclusion_reasons: reasons }); } diff --git a/packages/sync-rules/src/SqlParameterQuery.ts b/packages/sync-rules/src/SqlParameterQuery.ts index a49421762..37bf7e11a 100644 --- a/packages/sync-rules/src/SqlParameterQuery.ts +++ b/packages/sync-rules/src/SqlParameterQuery.ts @@ -372,7 +372,6 @@ export class SqlParameterQuery { } return { - definition: this.descriptorName, bucket: getBucketId(this.descriptorName, this.bucketParameters, result), priority: this.priority }; @@ -480,6 +479,7 @@ export class SqlParameterQuery { const bucketParameters = await source.getParameterSets(lookups); return this.resolveBucketDescriptions(bucketParameters, requestParameters).map((bucket) => ({ ...bucket, + definition: this.descriptorName, inclusion_reasons: reasons })); } diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index b3b5a046b..cfaaa4741 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -358,7 +358,9 @@ export class SqlSyncRules implements SyncRules { getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { const queriers: BucketParameterQuerier[] = []; for (const source of this.bucketSources) { - source.pushBucketParameterQueriers(queriers, options); + if ((source.subscribedToByDefault && options.hasDefaultStreams) || source.name in options.streams) { + source.pushBucketParameterQueriers(queriers, options); + } } return mergeBucketParameterQueriers(queriers); diff --git a/packages/sync-rules/src/StaticSqlParameterQuery.ts b/packages/sync-rules/src/StaticSqlParameterQuery.ts index ed461401e..414ee2bdd 100644 --- a/packages/sync-rules/src/StaticSqlParameterQuery.ts +++ b/packages/sync-rules/src/StaticSqlParameterQuery.ts @@ -177,7 +177,6 @@ export class StaticSqlParameterQuery { return [ { - definition: this.descriptorName, bucket: getBucketId(this.descriptorName, this.bucketParameters, result), priority: this.priority } diff --git a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts index c56ef4b14..a19930af3 100644 --- a/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts +++ b/packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts @@ -247,7 +247,6 @@ export class TableValuedFunctionSqlParameterQuery { } return { - definition: this.descriptorName, bucket: getBucketId(this.descriptorName, this.bucketParameters, result), priority: this.priority }; From bfd62d47a3e7c913d2c2d647c3d0b391fdb45ca7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 31 Jul 2025 16:10:35 +0200 Subject: [PATCH 12/21] Fix more tsts --- .../src/test-utils/general-utils.ts | 10 ++++- .../src/tests/register-data-storage-tests.ts | 35 +++++++++------- .../src/tests/register-sync-tests.ts | 41 ++++++------------- 3 files changed, 42 insertions(+), 44 deletions(-) diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 738db1c4a..524a00116 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -1,5 +1,5 @@ import { storage, utils } from '@powersync/service-core'; -import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { GetQuerierOptions, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; export const ZERO_LSN = '0/0'; @@ -102,3 +102,11 @@ function getFirst( export function rid(id: string): bson.UUID { return utils.getUuidReplicaIdentityBson({ id: id }, [{ name: 'id', type: 'VARCHAR', typeId: 25 }]); } + +export function querierOptions(globalParameters: RequestParameters): GetQuerierOptions { + return { + globalParameters, + hasDefaultStreams: true, + streams: {} + }; +} diff --git a/packages/service-core-tests/src/tests/register-data-storage-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-tests.ts index 3eec6dab2..1feb12426 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-tests.ts @@ -8,6 +8,7 @@ import { import { ParameterLookup, RequestParameters } from '@powersync/service-sync-rules'; import { expect, test, describe, beforeEach } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { SqlBucketDescriptor } from '@powersync/service-sync-rules/src/SqlBucketDescriptor.js'; export const TEST_TABLE = test_utils.makeTestTable('test', ['id']); @@ -411,7 +412,7 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); - const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0]; + const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; const lookups = q1.getLookups(parameters); expect(lookups).toEqual([ParameterLookup.normalized('by_workspace', '1', ['u1'])]); @@ -419,11 +420,13 @@ bucket_definitions: const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); - const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }); + const buckets = await sync_rules + .getBucketParameterQuerier(test_utils.querierOptions(parameters)) + .queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return bucketStorage.getParameterSets(lookups); + } + }); expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]); }); @@ -482,7 +485,7 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'unknown' }, {}); - const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0]; + const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; const lookups = q1.getLookups(parameters); expect(lookups).toEqual([ParameterLookup.normalized('by_public_workspace', '1', [])]); @@ -491,11 +494,13 @@ bucket_definitions: parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); - const buckets = await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }); + const buckets = await sync_rules + .getBucketParameterQuerier(test_utils.querierOptions(parameters)) + .queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return bucketStorage.getParameterSets(lookups); + } + }); buckets.sort((a, b) => a.bucket.localeCompare(b.bucket)); expect(buckets).toEqual([ { bucket: 'by_public_workspace["workspace1"]', priority: 3 }, @@ -573,7 +578,7 @@ bucket_definitions: const parameters = new RequestParameters({ sub: 'u1' }, {}); // Test intermediate values - could be moved to sync_rules.test.ts - const q1 = sync_rules.bucketDescriptors[0].parameterQueries[0]; + const q1 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[0]; const lookups1 = q1.getLookups(parameters); expect(lookups1).toEqual([ParameterLookup.normalized('by_workspace', '1', [])]); @@ -581,7 +586,7 @@ bucket_definitions: parameter_sets1.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets1).toEqual([{ workspace_id: 'workspace1' }]); - const q2 = sync_rules.bucketDescriptors[0].parameterQueries[1]; + const q2 = (sync_rules.bucketSources[0] as SqlBucketDescriptor).parameterQueries[1]; const lookups2 = q2.getLookups(parameters); expect(lookups2).toEqual([ParameterLookup.normalized('by_workspace', '2', ['u1'])]); @@ -591,7 +596,7 @@ bucket_definitions: // Test final values - the important part const buckets = ( - await sync_rules.getBucketParameterQuerier(parameters).queryDynamicBucketDescriptions({ + await sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).queryDynamicBucketDescriptions({ getParameterSets(lookups) { return checkpoint.getParameterSets(lookups); } diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index fc661dd48..4c7224b43 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -89,8 +89,7 @@ export function registerSyncTests(factory: storage.TestStorageFactory) { raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); const lines = await consumeCheckpointLines(stream); @@ -150,8 +149,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); const lines = await consumeCheckpointLines(stream); @@ -213,8 +211,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); let sentCheckpoints = 0; @@ -323,7 +320,6 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: 'user_one' }, {}), token: { sub: 'user_one', exp: Date.now() / 1000 + 100000 } as any }); @@ -464,8 +460,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); let sentRows = 0; @@ -580,8 +575,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 100000 } as any + token: { sub: '', exp: Date.now() / 1000 + 100000 } as any }); const lines: any[] = []; @@ -646,8 +640,7 @@ bucket_definitions: raw_data: false }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); const lines = await consumeCheckpointLines(stream); @@ -675,8 +668,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: 0 } as any + token: { sub: '', exp: 0 } as any }); const lines = await consumeCheckpointLines(stream); @@ -706,8 +698,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); const iter = stream[Symbol.asyncIterator](); context.onTestFinished(() => { @@ -780,8 +771,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: 'user1' }, {}), - token: { exp: Date.now() / 1000 + 100 } as any + token: { sub: 'user1', exp: Date.now() / 1000 + 100 } as any }); const iter = stream[Symbol.asyncIterator](); context.onTestFinished(() => { @@ -856,8 +846,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: 'user1' }, {}), - token: { exp: Date.now() / 1000 + 100 } as any + token: { sub: 'user1', exp: Date.now() / 1000 + 100 } as any }); const iter = stream[Symbol.asyncIterator](); context.onTestFinished(() => { @@ -923,8 +912,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: 'user1' }, {}), - token: { exp: Date.now() / 1000 + 100 } as any + token: { sub: 'user1', exp: Date.now() / 1000 + 100 } as any }); const iter = stream[Symbol.asyncIterator](); context.onTestFinished(() => { @@ -991,8 +979,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: exp } as any + token: { sub: '', exp: exp } as any }); const iter = stream[Symbol.asyncIterator](); context.onTestFinished(() => { @@ -1054,8 +1041,7 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: '' }, {}), - token: { exp: Date.now() / 1000 + 10 } as any + token: { sub: '', exp: Date.now() / 1000 + 10 } as any }); const iter = stream[Symbol.asyncIterator](); @@ -1180,7 +1166,6 @@ bucket_definitions: raw_data: true }, tracker, - syncParams: new RequestParameters({ sub: 'test' }, {}), token: { sub: 'test', exp: Date.now() / 1000 + 10 } as any }; const stream1 = sync.streamResponse(params); From 3f5e9c3bf0561b189edce9d49b80cfbf592333ef Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 31 Jul 2025 16:47:29 +0200 Subject: [PATCH 13/21] Build correct index --- .../src/sync/BucketChecksumState.ts | 34 +++++++++++++----- .../test/src/parameter_queries.test.ts | 6 ++-- .../test/src/static_parameter_queries.test.ts | 32 ++++++++--------- .../src/table_valued_function_queries.test.ts | 36 +++++++++---------- 4 files changed, 63 insertions(+), 45 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 961658d31..818f3dffb 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -2,7 +2,6 @@ import { BucketDescription, BucketPriority, BucketSource, - BucketSourceType, RequestedStream, RequestJwtPayload, RequestParameters, @@ -60,6 +59,17 @@ export class BucketChecksumState { */ private lastChecksums: util.ChecksumMap | null = null; private lastWriteCheckpoint: bigint | null = null; + /** + * Once we've sent the first full checkpoint line including all {@link util.Checkpoint.streams} that the user is + * subscribed to, we keep an index of the stream names to their index in that array. + * + * This is used to compress the representation of buckets in `checkpoint` and `checkpoint_diff` lines: For buckets + * that are part of sync rules or default streams, we need to include the name of the defining sync rule or definition + * yielding that bucket (so that clients can track progress for default streams). + * But instead of sending the name for each bucket, we use the fact that it's part of the streams array and only send + * their index, reducing the size of those messages. + */ + private streamNameToIndex: Map | null = null; private readonly parameterState: BucketParameterState; @@ -111,9 +121,7 @@ export class BucketChecksumState { const { buckets: allBuckets, updatedBuckets } = update; /** Set of all buckets in this checkpoint. */ - const bucketDescriptionMap = new Map( - allBuckets.map((b) => [b.bucket, this.parameterState.translateResolvedBucket(b)]) - ); + const bucketDescriptionMap = new Map(allBuckets.map((b) => [b.bucket, b])); if (bucketDescriptionMap.size > this.context.maxBuckets) { throw new ServiceError( @@ -171,6 +179,7 @@ export class BucketChecksumState { // TODO: If updatedBuckets is present, we can use that to more efficiently calculate a diff, // and avoid any unnecessary loops through the entire list of buckets. const diff = util.checksumsDiff(this.lastChecksums, checksumMap); + const streamNameToIndex = this.streamNameToIndex!; if ( this.lastWriteCheckpoint == writeCheckpoint && @@ -195,7 +204,7 @@ export class BucketChecksumState { const updatedBucketDescriptions = diff.updatedBuckets.map((e) => ({ ...e, - ...bucketDescriptionMap.get(e.bucket)! + ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex) })); bucketsToFetch = [...generateBucketsToFetch].map((b) => { return { @@ -236,8 +245,13 @@ export class BucketChecksumState { bucketsToFetch = allBuckets.map((b) => ({ bucket: b.bucket, priority: b.priority })); const subscriptions: util.StreamDescription[] = []; + const streamNameToIndex = new Map(); + this.streamNameToIndex = streamNameToIndex; + for (const source of this.parameterState.syncRules.bucketSources) { if (this.parameterState.isSubscribedToStream(source)) { + streamNameToIndex.set(source.name, subscriptions.length); + subscriptions.push({ name: source.name, is_default: source.subscribedToByDefault @@ -251,7 +265,7 @@ export class BucketChecksumState { write_checkpoint: writeCheckpoint ? String(writeCheckpoint) : undefined, buckets: [...checksumMap.values()].map((e) => ({ ...e, - ...bucketDescriptionMap.get(e.bucket)! + ...this.parameterState.translateResolvedBucket(bucketDescriptionMap.get(e.bucket)!, streamNameToIndex) })), streams: subscriptions } @@ -426,8 +440,11 @@ export class BucketParameterState { /** * Translates an internal sync-rules {@link ResolvedBucket} instance to the public * {@link util.ClientBucketDescription}. + * + * @param lookupIndex A map from stream names to their index in {@link util.Checkpoint.streams}. These are used to + * reference default buckets by their stream index instead of duplicating the name on wire. */ - translateResolvedBucket(description: ResolvedBucket): util.ClientBucketDescription { + translateResolvedBucket(description: ResolvedBucket, lookupIndex: Map): util.ClientBucketDescription { // If the client is overriding the priority of any stream that yields this bucket, sync the bucket with that // priority. let priorityOverride: BucketPriority | null = null; @@ -449,7 +466,8 @@ export class BucketParameterState { priority: priorityOverride ?? description.priority, subscriptions: description.inclusion_reasons.map((reason) => { if (reason == 'default') { - return { default: 0 }; // TODO + const stream = description.definition; + return { default: lookupIndex.get(stream)! }; } else { return reason.subscription; } diff --git a/packages/sync-rules/test/src/parameter_queries.test.ts b/packages/sync-rules/test/src/parameter_queries.test.ts index 93125eadd..12835dbec 100644 --- a/packages/sync-rules/test/src/parameter_queries.test.ts +++ b/packages/sync-rules/test/src/parameter_queries.test.ts @@ -84,11 +84,11 @@ describe('parameter queries', () => { // We _do_ need to care about the bucket string representation. expect( query.resolveBucketDescriptions([{ int1: 314, float1: 3.14, float2: 314 }], normalizeTokenParameters({})) - ).toEqual([{ bucket: 'mybucket[314,3.14,314]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); expect( query.resolveBucketDescriptions([{ int1: 314n, float1: 3.14, float2: 314 }], normalizeTokenParameters({})) - ).toEqual([{ bucket: 'mybucket[314,3.14,314]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[314,3.14,314]', priority: 3 }]); }); test('plain token_parameter (baseline)', () => { @@ -365,7 +365,7 @@ describe('parameter queries', () => { [{ user_id: 'user1' }], normalizeTokenParameters({ user_id: 'user1', is_admin: true }) ) - ).toEqual([{ bucket: 'mybucket["user1",1]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket["user1",1]', priority: 3 }]); }); test('case-sensitive parameter queries (1)', () => { diff --git a/packages/sync-rules/test/src/static_parameter_queries.test.ts b/packages/sync-rules/test/src/static_parameter_queries.test.ts index bfc28676f..611afd7b1 100644 --- a/packages/sync-rules/test/src/static_parameter_queries.test.ts +++ b/packages/sync-rules/test/src/static_parameter_queries.test.ts @@ -10,7 +10,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual(['user_id']); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["user1"]', priority: 3 } ]); }); @@ -20,7 +20,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.bucketParameters!).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -29,7 +29,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: true }))).toEqual([ - { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["user1"]', priority: 3 } ]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1', is_admin: false }))).toEqual( [] @@ -41,7 +41,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["USER1"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["USER1"]', priority: 3 } ]); expect(query.bucketParameters!).toEqual(['upper_id']); }); @@ -51,7 +51,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'admin' }))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ role: 'user' }))).toEqual([]); }); @@ -61,7 +61,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't1' }))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ id1: 't1', id2: 't2' }))).toEqual([]); }); @@ -80,7 +80,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({}, { org_id: 'test' }))).toEqual([ - { bucket: 'mybucket["test"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["test"]', priority: 3 } ]); }); @@ -91,7 +91,7 @@ describe('static parameter queries', () => { expect(query.bucketParameters).toEqual(['user_id']); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["user1"]', priority: 3 } ]); }); @@ -102,7 +102,7 @@ describe('static parameter queries', () => { expect(query.bucketParameters).toEqual(['user_id']); expect(query.getStaticBucketDescriptions(normalizeTokenParameters({ user_id: 'user1' }))).toEqual([ - { bucket: 'mybucket["user1"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["user1"]', priority: 3 } ]); }); @@ -111,7 +111,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -120,7 +120,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -136,7 +136,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); }); @@ -147,10 +147,10 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {})) - ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {})) - ).toEqual([{ bucket: 'mybucket[0]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[0]', priority: 3 }]); }); test('IN for permissions in request.jwt() (2)', function () { @@ -160,7 +160,7 @@ describe('static parameter queries', () => { expect(query.errors).toEqual([]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {})) - ).toEqual([{ bucket: 'mybucket[]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[]', priority: 3 }]); expect( query.getStaticBucketDescriptions(new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {})) ).toEqual([]); @@ -171,7 +171,7 @@ describe('static parameter queries', () => { const query = SqlParameterQuery.fromSql('mybucket', sql, PARSE_OPTIONS, '1') as StaticSqlParameterQuery; expect(query.errors).toEqual([]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superuser' }, {}))).toEqual([ - { bucket: 'mybucket[]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[]', priority: 3 } ]); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '', role: 'superadmin' }, {}))).toEqual([]); }); diff --git a/packages/sync-rules/test/src/table_valued_function_queries.test.ts b/packages/sync-rules/test/src/table_valued_function_queries.test.ts index 5454f81e1..596ade82b 100644 --- a/packages/sync-rules/test/src/table_valued_function_queries.test.ts +++ b/packages/sync-rules/test/src/table_valued_function_queries.test.ts @@ -19,9 +19,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[1]', priority: 3 }, + { bucket: 'mybucket[2]', priority: 3 }, + { bucket: 'mybucket[3]', priority: 3 } ]); }); @@ -32,9 +32,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[1]', priority: 3 }, + { bucket: 'mybucket[2]', priority: 3 }, + { bucket: 'mybucket[3]', priority: 3 } ]); }); @@ -88,9 +88,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, {}))).toEqual([ - { bucket: 'mybucket["a"]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket["b"]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket["c"]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket["a"]', priority: 3 }, + { bucket: 'mybucket["b"]', priority: 3 }, + { bucket: 'mybucket["c"]', priority: 3 } ]); }); @@ -109,9 +109,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[1]', priority: 3 }, + { bucket: 'mybucket[2]', priority: 3 }, + { bucket: 'mybucket[3]', priority: 3 } ]); }); @@ -130,9 +130,9 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['value']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[1]', priority: 3 }, + { bucket: 'mybucket[2]', priority: 3 }, + { bucket: 'mybucket[3]', priority: 3 } ]); }); @@ -151,8 +151,8 @@ describe('table-valued function queries', () => { expect(query.bucketParameters).toEqual(['v']); expect(query.getStaticBucketDescriptions(new RequestParameters({ sub: '' }, { array: [1, 2, 3] }))).toEqual([ - { bucket: 'mybucket[2]', definition: 'mybucket', priority: 3 }, - { bucket: 'mybucket[3]', definition: 'mybucket', priority: 3 } + { bucket: 'mybucket[2]', priority: 3 }, + { bucket: 'mybucket[3]', priority: 3 } ]); }); @@ -184,7 +184,7 @@ describe('table-valued function queries', () => { {} ) ) - ).toEqual([{ bucket: 'mybucket[1]', definition: 'mybucket', priority: 3 }]); + ).toEqual([{ bucket: 'mybucket[1]', priority: 3 }]); }); describe('dangerous queries', function () { From c4b8496b56cee639e7c8bcacd5b3587364a3e7f5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 5 Aug 2025 14:30:17 +0200 Subject: [PATCH 14/21] Review feedback --- .../src/sync/BucketChecksumState.ts | 14 ++++---- .../service-core/src/util/protocol-types.ts | 34 +++++++++++++++---- packages/sync-rules/src/BucketDescription.ts | 2 +- packages/sync-rules/src/SqlSyncRules.ts | 2 +- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 818f3dffb..25c6c2a92 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -381,7 +381,7 @@ export class BucketParameterState { private readonly staticBuckets: Map; private readonly includeDefaultStreams: boolean; // Indexed by the client-side id - private readonly explicitStreamSubscriptions: Record; + private readonly explicitStreamSubscriptions: util.RequestedStreamSubscription[]; private readonly subscribedStreamNames: Set; private readonly logger: Logger; private cachedDynamicBuckets: ResolvedBucket[] | null = null; @@ -403,16 +403,16 @@ export class BucketParameterState { this.syncParams = new RequestParameters(tokenPayload, request.parameters ?? {}); this.logger = logger; - const idToStreamSubscription: Record = {}; const streamsByName: Record = {}; const subscriptions = request.streams; + const explicitStreamSubscriptions: util.RequestedStreamSubscription[] = subscriptions?.subscriptions ?? []; if (subscriptions) { - for (const subscription of subscriptions.subscriptions) { - idToStreamSubscription[subscription.client_id] = subscription; + for (let i = 0; i < explicitStreamSubscriptions.length; i++) { + const subscription = explicitStreamSubscriptions[i]; const syncRuleStream: RequestedStream = { parameters: subscription.parameters ?? {}, - opaque_id: subscription.client_id + opaque_id: i }; if (Object.hasOwn(streamsByName, subscription.stream)) { streamsByName[subscription.stream].push(syncRuleStream); @@ -422,7 +422,7 @@ export class BucketParameterState { } } this.includeDefaultStreams = subscriptions?.include_defaults ?? true; - this.explicitStreamSubscriptions = idToStreamSubscription; + this.explicitStreamSubscriptions = explicitStreamSubscriptions; this.querier = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, @@ -469,7 +469,7 @@ export class BucketParameterState { const stream = description.definition; return { default: lookupIndex.get(stream)! }; } else { - return reason.subscription; + return { sub: reason.subscription }; } }) }; diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index dbf408fda..3b9d05d07 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -21,10 +21,6 @@ export const RequestedStreamSubscription = t.object({ * The defined name of the stream as it appears in sync stream definitions. */ stream: t.string, - /** - * An opaque textual identifier assigned to this request by the client. - */ - client_id: t.string, /** * An optional dictionary of parameters to pass to this specific stream. */ @@ -151,7 +147,16 @@ export type StreamingSyncLine = export type ProtocolOpId = string; export interface StreamDescription { + /** + * The name of the stream as it appears in the sync configuration. + */ name: string; + + /** + * Whether this stream is subscribed to by default. + * + * For default streams, this field is still `true` if clients have an explicit subscription to the stream. + */ is_default: boolean; } @@ -159,6 +164,18 @@ export interface Checkpoint { last_op_id: ProtocolOpId; write_checkpoint?: ProtocolOpId; buckets: CheckpointBucket[]; + + /** + * All streams that the client is subscribed to. + * + * This field has two purposes: + * + * 1. It allows clients to determine which of their subscriptions actually works. E.g. if a user does + * `db.syncStream('non_existent_stream').subscribe()`, clients don't immediately know that the stream doesn't + * exist. Only after the next `checkpoint` line can they query this field and mark unresolved subscriptions. + *. 2. It allows clients to learn which default streams they have been subscribed to. This is relevant for APIs + * listing all streams on the client-side. + */ streams: StreamDescription[]; } @@ -236,10 +253,13 @@ export type BucketDerivedFromDefaultStream = { /** * The bucket has been included in a checkpoint because it's part of a stream that a client has explicitly subscribed * to. - * - * The string is the client id associated with the subscription in {@link RequestedStreamSubscription}. */ -export type BucketDerivedFromExplicitSubscription = string; +export type BucketDerivedFromExplicitSubscription = { + /** + * The index (into {@link StreamSubscriptionRequest.subscriptions}) of the subscription yielding this bucket. + */ + sub: number; +}; export interface ClientBucketDescription { /** diff --git a/packages/sync-rules/src/BucketDescription.ts b/packages/sync-rules/src/BucketDescription.ts index eb9a3a628..8dd732f34 100644 --- a/packages/sync-rules/src/BucketDescription.ts +++ b/packages/sync-rules/src/BucketDescription.ts @@ -45,4 +45,4 @@ export interface ResolvedBucket extends BucketDescription { inclusion_reasons: BucketInclusionReason[]; } -export type BucketInclusionReason = 'default' | { subscription: string }; +export type BucketInclusionReason = 'default' | { subscription: number }; diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index cfaaa4741..a33404b9c 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -52,7 +52,7 @@ export interface RequestedStream { * An opaque id of the stream subscription, used to associate buckets with the stream subscriptions that have caused * them to be included. */ - opaque_id: string; + opaque_id: number; } export interface GetQuerierOptions { From 38f61957cf887cae2b8db93a72a672a10094752d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 5 Aug 2025 15:01:39 +0200 Subject: [PATCH 15/21] Update test snapshots --- .../__snapshots__/storage_sync.test.ts.snap | 183 ++++++++++++++++++ .../__snapshots__/storage_sync.test.ts.snap | 183 ++++++++++++++++++ .../src/tests/register-data-storage-tests.ts | 22 ++- .../test/src/sync/BucketChecksumState.test.ts | 42 ++-- tsconfig.json | 3 + 5 files changed, 409 insertions(+), 24 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index ff9bfdaea..7e6d77f19 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -10,9 +10,20 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = ` "checksum": -93886621, "count": 2, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "2", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -46,6 +57,11 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = ` "checksum": 499012468, "count": 4, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -105,9 +121,20 @@ exports[`sync - mongodb > expiring token 1`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -137,9 +164,20 @@ exports[`sync - mongodb > sends checkpoint complete line for empty checkpoint 1` "checksum": -1221282404, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "1", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -193,15 +231,35 @@ exports[`sync - mongodb > sync buckets in order 1`] = ` "checksum": 920318466, "count": 1, "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], }, { "bucket": "b1[]", "checksum": -1382098757, "count": 1, "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], }, ], "last_op_id": "2", + "streams": [ + { + "is_default": true, + "name": "b0", + }, + { + "is_default": true, + "name": "b1", + }, + ], "write_checkpoint": undefined, }, }, @@ -267,9 +325,20 @@ exports[`sync - mongodb > sync global data 1`] = ` "checksum": -93886621, "count": 2, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "2", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -319,21 +388,50 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint "checksum": -659831575, "count": 2000, "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], }, { "bucket": "b0b[]", "checksum": -659831575, "count": 2000, "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], }, { "bucket": "b1[]", "checksum": -1096116670, "count": 1, "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], }, ], "last_op_id": "4001", + "streams": [ + { + "is_default": true, + "name": "b0a", + }, + { + "is_default": true, + "name": "b0b", + }, + { + "is_default": true, + "name": "b1", + }, + ], "write_checkpoint": undefined, }, }, @@ -380,18 +478,33 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint "checksum": 883076828, "count": 2001, "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], }, { "bucket": "b0b[]", "checksum": 883076828, "count": 2001, "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], }, { "bucket": "b1[]", "checksum": 1841937527, "count": 2, "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], }, ], "write_checkpoint": undefined, @@ -466,9 +579,20 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = ` "checksum": -852817836, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "1", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -514,9 +638,20 @@ exports[`sync - mongodb > sync updates to data query only 1`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "1", + "streams": [ + { + "is_default": true, + "name": "by_user", + }, + ], "write_checkpoint": undefined, }, }, @@ -540,6 +675,11 @@ exports[`sync - mongodb > sync updates to data query only 2`] = ` "checksum": 1418351250, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -582,9 +722,20 @@ exports[`sync - mongodb > sync updates to global data 1`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -608,6 +759,11 @@ exports[`sync - mongodb > sync updates to global data 2`] = ` "checksum": 920318466, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -652,6 +808,11 @@ exports[`sync - mongodb > sync updates to global data 3`] = ` "checksum": -93886621, "count": 2, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -690,6 +851,12 @@ exports[`sync - mongodb > sync updates to parameter query + data 1`] = ` "checkpoint": { "buckets": [], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "by_user", + }, + ], "write_checkpoint": undefined, }, }, @@ -713,6 +880,11 @@ exports[`sync - mongodb > sync updates to parameter query + data 2`] = ` "checksum": 1418351250, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -751,6 +923,12 @@ exports[`sync - mongodb > sync updates to parameter query only 1`] = ` "checkpoint": { "buckets": [], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "by_user", + }, + ], "write_checkpoint": undefined, }, }, @@ -774,6 +952,11 @@ exports[`sync - mongodb > sync updates to parameter query only 2`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap index 54b14a8dc..c57d4f496 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -10,9 +10,20 @@ exports[`sync - postgres > compacting data - invalidate checkpoint 1`] = ` "checksum": -93886621, "count": 2, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "2", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -46,6 +57,11 @@ exports[`sync - postgres > compacting data - invalidate checkpoint 2`] = ` "checksum": 499012468, "count": 4, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -105,9 +121,20 @@ exports[`sync - postgres > expiring token 1`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -137,9 +164,20 @@ exports[`sync - postgres > sends checkpoint complete line for empty checkpoint 1 "checksum": -1221282404, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "1", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -193,15 +231,35 @@ exports[`sync - postgres > sync buckets in order 1`] = ` "checksum": 920318466, "count": 1, "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], }, { "bucket": "b1[]", "checksum": -1382098757, "count": 1, "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], }, ], "last_op_id": "2", + "streams": [ + { + "is_default": true, + "name": "b0", + }, + { + "is_default": true, + "name": "b1", + }, + ], "write_checkpoint": undefined, }, }, @@ -267,9 +325,20 @@ exports[`sync - postgres > sync global data 1`] = ` "checksum": -93886621, "count": 2, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "2", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -319,21 +388,50 @@ exports[`sync - postgres > sync interrupts low-priority buckets on new checkpoin "checksum": -659831575, "count": 2000, "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], }, { "bucket": "b0b[]", "checksum": -659831575, "count": 2000, "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], }, { "bucket": "b1[]", "checksum": -1096116670, "count": 1, "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], }, ], "last_op_id": "4001", + "streams": [ + { + "is_default": true, + "name": "b0a", + }, + { + "is_default": true, + "name": "b0b", + }, + { + "is_default": true, + "name": "b1", + }, + ], "write_checkpoint": undefined, }, }, @@ -380,18 +478,33 @@ exports[`sync - postgres > sync interrupts low-priority buckets on new checkpoin "checksum": 883076828, "count": 2001, "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], }, { "bucket": "b0b[]", "checksum": 883076828, "count": 2001, "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], }, { "bucket": "b1[]", "checksum": 1841937527, "count": 2, "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], }, ], "write_checkpoint": undefined, @@ -466,9 +579,20 @@ exports[`sync - postgres > sync legacy non-raw data 1`] = ` "checksum": -852817836, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "1", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -514,9 +638,20 @@ exports[`sync - postgres > sync updates to data query only 1`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "1", + "streams": [ + { + "is_default": true, + "name": "by_user", + }, + ], "write_checkpoint": undefined, }, }, @@ -540,6 +675,11 @@ exports[`sync - postgres > sync updates to data query only 2`] = ` "checksum": 1418351250, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -582,9 +722,20 @@ exports[`sync - postgres > sync updates to global data 1`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "mybucket", + }, + ], "write_checkpoint": undefined, }, }, @@ -608,6 +759,11 @@ exports[`sync - postgres > sync updates to global data 2`] = ` "checksum": 920318466, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -652,6 +808,11 @@ exports[`sync - postgres > sync updates to global data 3`] = ` "checksum": -93886621, "count": 2, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -690,6 +851,12 @@ exports[`sync - postgres > sync updates to parameter query + data 1`] = ` "checkpoint": { "buckets": [], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "by_user", + }, + ], "write_checkpoint": undefined, }, }, @@ -713,6 +880,11 @@ exports[`sync - postgres > sync updates to parameter query + data 2`] = ` "checksum": 1418351250, "count": 1, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, @@ -751,6 +923,12 @@ exports[`sync - postgres > sync updates to parameter query only 1`] = ` "checkpoint": { "buckets": [], "last_op_id": "0", + "streams": [ + { + "is_default": true, + "name": "by_user", + }, + ], "write_checkpoint": undefined, }, }, @@ -774,6 +952,11 @@ exports[`sync - postgres > sync updates to parameter query only 2`] = ` "checksum": 0, "count": 0, "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], }, ], "write_checkpoint": undefined, diff --git a/packages/service-core-tests/src/tests/register-data-storage-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-tests.ts index 1feb12426..f6c15f4b1 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-tests.ts @@ -424,10 +424,12 @@ bucket_definitions: .getBucketParameterQuerier(test_utils.querierOptions(parameters)) .queryDynamicBucketDescriptions({ getParameterSets(lookups) { - return bucketStorage.getParameterSets(lookups); + return checkpoint.getParameterSets(lookups); } }); - expect(buckets).toEqual([{ bucket: 'by_workspace["workspace1"]', priority: 3 }]); + expect(buckets).toEqual([ + { bucket: 'by_workspace["workspace1"]', priority: 3, definition: 'by_workspace', inclusion_reasons: ['default'] } + ]); }); test('save and load parameters with dynamic global buckets', async () => { @@ -498,13 +500,23 @@ bucket_definitions: .getBucketParameterQuerier(test_utils.querierOptions(parameters)) .queryDynamicBucketDescriptions({ getParameterSets(lookups) { - return bucketStorage.getParameterSets(lookups); + return checkpoint.getParameterSets(lookups); } }); buckets.sort((a, b) => a.bucket.localeCompare(b.bucket)); expect(buckets).toEqual([ - { bucket: 'by_public_workspace["workspace1"]', priority: 3 }, - { bucket: 'by_public_workspace["workspace3"]', priority: 3 } + { + bucket: 'by_public_workspace["workspace1"]', + priority: 3, + definition: 'by_public_workspace', + inclusion_reasons: ['default'] + }, + { + bucket: 'by_public_workspace["workspace3"]', + priority: 3, + definition: 'by_public_workspace', + inclusion_reasons: ['default'] + } ]); }); diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index 40a3f98c3..0e7c2d893 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -622,7 +622,7 @@ bucket_definitions: }); } - function createQuerier(ids: string[], subscription: string | null): BucketParameterQuerier { + function createQuerier(ids: string[], subscription: number | null): BucketParameterQuerier { return { staticBuckets: ids.map((bucket) => ({ definition: 'stream', @@ -686,7 +686,7 @@ bucket_definitions: const state = checksumState(); const line = await state.buildNextCheckpointLine({ - base: { checkpoint: 1n, lsn: '1' }, + base: storage.makeCheckpoint(1n), writeCheckpoint: null, update: CHECKPOINT_INVALIDATE_ALL })!; @@ -708,7 +708,7 @@ bucket_definitions: const state = checksumState({ syncRequest: { streams: { include_defaults: false, subscriptions: [] } } }); const line = await state.buildNextCheckpointLine({ - base: { checkpoint: 1n, lsn: '1' }, + base: storage.makeCheckpoint(1n), writeCheckpoint: null, update: CHECKPOINT_INVALIDATE_ALL })!; @@ -730,15 +730,15 @@ bucket_definitions: syncRequest: { streams: { subscriptions: [ - { stream: 'stream', client_id: '1', parameters: { ids: '["a"]' } }, - { stream: 'stream', client_id: '2', parameters: { ids: '["b"]' }, override_priority: 1 } + { stream: 'stream', parameters: { ids: '["a"]' } }, + { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 } ] } } }); const line = await state.buildNextCheckpointLine({ - base: { checkpoint: 1n, lsn: '1' }, + base: storage.makeCheckpoint(1n), writeCheckpoint: null, update: CHECKPOINT_INVALIDATE_ALL })!; @@ -746,8 +746,8 @@ bucket_definitions: expect(line?.checkpointLine).toEqual({ checkpoint: { buckets: [ - { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: ['1'] }, - { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: ['2'] }, + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] }, + { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 1 }] }, { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] } ], last_op_id: '1', @@ -762,15 +762,15 @@ bucket_definitions: syncRequest: { streams: { subscriptions: [ - { stream: 'stream', client_id: '1', parameters: { ids: '["a", "b"]' } }, - { stream: 'stream', client_id: '2', parameters: { ids: '["b"]' }, override_priority: 1 } + { stream: 'stream', parameters: { ids: '["a", "b"]' } }, + { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 } ] } } }); const line = await state.buildNextCheckpointLine({ - base: { checkpoint: 1n, lsn: '1' }, + base: storage.makeCheckpoint(1n), writeCheckpoint: null, update: CHECKPOINT_INVALIDATE_ALL })!; @@ -778,8 +778,8 @@ bucket_definitions: expect(line?.checkpointLine).toEqual({ checkpoint: { buckets: [ - { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: ['1'] }, - { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: ['1', '2'] } + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 3, subscriptions: [{ sub: 0 }] }, + { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }, { sub: 1 }] } ], last_op_id: '1', write_checkpoint: undefined, @@ -793,15 +793,13 @@ bucket_definitions: const state = checksumState({ syncRequest: { streams: { - subscriptions: [ - { stream: 'stream', client_id: '1', parameters: { ids: '["a", "default"]' }, override_priority: 1 } - ] + subscriptions: [{ stream: 'stream', parameters: { ids: '["a", "default"]' }, override_priority: 1 }] } } }); const line = await state.buildNextCheckpointLine({ - base: { checkpoint: 1n, lsn: '1' }, + base: storage.makeCheckpoint(1n), writeCheckpoint: null, update: CHECKPOINT_INVALIDATE_ALL })!; @@ -809,8 +807,14 @@ bucket_definitions: expect(line?.checkpointLine).toEqual({ checkpoint: { buckets: [ - { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: ['1'] }, - { bucket: 'stream["default"]', checksum: 1, count: 1, priority: 1, subscriptions: ['1', { default: 0 }] } + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] }, + { + bucket: 'stream["default"]', + checksum: 1, + count: 1, + priority: 1, + subscriptions: [{ sub: 0 }, { default: 0 }] + } ], last_op_id: '1', write_checkpoint: undefined, diff --git a/tsconfig.json b/tsconfig.json index e9d0017c5..73c0901d8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,9 @@ }, { "path": "./packages/service-core" + }, + { + "path": "./packages/service-core-tests" }, { "path": "./packages/service-errors" From 7fb3be1e43a452a5dd6c233df2a6c3c58842d424 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 6 Aug 2025 18:01:33 +0200 Subject: [PATCH 16/21] Include potential errors --- .../__snapshots__/storage_sync.test.ts.snap | 14 +++ .../__snapshots__/storage_sync.test.ts.snap | 14 +++ .../src/tests/register-data-storage-tests.ts | 16 +-- .../src/sync/BucketChecksumState.ts | 15 ++- .../service-core/src/util/protocol-types.ts | 16 +++ .../test/src/sync/BucketChecksumState.test.ts | 104 ++++++++++++++---- .../sync-rules/src/BucketParameterQuerier.ts | 18 +++ packages/sync-rules/src/BucketSource.ts | 6 +- .../sync-rules/src/SqlBucketDescriptor.ts | 10 +- packages/sync-rules/src/SqlSyncRules.ts | 17 ++- .../sync-rules/test/src/sync_rules.test.ts | 32 +++--- 11 files changed, 202 insertions(+), 60 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index 7e6d77f19..2780b3a21 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -20,6 +20,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = ` "last_op_id": "2", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -131,6 +132,7 @@ exports[`sync - mongodb > expiring token 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -174,6 +176,7 @@ exports[`sync - mongodb > sends checkpoint complete line for empty checkpoint 1` "last_op_id": "1", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -252,10 +255,12 @@ exports[`sync - mongodb > sync buckets in order 1`] = ` "last_op_id": "2", "streams": [ { + "errors": [], "is_default": true, "name": "b0", }, { + "errors": [], "is_default": true, "name": "b1", }, @@ -335,6 +340,7 @@ exports[`sync - mongodb > sync global data 1`] = ` "last_op_id": "2", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -420,14 +426,17 @@ exports[`sync - mongodb > sync interrupts low-priority buckets on new checkpoint "last_op_id": "4001", "streams": [ { + "errors": [], "is_default": true, "name": "b0a", }, { + "errors": [], "is_default": true, "name": "b0b", }, { + "errors": [], "is_default": true, "name": "b1", }, @@ -589,6 +598,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = ` "last_op_id": "1", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -648,6 +658,7 @@ exports[`sync - mongodb > sync updates to data query only 1`] = ` "last_op_id": "1", "streams": [ { + "errors": [], "is_default": true, "name": "by_user", }, @@ -732,6 +743,7 @@ exports[`sync - mongodb > sync updates to global data 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -853,6 +865,7 @@ exports[`sync - mongodb > sync updates to parameter query + data 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "by_user", }, @@ -925,6 +938,7 @@ exports[`sync - mongodb > sync updates to parameter query only 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "by_user", }, diff --git a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap index c57d4f496..dbbf0b515 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -20,6 +20,7 @@ exports[`sync - postgres > compacting data - invalidate checkpoint 1`] = ` "last_op_id": "2", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -131,6 +132,7 @@ exports[`sync - postgres > expiring token 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -174,6 +176,7 @@ exports[`sync - postgres > sends checkpoint complete line for empty checkpoint 1 "last_op_id": "1", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -252,10 +255,12 @@ exports[`sync - postgres > sync buckets in order 1`] = ` "last_op_id": "2", "streams": [ { + "errors": [], "is_default": true, "name": "b0", }, { + "errors": [], "is_default": true, "name": "b1", }, @@ -335,6 +340,7 @@ exports[`sync - postgres > sync global data 1`] = ` "last_op_id": "2", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -420,14 +426,17 @@ exports[`sync - postgres > sync interrupts low-priority buckets on new checkpoin "last_op_id": "4001", "streams": [ { + "errors": [], "is_default": true, "name": "b0a", }, { + "errors": [], "is_default": true, "name": "b0b", }, { + "errors": [], "is_default": true, "name": "b1", }, @@ -589,6 +598,7 @@ exports[`sync - postgres > sync legacy non-raw data 1`] = ` "last_op_id": "1", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -648,6 +658,7 @@ exports[`sync - postgres > sync updates to data query only 1`] = ` "last_op_id": "1", "streams": [ { + "errors": [], "is_default": true, "name": "by_user", }, @@ -732,6 +743,7 @@ exports[`sync - postgres > sync updates to global data 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "mybucket", }, @@ -853,6 +865,7 @@ exports[`sync - postgres > sync updates to parameter query + data 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "by_user", }, @@ -925,6 +938,7 @@ exports[`sync - postgres > sync updates to parameter query only 1`] = ` "last_op_id": "0", "streams": [ { + "errors": [], "is_default": true, "name": "by_user", }, diff --git a/packages/service-core-tests/src/tests/register-data-storage-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-tests.ts index f6c15f4b1..870677144 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-tests.ts @@ -422,7 +422,7 @@ bucket_definitions: const buckets = await sync_rules .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .queryDynamicBucketDescriptions({ + .querier.queryDynamicBucketDescriptions({ getParameterSets(lookups) { return checkpoint.getParameterSets(lookups); } @@ -498,7 +498,7 @@ bucket_definitions: const buckets = await sync_rules .getBucketParameterQuerier(test_utils.querierOptions(parameters)) - .queryDynamicBucketDescriptions({ + .querier.queryDynamicBucketDescriptions({ getParameterSets(lookups) { return checkpoint.getParameterSets(lookups); } @@ -608,11 +608,13 @@ bucket_definitions: // Test final values - the important part const buckets = ( - await sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).queryDynamicBucketDescriptions({ - getParameterSets(lookups) { - return checkpoint.getParameterSets(lookups); - } - }) + await sync_rules + .getBucketParameterQuerier(test_utils.querierOptions(parameters)) + .querier.queryDynamicBucketDescriptions({ + getParameterSets(lookups) { + return checkpoint.getParameterSets(lookups); + } + }) ).map((e) => e.bucket); buckets.sort(); expect(buckets).toEqual(['by_workspace["workspace1"]', 'by_workspace["workspace3"]']); diff --git a/packages/service-core/src/sync/BucketChecksumState.ts b/packages/service-core/src/sync/BucketChecksumState.ts index 25c6c2a92..2679e21f7 100644 --- a/packages/service-core/src/sync/BucketChecksumState.ts +++ b/packages/service-core/src/sync/BucketChecksumState.ts @@ -20,7 +20,7 @@ import { logger as defaultLogger } from '@powersync/lib-services-framework'; import { JSONBig } from '@powersync/service-jsonbig'; -import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js'; +import { BucketParameterQuerier, QuerierError } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js'; import { SyncContext } from './SyncContext.js'; import { getIntersection, hasIntersection } from './util.js'; @@ -254,7 +254,12 @@ export class BucketChecksumState { subscriptions.push({ name: source.name, - is_default: source.subscribedToByDefault + is_default: source.subscribedToByDefault, + errors: + this.parameterState.streamErrors[source.name]?.map((e) => ({ + subscription: e.subscription?.opaque_id ?? 'default', + message: e.message + })) ?? [] }); } } @@ -382,6 +387,8 @@ export class BucketParameterState { private readonly includeDefaultStreams: boolean; // Indexed by the client-side id private readonly explicitStreamSubscriptions: util.RequestedStreamSubscription[]; + // Indexed by descriptor name. + readonly streamErrors: Record; private readonly subscribedStreamNames: Set; private readonly logger: Logger; private cachedDynamicBuckets: ResolvedBucket[] | null = null; @@ -424,11 +431,13 @@ export class BucketParameterState { this.includeDefaultStreams = subscriptions?.include_defaults ?? true; this.explicitStreamSubscriptions = explicitStreamSubscriptions; - this.querier = syncRules.getBucketParameterQuerier({ + const { querier, errors } = syncRules.getBucketParameterQuerier({ globalParameters: this.syncParams, hasDefaultStreams: this.includeDefaultStreams, streams: streamsByName }); + this.querier = querier; + this.streamErrors = Object.groupBy(errors, (e) => e.descriptor) as Record; this.staticBuckets = new Map( mergeBuckets(this.querier.staticBuckets).map((b) => [b.bucket, b]) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 3b9d05d07..1f109329b 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -158,6 +158,22 @@ export interface StreamDescription { * For default streams, this field is still `true` if clients have an explicit subscription to the stream. */ is_default: boolean; + + /** + * If some subscriptions on this stream could not be resolved, e.g. due to an error, tis + */ + errors: StreamSubscriptionError[]; +} + +export interface StreamSubscriptionError { + /** + * The subscription that errored - either the default subscription or some of the explicit subscriptions. + */ + subscription: 'default' | number; + /** + * A message describing the error on the subscription. + */ + message: string; } export interface Checkpoint { diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index 0e7c2d893..a54ef6d00 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -92,7 +92,7 @@ bucket_definitions: buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'global', is_default: true }] + streams: [{ name: 'global', is_default: true, errors: [] }] } }); expect(line.bucketsToFetch).toEqual([ @@ -162,7 +162,7 @@ bucket_definitions: buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'global', is_default: true }] + streams: [{ name: 'global', is_default: true, errors: [] }] } }); expect(line.bucketsToFetch).toEqual([ @@ -202,7 +202,7 @@ bucket_definitions: ], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'global', is_default: true }] + streams: [{ name: 'global', is_default: true, errors: [] }] } }); expect(line.bucketsToFetch).toEqual([ @@ -270,7 +270,7 @@ bucket_definitions: buckets: [{ bucket: 'global[]', checksum: 1, count: 1, priority: 3, subscriptions: [{ default: 0 }] }], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'global', is_default: true }] + streams: [{ name: 'global', is_default: true, errors: [] }] } }); expect(line.bucketsToFetch).toEqual([ @@ -420,7 +420,7 @@ bucket_definitions: ], last_op_id: '3', write_checkpoint: undefined, - streams: [{ name: 'global', is_default: true }] + streams: [{ name: 'global', is_default: true, errors: [] }] } }); expect(line.bucketsToFetch).toEqual([ @@ -540,7 +540,8 @@ bucket_definitions: streams: [ { is_default: true, - name: 'by_project' + name: 'by_project', + errors: [] } ], write_checkpoint: undefined @@ -645,7 +646,7 @@ bucket_definitions: type: BucketSourceType.SYNC_STREAM, subscribedToByDefault: false, pushBucketParameterQueriers(result, options) { - // Create a fake querier that resolves teh global stream["default"] bucket by default and allows extracting + // Create a fake querier that resolves the global stream["default"] bucket by default and allows extracting // additional buckets from parameters. const subscriptions = options.streams['stream'] ?? []; if (!this.subscribedToByDefault && !subscriptions.length) { @@ -654,23 +655,31 @@ bucket_definitions: let hasExplicitDefaultSubscription = false; for (const subscription of subscriptions) { - let subscriptionParameters = []; - - if (subscription.parameters != null) { - subscriptionParameters = JSON.parse(subscription.parameters['ids'] as string).map( - (e: string) => `stream["${e}"]` - ); - } else { - hasExplicitDefaultSubscription = true; + try { + let subscriptionParameters = []; + + if (subscription.parameters != null) { + subscriptionParameters = JSON.parse(subscription.parameters['ids'] as string).map( + (e: string) => `stream["${e}"]` + ); + } else { + hasExplicitDefaultSubscription = true; + } + + result.queriers.push(createQuerier([...subscriptionParameters], subscription.opaque_id)); + } catch (e) { + result.errors.push({ + descriptor: 'stream', + subscription, + message: `Error evaluating bucket ids: ${e.message}` + }); } - - result.push(createQuerier([...subscriptionParameters], subscription.opaque_id)); } // If the stream is subscribed to by default and there is no explicit subscription that would match the default // subscription, also include the default querier. if (this.subscribedToByDefault && !hasExplicitDefaultSubscription) { - result.push(createQuerier(['stream["default"]'], null)); + result.queriers.push(createQuerier(['stream["default"]'], null)); } } } satisfies Partial as any; @@ -698,7 +707,7 @@ bucket_definitions: ], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'stream', is_default: true }] + streams: [{ name: 'stream', is_default: true, errors: [] }] } }); }); @@ -752,7 +761,7 @@ bucket_definitions: ], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'stream', is_default: true }] + streams: [{ name: 'stream', is_default: true, errors: [] }] } }); }); @@ -783,7 +792,7 @@ bucket_definitions: ], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'stream', is_default: false }] + streams: [{ name: 'stream', is_default: false, errors: [] }] } }); }); @@ -818,7 +827,58 @@ bucket_definitions: ], last_op_id: '1', write_checkpoint: undefined, - streams: [{ name: 'stream', is_default: true }] + streams: [{ name: 'stream', is_default: true, errors: [] }] + } + }); + }); + + test('reports errors', async () => { + source.subscribedToByDefault = true; + + const state = checksumState({ + syncRequest: { + streams: { + subscriptions: [ + { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: 1 }, + { stream: 'stream', parameters: { ids: 'invalid json' } } + ] + } + } + }); + + const line = await state.buildNextCheckpointLine({ + base: storage.makeCheckpoint(1n), + writeCheckpoint: null, + update: CHECKPOINT_INVALIDATE_ALL + })!; + line?.advance(); + expect(line?.checkpointLine).toEqual({ + checkpoint: { + buckets: [ + { bucket: 'stream["a"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] }, + { bucket: 'stream["b"]', checksum: 1, count: 1, priority: 1, subscriptions: [{ sub: 0 }] }, + { + bucket: 'stream["default"]', + checksum: 1, + count: 1, + priority: 3, + subscriptions: [{ default: 0 }] + } + ], + last_op_id: '1', + write_checkpoint: undefined, + streams: [ + { + name: 'stream', + is_default: true, + errors: [ + { + message: 'Error evaluating bucket ids: Unexpected token \'i\', "invalid json" is not valid JSON', + subscription: 1 + } + ] + } + ] } }); }); diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index 08a7fe115..8842c6d7c 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -1,4 +1,5 @@ import { BucketDescription, ResolvedBucket } from './BucketDescription.js'; +import { RequestedStream } from './SqlSyncRules.js'; import { RequestParameters, SqliteJsonRow, SqliteJsonValue } from './types.js'; import { normalizeParameterValue } from './utils.js'; @@ -39,6 +40,23 @@ export interface BucketParameterQuerier { queryDynamicBucketDescriptions(source: ParameterLookupSource): Promise; } +/** + * An error that occurred while trying to resolve the bucket ids a request should have access to. + * + * A common scenario that could cause this to happen is when parameters need to have a certain structure. For instance, + * `... WHERE id IN stream.parameters -> 'ids'` is unresolvable when `ids` is not set to a JSON array. + */ +export interface QuerierError { + descriptor: string; + subscription?: RequestedStream; + message: string; +} + +export interface PendingQueriers { + queriers: BucketParameterQuerier[]; + errors: QuerierError[]; +} + export interface ParameterLookupSource { getParameterSets: (lookups: ParameterLookup[]) => Promise; } diff --git a/packages/sync-rules/src/BucketSource.ts b/packages/sync-rules/src/BucketSource.ts index 586f86d2e..6131d78d7 100644 --- a/packages/sync-rules/src/BucketSource.ts +++ b/packages/sync-rules/src/BucketSource.ts @@ -1,4 +1,4 @@ -import { BucketParameterQuerier, ParameterLookup } from './BucketParameterQuerier.js'; +import { BucketParameterQuerier, ParameterLookup, PendingQueriers } from './BucketParameterQuerier.js'; import { ColumnDefinition } from './ExpressionType.js'; import { SourceTableInterface } from './SourceTableInterface.js'; import { GetQuerierOptions } from './SqlSyncRules.js'; @@ -39,10 +39,10 @@ export interface BucketSource { /** * Reports {@link BucketParameterQuerier}s resolving buckets that a specific stream request should have access to. * - * @param result The target array to insert queriers into. + * @param result The target array to insert queriers and errors into. * @param options Options, including parameters that may affect the buckets loaded by this source. */ - pushBucketParameterQueriers(result: BucketParameterQuerier[], options: GetQuerierOptions): void; + pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions): void; /** * Whether {@link pushBucketParameterQueriers} may include a querier where diff --git a/packages/sync-rules/src/SqlBucketDescriptor.ts b/packages/sync-rules/src/SqlBucketDescriptor.ts index 52fd9709f..66ae8440c 100644 --- a/packages/sync-rules/src/SqlBucketDescriptor.ts +++ b/packages/sync-rules/src/SqlBucketDescriptor.ts @@ -1,5 +1,5 @@ import { BucketInclusionReason, ResolvedBucket } from './BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers } from './BucketParameterQuerier.js'; +import { BucketParameterQuerier, mergeBucketParameterQueriers, PendingQueriers } from './BucketParameterQuerier.js'; import { BucketSource, BucketSourceType, ResultSetDescription } from './BucketSource.js'; import { ColumnDefinition } from './ExpressionType.js'; import { IdSequence } from './IdSequence.js'; @@ -119,12 +119,12 @@ export class SqlBucketDescriptor implements BucketSource { */ getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { const queriers: BucketParameterQuerier[] = []; - this.pushBucketParameterQueriers(queriers, options); + this.pushBucketParameterQueriers({ queriers, errors: [] }, options); return mergeBucketParameterQueriers(queriers); } - pushBucketParameterQueriers(result: BucketParameterQuerier[], options: GetQuerierOptions) { + pushBucketParameterQueriers(result: PendingQueriers, options: GetQuerierOptions) { const reasons = [this.bucketInclusionReason()]; const staticBuckets = this.getStaticBucketDescriptions(options.globalParameters, reasons); const staticQuerier = { @@ -133,7 +133,7 @@ export class SqlBucketDescriptor implements BucketSource { parameterQueryLookups: [], queryDynamicBucketDescriptions: async () => [] } satisfies BucketParameterQuerier; - result.push(staticQuerier); + result.queriers.push(staticQuerier); if (this.parameterQueries.length == 0) { return; @@ -142,7 +142,7 @@ export class SqlBucketDescriptor implements BucketSource { const dynamicQueriers = this.parameterQueries.map((query) => query.getBucketParameterQuerier(options.globalParameters, reasons) ); - result.push(...dynamicQueriers); + result.queriers.push(...dynamicQueriers); } getStaticBucketDescriptions(parameters: RequestParameters, reasons: BucketInclusionReason[]): ResolvedBucket[] { diff --git a/packages/sync-rules/src/SqlSyncRules.ts b/packages/sync-rules/src/SqlSyncRules.ts index a33404b9c..fdf9cc5cf 100644 --- a/packages/sync-rules/src/SqlSyncRules.ts +++ b/packages/sync-rules/src/SqlSyncRules.ts @@ -1,6 +1,6 @@ import { isScalar, LineCounter, parseDocument, Scalar, YAMLMap, YAMLSeq } from 'yaml'; import { isValidPriority } from './BucketDescription.js'; -import { BucketParameterQuerier, mergeBucketParameterQueriers } from './BucketParameterQuerier.js'; +import { BucketParameterQuerier, mergeBucketParameterQueriers, QuerierError } from './BucketParameterQuerier.js'; import { SqlRuleError, SyncRulesErrors, YamlError } from './errors.js'; import { SqlEventDescriptor } from './events/SqlEventDescriptor.js'; import { validateSyncRulesSchema } from './json_schema.js'; @@ -74,6 +74,11 @@ export interface GetQuerierOptions { streams: Record; } +export interface GetBucketParameterQuerierResult { + querier: BucketParameterQuerier; + errors: QuerierError[]; +} + export class SqlSyncRules implements SyncRules { bucketSources: BucketSource[] = []; eventDescriptors: SqlEventDescriptor[] = []; @@ -355,15 +360,19 @@ export class SqlSyncRules implements SyncRules { return { results, errors }; } - getBucketParameterQuerier(options: GetQuerierOptions): BucketParameterQuerier { + getBucketParameterQuerier(options: GetQuerierOptions): GetBucketParameterQuerierResult { const queriers: BucketParameterQuerier[] = []; + const errors: QuerierError[] = []; + const pending = { queriers, errors }; + for (const source of this.bucketSources) { if ((source.subscribedToByDefault && options.hasDefaultStreams) || source.name in options.streams) { - source.pushBucketParameterQueriers(queriers, options); + source.pushBucketParameterQueriers(pending, options); } } - return mergeBucketParameterQueriers(queriers); + const querier = mergeBucketParameterQueriers(queriers); + return { querier, errors }; } hasDynamicBucketQueries() { diff --git a/packages/sync-rules/test/src/sync_rules.test.ts b/packages/sync-rules/test/src/sync_rules.test.ts index 2456866ad..eb92f85a8 100644 --- a/packages/sync-rules/test/src/sync_rules.test.ts +++ b/packages/sync-rules/test/src/sync_rules.test.ts @@ -46,7 +46,7 @@ bucket_definitions: } ]); expect(rules.hasDynamicBucketQueries()).toBe(false); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); @@ -70,15 +70,15 @@ bucket_definitions: expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 1n }))).toEqual(1n); expect(param_query.filter!.lookupParameterValue(normalizeTokenParameters({ is_admin: 0n }))).toEqual(0n); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[]', priority: 3 }], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: false })).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [], hasDynamicBuckets: false }); @@ -123,7 +123,7 @@ bucket_definitions: const param_query = bucket.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id', 'device_id']); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })) + rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }, { device_id: 'device1' })).querier .staticBuckets ).toEqual([ { bucket: 'mybucket["user1","device1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } @@ -170,9 +170,9 @@ bucket_definitions: expect(bucket.bucketParameters).toEqual(['user_id']); const param_query = bucket.globalParameterQueries[0]; expect(param_query.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).staticBuckets).toEqual([ - { bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } - ]); + expect( + rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier.staticBuckets + ).toEqual([{ bucket: 'mybucket["user1"]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 }]); const data_query = bucket.dataQueries[0]; expect(data_query.bucketParameters).toEqual(['user_id']); @@ -312,7 +312,7 @@ bucket_definitions: ); const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); @@ -349,7 +349,7 @@ bucket_definitions: ); const bucket = rules.bucketSources[0] as SqlBucketDescriptor; expect(bucket.bucketParameters).toEqual(['user_id']); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["USER1"]', priority: 3 }], hasDynamicBuckets: false }); @@ -514,7 +514,7 @@ bucket_definitions: } ]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).staticBuckets).toEqual([ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ is_admin: true })).querier.staticBuckets).toEqual([ { bucket: 'mybucket[1]', definition: 'mybucket', inclusion_reasons: ['default'], priority: 3 } ]); }); @@ -557,7 +557,7 @@ bucket_definitions: PARSE_OPTIONS ); expect( - rules.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })) + rules.getBucketParameterQuerier(normalizeQuerierOptions({ int1: 314, float1: 3.14, float2: 314 })).querier ).toMatchObject({ staticBuckets: [{ bucket: 'mybucket[314,3.14,314]', priority: 3 }] }); expect( @@ -585,7 +585,7 @@ bucket_definitions: PARSE_OPTIONS ); expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'test' })).querier).toMatchObject({ staticBuckets: [{ bucket: 'mybucket["TEST"]', priority: 3 }], hasDynamicBuckets: false }); @@ -838,7 +838,7 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -863,7 +863,7 @@ bucket_definitions: expect(rules.errors).toEqual([]); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({}))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({})).querier).toMatchObject({ staticBuckets: [ { bucket: 'highprio[]', priority: 0 }, { bucket: 'defaultprio[]', priority: 3 } @@ -928,7 +928,7 @@ bucket_definitions: expect(bucket.bucketParameters).toEqual(['user_id']); expect(rules.hasDynamicBucketQueries()).toBe(true); - expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' }))).toMatchObject({ + expect(rules.getBucketParameterQuerier(normalizeQuerierOptions({ user_id: 'user1' })).querier).toMatchObject({ hasDynamicBuckets: true, parameterQueryLookups: [ ParameterLookup.normalized('mybucket', '2', ['user1']), From 7a280bac3ee8121612930fb904ded877f5eaa1ef Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 6 Aug 2025 21:31:07 +0200 Subject: [PATCH 17/21] Remove core tests again --- tsconfig.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/tsconfig.json b/tsconfig.json index 73c0901d8..e9d0017c5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,9 +12,6 @@ }, { "path": "./packages/service-core" - }, - { - "path": "./packages/service-core-tests" }, { "path": "./packages/service-errors" From aa21126ab9a51a6d6d91aec89d03db287fb5fc9d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 7 Aug 2025 14:55:12 +0200 Subject: [PATCH 18/21] Complete docs sentence --- packages/service-core/src/util/protocol-types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 1f109329b..3f94ff57b 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -160,7 +160,8 @@ export interface StreamDescription { is_default: boolean; /** - * If some subscriptions on this stream could not be resolved, e.g. due to an error, tis + * If some subscriptions on this stream could not be resolved, e.g. due to an error, this array contains the faulty + * subscriptions along with an error message. */ errors: StreamSubscriptionError[]; } From d2be18420e7e57ab5e52bc7a1136aea9a3e5e0a8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 11 Aug 2025 14:41:55 +0200 Subject: [PATCH 19/21] Add changeset --- .changeset/great-rabbits-exercise.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/great-rabbits-exercise.md diff --git a/.changeset/great-rabbits-exercise.md b/.changeset/great-rabbits-exercise.md new file mode 100644 index 000000000..fa448e3d4 --- /dev/null +++ b/.changeset/great-rabbits-exercise.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-sync-rules': minor +'@powersync/service-core': patch +'@powersync/service-image': patch +--- + +Refactor interface between service and sync rule bindings in preparation for sync streams. From cc1adcd1868a395fc4b731bb27025a023fd4ff37 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 13 Aug 2025 11:55:38 +0200 Subject: [PATCH 20/21] Fix definition of override_priority --- packages/service-core/src/util/protocol-types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-core/src/util/protocol-types.ts b/packages/service-core/src/util/protocol-types.ts index 3f94ff57b..6edafbae4 100644 --- a/packages/service-core/src/util/protocol-types.ts +++ b/packages/service-core/src/util/protocol-types.ts @@ -31,7 +31,7 @@ export const RequestedStreamSubscription = t.object({ * Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be * useful when the priority for partial syncs depends on e.g. the current page opened in a client. */ - override_priority: t.number.optional() + override_priority: t.union(t.number, t.Null) }); export type RequestedStreamSubscription = t.Decoded; From c77b754f8f73faabe9537657f3f084ea20e86282 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 13 Aug 2025 15:11:46 +0200 Subject: [PATCH 21/21] Fix tests --- .../service-core/test/src/sync/BucketChecksumState.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/service-core/test/src/sync/BucketChecksumState.test.ts b/packages/service-core/test/src/sync/BucketChecksumState.test.ts index a54ef6d00..df43cc9bd 100644 --- a/packages/service-core/test/src/sync/BucketChecksumState.test.ts +++ b/packages/service-core/test/src/sync/BucketChecksumState.test.ts @@ -739,7 +739,7 @@ bucket_definitions: syncRequest: { streams: { subscriptions: [ - { stream: 'stream', parameters: { ids: '["a"]' } }, + { stream: 'stream', parameters: { ids: '["a"]' }, override_priority: null }, { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 } ] } @@ -771,7 +771,7 @@ bucket_definitions: syncRequest: { streams: { subscriptions: [ - { stream: 'stream', parameters: { ids: '["a", "b"]' } }, + { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: null }, { stream: 'stream', parameters: { ids: '["b"]' }, override_priority: 1 } ] } @@ -840,7 +840,7 @@ bucket_definitions: streams: { subscriptions: [ { stream: 'stream', parameters: { ids: '["a", "b"]' }, override_priority: 1 }, - { stream: 'stream', parameters: { ids: 'invalid json' } } + { stream: 'stream', parameters: { ids: 'invalid json' }, override_priority: null } ] } }