From ceba051e0419318eaa71cd5f9fcf4c56815f5aaa Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Wed, 9 Aug 2023 09:55:21 +0100 Subject: [PATCH 1/9] initial sync --- docs/manifest.json | 6 ++ package-lock.json | 115 +++++++++++++++++++++++++++ package.json | 1 + packages/core-data/package.json | 1 + packages/core-data/src/entities.js | 34 ++++++++ packages/core-data/src/resolvers.js | 106 ++++++++++++++---------- packages/core-data/src/sync.js | 13 +++ packages/core-data/tsconfig.json | 1 + packages/sync/.npmrc | 1 + packages/sync/CHANGELOG.md | 3 + packages/sync/README.md | 52 ++++++++++++ packages/sync/package.json | 37 +++++++++ packages/sync/src/connect-indexdb.js | 31 ++++++++ packages/sync/src/index.js | 2 + packages/sync/src/provider.js | 102 ++++++++++++++++++++++++ packages/sync/src/types.ts | 26 ++++++ packages/sync/tsconfig.json | 8 ++ tsconfig.json | 1 + 18 files changed, 499 insertions(+), 41 deletions(-) create mode 100644 packages/core-data/src/sync.js create mode 100644 packages/sync/.npmrc create mode 100644 packages/sync/CHANGELOG.md create mode 100644 packages/sync/README.md create mode 100644 packages/sync/package.json create mode 100644 packages/sync/src/connect-indexdb.js create mode 100644 packages/sync/src/index.js create mode 100644 packages/sync/src/provider.js create mode 100644 packages/sync/src/types.ts create mode 100644 packages/sync/tsconfig.json diff --git a/docs/manifest.json b/docs/manifest.json index 578d4762224763..a806da3edb9aee 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -1889,6 +1889,12 @@ "markdown_source": "../packages/stylelint-config/README.md", "parent": "packages" }, + { + "title": "@wordpress/sync", + "slug": "packages-sync", + "markdown_source": "../packages/sync/README.md", + "parent": "packages" + }, { "title": "@wordpress/token-list", "slug": "packages-token-list", diff --git a/package-lock.json b/package-lock.json index ad5e4cec635122..9ebdcecd97b9af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,6 +71,7 @@ "@wordpress/server-side-render": "file:packages/server-side-render", "@wordpress/shortcode": "file:packages/shortcode", "@wordpress/style-engine": "file:packages/style-engine", + "@wordpress/sync": "file:packages/sync", "@wordpress/token-list": "file:packages/token-list", "@wordpress/url": "file:packages/url", "@wordpress/viewport": "file:packages/viewport", @@ -19161,6 +19162,10 @@ "resolved": "packages/stylelint-config", "link": true }, + "node_modules/@wordpress/sync": { + "resolved": "packages/sync", + "link": true + }, "node_modules/@wordpress/token-list": { "resolved": "packages/token-list", "link": true @@ -38970,6 +38975,15 @@ "unfetch": "^4.2.0" } }, + "node_modules/isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -41019,6 +41033,25 @@ "node": ">= 0.8.0" } }, + "node_modules/lib0": { + "version": "0.2.79", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.79.tgz", + "integrity": "sha512-fIdPbxzMVq10wt3ou1lp3/f9n5ciHZ6t+P1vyGy3XXr018AntTYM4eg24sNFcNq8SYDQwmhhoGdS58IlYBzfBw==", + "dependencies": { + "isomorphic.js": "^0.2.4" + }, + "bin": { + "0gentesthtml": "bin/gentesthtml.js", + "0serve": "bin/0serve.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/libnpmaccess": { "version": "6.0.4", "resolved": "https://registry.npmjs.org/libnpmaccess/-/libnpmaccess-6.0.4.tgz", @@ -59825,6 +59858,21 @@ "node": ">=0.4" } }, + "node_modules/y-indexeddb": { + "version": "9.0.11", + "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.11.tgz", + "integrity": "sha512-HOKQ70qW1h2WJGtOKu9rE8fbX86ExVZedecndMuhwax3yM4DQsQzCTGHt/jvTrFZr/9Ahvd8neD6aZ4dMMjtdg==", + "dependencies": { + "lib0": "^0.2.74" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "yjs": "^13.0.0" + } + }, "node_modules/y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", @@ -59947,6 +59995,22 @@ "fd-slicer": "~1.1.0" } }, + "node_modules/yjs": { + "version": "13.6.7", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.7.tgz", + "integrity": "sha512-mCZTh4kjvUS2DnaktsYN6wLH3WZCJBLqrTdkWh1bIDpA/sB/GNFaLA/dyVJj2Hc7KwONuuoC/vWe9bwBBosZLQ==", + "dependencies": { + "lib0": "^0.2.74" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -60522,6 +60586,7 @@ "@wordpress/i18n": "file:../i18n", "@wordpress/is-shallow-equal": "file:../is-shallow-equal", "@wordpress/private-apis": "file:../private-apis", + "@wordpress/sync": "file:../sync", "@wordpress/url": "file:../url", "change-case": "^4.1.2", "equivalent-key-map": "^0.2.2", @@ -62015,6 +62080,18 @@ "stylelint": "^14.2" } }, + "packages/sync": { + "version": "0.1.0", + "license": "GPL-2.0-or-later", + "dependencies": { + "@babel/runtime": "^7.16.0", + "y-indexeddb": "~9.0.11", + "yjs": "~13.6.6" + }, + "engines": { + "node": ">=12" + } + }, "packages/token-list": { "name": "@wordpress/token-list", "version": "2.38.0", @@ -76449,6 +76526,7 @@ "@wordpress/i18n": "file:../i18n", "@wordpress/is-shallow-equal": "file:../is-shallow-equal", "@wordpress/private-apis": "file:../private-apis", + "@wordpress/sync": "file:../sync", "@wordpress/url": "file:../url", "change-case": "^4.1.2", "equivalent-key-map": "^0.2.2", @@ -77418,6 +77496,14 @@ "stylelint-config-recommended-scss": "^5.0.2" } }, + "@wordpress/sync": { + "version": "file:packages/sync", + "requires": { + "@babel/runtime": "^7.16.0", + "y-indexeddb": "~9.0.11", + "yjs": "~13.6.6" + } + }, "@wordpress/token-list": { "version": "file:packages/token-list", "requires": { @@ -94272,6 +94358,11 @@ "unfetch": "^4.2.0" } }, + "isomorphic.js": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", + "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -95826,6 +95917,14 @@ "type-check": "~0.3.2" } }, + "lib0": { + "version": "0.2.79", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.79.tgz", + "integrity": "sha512-fIdPbxzMVq10wt3ou1lp3/f9n5ciHZ6t+P1vyGy3XXr018AntTYM4eg24sNFcNq8SYDQwmhhoGdS58IlYBzfBw==", + "requires": { + "isomorphic.js": "^0.2.4" + } + }, "libnpmaccess": { "version": "6.0.4", "resolved": "https://registry.npmjs.org/libnpmaccess/-/libnpmaccess-6.0.4.tgz", @@ -110293,6 +110392,14 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, + "y-indexeddb": { + "version": "9.0.11", + "resolved": "https://registry.npmjs.org/y-indexeddb/-/y-indexeddb-9.0.11.tgz", + "integrity": "sha512-HOKQ70qW1h2WJGtOKu9rE8fbX86ExVZedecndMuhwax3yM4DQsQzCTGHt/jvTrFZr/9Ahvd8neD6aZ4dMMjtdg==", + "requires": { + "lib0": "^0.2.74" + } + }, "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", @@ -110393,6 +110500,14 @@ "fd-slicer": "~1.1.0" } }, + "yjs": { + "version": "13.6.7", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.7.tgz", + "integrity": "sha512-mCZTh4kjvUS2DnaktsYN6wLH3WZCJBLqrTdkWh1bIDpA/sB/GNFaLA/dyVJj2Hc7KwONuuoC/vWe9bwBBosZLQ==", + "requires": { + "lib0": "^0.2.74" + } + }, "yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 55e740059f206b..ec9691e6f07cd1 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "@wordpress/server-side-render": "file:packages/server-side-render", "@wordpress/shortcode": "file:packages/shortcode", "@wordpress/style-engine": "file:packages/style-engine", + "@wordpress/sync": "file:packages/sync", "@wordpress/token-list": "file:packages/token-list", "@wordpress/url": "file:packages/url", "@wordpress/viewport": "file:packages/viewport", diff --git a/packages/core-data/package.json b/packages/core-data/package.json index 5212ccbf1021a0..b01cc8ae53923f 100644 --- a/packages/core-data/package.json +++ b/packages/core-data/package.json @@ -42,6 +42,7 @@ "@wordpress/i18n": "file:../i18n", "@wordpress/is-shallow-equal": "file:../is-shallow-equal", "@wordpress/private-apis": "file:../private-apis", + "@wordpress/sync": "file:../sync", "@wordpress/url": "file:../url", "change-case": "^4.1.2", "equivalent-key-map": "^0.2.2", diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 3b8a443bcf1e39..90e95a87cd7946 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -13,6 +13,7 @@ import { __ } from '@wordpress/i18n'; * Internal dependencies */ import { addEntities } from './actions'; +import { getSyncProvider } from './sync'; export const DEFAULT_ENTITY_KEY = 'id'; @@ -37,6 +38,31 @@ export const rootEntitiesConfig = [ 'url', ].join( ',' ), }, + syncConfig: { + fetch: async () => { + await new Promise( ( resolve ) => setTimeout( resolve, 5000 ) ); + return apiFetch( { path: '/' } ); + }, + applyChangesToDoc: ( doc, changes ) => { + const document = doc.getMap( 'document' ); + [ 'name', 'description' ].forEach( ( key ) => { + if ( document.get( key ) !== changes[ key ] ) { + document.set( key, changes[ key ] ); + } + } ); + /*Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( document.get( key ) !== value ) { + document.set( key, value ); + } + } );*/ + }, + fromCRDTDoc: ( doc ) => { + return doc.getMap( 'document' ).toJSON(); + }, + handleChanges: () => {}, + }, + syncObjectType: 'root/base', + getSyncObjectId: () => 'index', }, { label: __( 'Site' ), @@ -299,6 +325,12 @@ export const getMethodName = ( return `${ prefix }${ kindPrefix }${ suffix }`; }; +function registerSyncConfigs( configs ) { + configs.forEach( ( { syncObjectType, syncConfig } ) => { + getSyncProvider().register( syncObjectType, syncConfig ); + } ); +} + /** * Loads the kind entities into the store. * @@ -311,6 +343,7 @@ export const getOrLoadEntitiesConfig = async ( { select, dispatch } ) => { let configs = select.getEntitiesConfig( kind ); if ( configs && configs.length !== 0 ) { + registerSyncConfigs( configs ); return configs; } @@ -322,6 +355,7 @@ export const getOrLoadEntitiesConfig = } configs = await loader.loadEntities(); + registerSyncConfigs( configs ); dispatch( addEntities( configs ) ); return configs; diff --git a/packages/core-data/src/resolvers.js b/packages/core-data/src/resolvers.js index 93dcfd6ee9014f..d85663199a1360 100644 --- a/packages/core-data/src/resolvers.js +++ b/packages/core-data/src/resolvers.js @@ -15,6 +15,7 @@ import apiFetch from '@wordpress/api-fetch'; import { STORE_NAME } from './name'; import { getOrLoadEntitiesConfig, DEFAULT_ENTITY_KEY } from './entities'; import { forwardResolver, getNormalizedCommaSeparable } from './utils'; +import { getSyncProvider } from './sync'; /** * Requests authors from the REST API. @@ -71,51 +72,74 @@ export const getEntityRecord = ); try { - if ( query !== undefined && query._fields ) { - // If requesting specific fields, items and query association to said - // records are stored by ID reference. Thus, fields must always include - // the ID. - query = { - ...query, - _fields: [ - ...new Set( [ - ...( getNormalizedCommaSeparable( query._fields ) || - [] ), - entityConfig.key || DEFAULT_ENTITY_KEY, - ] ), - ].join(), - }; - } - - // Disable reason: While true that an early return could leave `path` - // unused, it's important that path is derived using the query prior to - // additional query modifications in the condition below, since those - // modifications are relevant to how the data is tracked in state, and not - // for how the request is made to the REST API. - - // eslint-disable-next-line @wordpress/no-unused-vars-before-return - const path = addQueryArgs( - entityConfig.baseURL + ( key ? '/' + key : '' ), - { - ...entityConfig.baseURLParams, - ...query, + // Entity supports configs, + // use the sync algorithm instead of the old fetch behavior. + if ( entityConfig.syncConfig && ! query ) { + const objectId = entityConfig.getSyncObjectId( key ); + await getSyncProvider().bootstrap( + entityConfig.syncObjectType, + objectId, + ( record ) => { + dispatch.receiveEntityRecords( + kind, + name, + record, + query + ); + } + ); + } else { + if ( query !== undefined && query._fields ) { + // If requesting specific fields, items and query association to said + // records are stored by ID reference. Thus, fields must always include + // the ID. + query = { + ...query, + _fields: [ + ...new Set( [ + ...( getNormalizedCommaSeparable( + query._fields + ) || [] ), + entityConfig.key || DEFAULT_ENTITY_KEY, + ] ), + ].join(), + }; } - ); - - if ( query !== undefined ) { - query = { ...query, include: [ key ] }; - // The resolution cache won't consider query as reusable based on the - // fields, so it's tested here, prior to initiating the REST request, - // and without causing `getEntityRecords` resolution to occur. - const hasRecords = select.hasEntityRecords( kind, name, query ); - if ( hasRecords ) { - return; + // Disable reason: While true that an early return could leave `path` + // unused, it's important that path is derived using the query prior to + // additional query modifications in the condition below, since those + // modifications are relevant to how the data is tracked in state, and not + // for how the request is made to the REST API. + + // eslint-disable-next-line @wordpress/no-unused-vars-before-return + const path = addQueryArgs( + entityConfig.baseURL + ( key ? '/' + key : '' ), + { + ...entityConfig.baseURLParams, + ...query, + } + ); + + if ( query !== undefined ) { + query = { ...query, include: [ key ] }; + + // The resolution cache won't consider query as reusable based on the + // fields, so it's tested here, prior to initiating the REST request, + // and without causing `getEntityRecords` resolution to occur. + const hasRecords = select.hasEntityRecords( + kind, + name, + query + ); + if ( hasRecords ) { + return; + } } - } - const record = await apiFetch( { path } ); - dispatch.receiveEntityRecords( kind, name, record, query ); + const record = await apiFetch( { path } ); + dispatch.receiveEntityRecords( kind, name, record, query ); + } } finally { dispatch.__unstableReleaseStoreLock( lock ); } diff --git a/packages/core-data/src/sync.js b/packages/core-data/src/sync.js new file mode 100644 index 00000000000000..e02d524abc5da7 --- /dev/null +++ b/packages/core-data/src/sync.js @@ -0,0 +1,13 @@ +/** + * WordPress dependencies + */ +import { createSyncProvider, connectIndexDb } from '@wordpress/sync'; + +let syncProvider; +export function getSyncProvider() { + if ( ! syncProvider ) { + syncProvider = createSyncProvider( connectIndexDb ); + } + + return syncProvider; +} diff --git a/packages/core-data/tsconfig.json b/packages/core-data/tsconfig.json index 3ba9daad756019..031d697f8dbe6b 100644 --- a/packages/core-data/tsconfig.json +++ b/packages/core-data/tsconfig.json @@ -18,6 +18,7 @@ { "path": "../i18n" }, { "path": "../is-shallow-equal" }, { "path": "../private-apis" }, + { "path": "../sync" }, { "path": "../url" } ], "include": [ "src/**/*" ] diff --git a/packages/sync/.npmrc b/packages/sync/.npmrc new file mode 100644 index 00000000000000..43c97e719a5a82 --- /dev/null +++ b/packages/sync/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/packages/sync/CHANGELOG.md b/packages/sync/CHANGELOG.md new file mode 100644 index 00000000000000..6ed52df1077824 --- /dev/null +++ b/packages/sync/CHANGELOG.md @@ -0,0 +1,3 @@ + + +## Unreleased diff --git a/packages/sync/README.md b/packages/sync/README.md new file mode 100644 index 00000000000000..7f661921c8ac09 --- /dev/null +++ b/packages/sync/README.md @@ -0,0 +1,52 @@ +# Sync + +Sync data between frontend and backend. + +## Installation + +Install the module + +```bash +npm install @wordpress/sync --save +``` + +## API + + + +### connectIndexDb + +Connect function to the IndexedDB persistence provider. + +_Parameters_ + +- _objectId_ `ObjectID`: The object ID. +- _objectType_ `ObjectType`: The object type. +- _doc_ `CRDTDoc`: The CRDT document. + +_Returns_ + +- `Promise<() => void>`: Promise that resolves when the connection is established. + +### createSyncProvider + +Create a sync provider. + +_Parameters_ + +- _connectLocal_ `ConnectDoc`: Connect the document to a local database. +- _connectRemote_ `ConnectDoc`: Connect the document to a remote sync connection. + +_Returns_ + +- `SyncProvider`: Sync provider. + + + +## Contributing to this package + +This is an individual package that's part of the Gutenberg project. The project is organized as a monorepo. It's made up of multiple self-contained software packages, each with a specific purpose. The packages in this monorepo are published to [npm](https://www.npmjs.com/) and used by [WordPress](https://make.wordpress.org/core/) as well as other software projects. + +To find out more about contributing to this package or Gutenberg as a whole, please read the project's main [contributor guide](https://github.com/WordPress/gutenberg/tree/HEAD/CONTRIBUTING.md). + +

Code is Poetry.

diff --git a/packages/sync/package.json b/packages/sync/package.json new file mode 100644 index 00000000000000..268eceddaa1d2c --- /dev/null +++ b/packages/sync/package.json @@ -0,0 +1,37 @@ +{ + "name": "@wordpress/sync", + "version": "0.1.0", + "description": "Sync Data.", + "author": "The WordPress Contributors", + "license": "GPL-2.0-or-later", + "keywords": [ + "wordpress", + "gutenberg", + "sync" + ], + "homepage": "https://github.com/WordPress/gutenberg/tree/HEAD/packages/sync/README.md", + "repository": { + "type": "git", + "url": "https://github.com/WordPress/gutenberg.git", + "directory": "packages/sync" + }, + "bugs": { + "url": "https://github.com/WordPress/gutenberg/issues" + }, + "engines": { + "node": ">=12" + }, + "main": "build/index.js", + "module": "build-module/index.js", + "react-native": "src/index", + "types": "build-types", + "sideEffects": false, + "dependencies": { + "@babel/runtime": "^7.16.0", + "y-indexeddb": "~9.0.11", + "yjs": "~13.6.6" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/sync/src/connect-indexdb.js b/packages/sync/src/connect-indexdb.js new file mode 100644 index 00000000000000..ee56a463fd9956 --- /dev/null +++ b/packages/sync/src/connect-indexdb.js @@ -0,0 +1,31 @@ +/** + * External dependencies + */ +// @ts-ignore +import { IndexeddbPersistence } from 'y-indexeddb'; + +/** @typedef {import('./types').ObjectType} ObjectType */ +/** @typedef {import('./types').ObjectID} ObjectID */ +/** @typedef {import('./types').CRDTDoc} CRDTDoc */ +/** @typedef {import('./types').ConnectDoc} ConnectDoc */ +/** @typedef {import('./types').SyncProvider} SyncProvider */ + +/** + * Connect function to the IndexedDB persistence provider. + * + * @param {ObjectID} objectId The object ID. + * @param {ObjectType} objectType The object type. + * @param {CRDTDoc} doc The CRDT document. + * + * @return {Promise<() => void>} Promise that resolves when the connection is established. + */ +export function connectIndexDb( objectId, objectType, doc ) { + const docName = `${ objectType }-${ objectId }`; + const provider = new IndexeddbPersistence( docName, doc ); + + return new Promise( ( resolve ) => { + provider.on( 'synced', () => { + resolve( () => provider.destroy() ); + } ); + } ); +} diff --git a/packages/sync/src/index.js b/packages/sync/src/index.js new file mode 100644 index 00000000000000..8a876f85552877 --- /dev/null +++ b/packages/sync/src/index.js @@ -0,0 +1,2 @@ +export { connectIndexDb } from './connect-indexdb'; +export { createSyncProvider } from './provider'; diff --git a/packages/sync/src/provider.js b/packages/sync/src/provider.js new file mode 100644 index 00000000000000..39d896f511d887 --- /dev/null +++ b/packages/sync/src/provider.js @@ -0,0 +1,102 @@ +/** + * External dependencies + */ +// @ts-ignore +import * as Y from 'yjs'; + +/** @typedef {import('./types').ObjectType} ObjectType */ +/** @typedef {import('./types').ObjectID} ObjectID */ +/** @typedef {import('./types').ObjectConfig} ObjectConfig */ +/** @typedef {import('./types').ConnectDoc} ConnectDoc */ +/** @typedef {import('./types').SyncProvider} SyncProvider */ + +/** + * Create a sync provider. + * + * @param {ConnectDoc} connectLocal Connect the document to a local database. + * @param {ConnectDoc} connectRemote Connect the document to a remote sync connection. + * @return {SyncProvider} Sync provider. + */ +export const createSyncProvider = ( connectLocal, connectRemote ) => { + /** + * @type {Record} + */ + const config = {}; + + /** + * @type {Recordvoid>>} + */ + const listeners = {}; + + /** + * Registeres an object type. + * + * @param {ObjectType} objectType Object type to register. + * @param {ObjectConfig} objectConfig Object config. + */ + function register( objectType, objectConfig ) { + config[ objectType ] = objectConfig; + } + + /** + * Fetch data from local database or remote source. + * + * @param {ObjectType} objectType Object type to load. + * @param {ObjectID} objectId Object ID to load. + * @param {Function} handleChanges Callback to call when data changes. + */ + async function bootstrap( objectType, objectId, handleChanges ) { + const doc = new Y.Doc(); + + const update = () => { + const data = config[ objectType ].fromCRDTDoc( doc ); + handleChanges( data ); + }; + doc.on( 'update', update ); + + // connect to locally saved database. + const destroyLocalConnection = await connectLocal( + objectId, + objectType, + doc + ); + + // Once the database syncing is done, start the remote syncing + if ( connectRemote ) { + await connectRemote( objectId, objectType, doc ); + } + + const loadRemotely = config[ objectType ].fetch; + if ( loadRemotely ) { + loadRemotely( objectId ).then( ( data ) => { + doc.transact( () => { + config[ objectType ].applyChangesToDoc( doc, data ); + } ); + } ); + } + + listeners[ objectType ] = listeners[ objectType ] || {}; + listeners[ objectType ][ objectId ] = () => { + destroyLocalConnection(); + doc.off( 'update', update ); + }; + } + + /** + * Stop updating a document and discard it. + * + * @param {ObjectType} objectType Object type to load. + * @param {ObjectID} objectId Object ID to load. + */ + async function discard( objectType, objectId ) { + if ( listeners?.[ objectType ]?.[ objectId ] ) { + listeners[ objectType ][ objectId ](); + } + } + + return { + register, + bootstrap, + discard, + }; +}; diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts new file mode 100644 index 00000000000000..9ceb5d2e0e33bf --- /dev/null +++ b/packages/sync/src/types.ts @@ -0,0 +1,26 @@ +export type ObjectID = string; +export type ObjectType = string; +export type ObjectData = any; +export type CRDTDoc = any; + +export type ObjectConfig = { + fetch: ( id: ObjectID ) => Promise< ObjectData >; + applyChangesToDoc: ( doc: CRDTDoc, data: any ) => void; + fromCRDTDoc: ( doc: CRDTDoc ) => any; +}; + +export type ConnectDoc = ( + id: ObjectID, + type: ObjectType, + doc: CRDTDoc +) => Promise< () => void >; + +export type SyncProvider = { + register: ( type: ObjectType, config: ObjectConfig ) => void; + bootstrap: ( + type: ObjectType, + id: ObjectID, + handleChanges: ( data: any ) => void + ) => Promise< CRDTDoc >; + discard: ( type: ObjectType, id: ObjectID ) => Promise< CRDTDoc >; +}; diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json new file mode 100644 index 00000000000000..3c2c31f506f132 --- /dev/null +++ b/packages/sync/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "declarationDir": "build-types" + }, + "include": [ "src/**/*" ] +} diff --git a/tsconfig.json b/tsconfig.json index 7f4d4054bea88b..2c395450fb6a0e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -42,6 +42,7 @@ { "path": "packages/report-flaky-tests" }, { "path": "packages/rich-text" }, { "path": "packages/style-engine" }, + { "path": "packages/sync" }, { "path": "packages/token-list" }, { "path": "packages/url" }, { "path": "packages/warning" }, From ee064bf1cc05b85dd44cd88bf86fdc565a5b08dc Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Fri, 7 Jul 2023 12:12:00 +0100 Subject: [PATCH 2/9] Add more synced entities --- packages/core-data/src/entities.js | 69 ++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index 90e95a87cd7946..a5ae2ed1f9f289 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -40,26 +40,19 @@ export const rootEntitiesConfig = [ }, syncConfig: { fetch: async () => { - await new Promise( ( resolve ) => setTimeout( resolve, 5000 ) ); return apiFetch( { path: '/' } ); }, applyChangesToDoc: ( doc, changes ) => { const document = doc.getMap( 'document' ); - [ 'name', 'description' ].forEach( ( key ) => { - if ( document.get( key ) !== changes[ key ] ) { - document.set( key, changes[ key ] ); - } - } ); - /*Object.entries( changes ).forEach( ( [ key, value ] ) => { + Object.entries( changes ).forEach( ( [ key, value ] ) => { if ( document.get( key ) !== value ) { document.set( key, value ); } - } );*/ + } ); }, fromCRDTDoc: ( doc ) => { return doc.getMap( 'document' ).toJSON(); }, - handleChanges: () => {}, }, syncObjectType: 'root/base', getSyncObjectId: () => 'index', @@ -72,6 +65,24 @@ export const rootEntitiesConfig = [ getTitle: ( record ) => { return record?.title ?? __( 'Site Title' ); }, + syncConfig: { + fetch: async () => { + return apiFetch( { path: '/wp/v2/settings' } ); + }, + applyChangesToDoc: ( doc, changes ) => { + const document = doc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( document.get( key ) !== value ) { + document.set( key, value ); + } + } ); + }, + fromCRDTDoc: ( doc ) => { + return doc.getMap( 'document' ).toJSON(); + }, + }, + syncObjectType: 'root/site', + getSyncObjectId: () => 'index', }, { label: __( 'Post Type' ), @@ -80,6 +91,26 @@ export const rootEntitiesConfig = [ key: 'slug', baseURL: '/wp/v2/types', baseURLParams: { context: 'edit' }, + syncConfig: { + fetch: async ( id ) => { + return apiFetch( { + path: `/wp/v2/types/${ id }?context=edit`, + } ); + }, + applyChangesToDoc: ( doc, changes ) => { + const document = doc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( document.get( key ) !== value ) { + document.set( key, value ); + } + } ); + }, + fromCRDTDoc: ( doc ) => { + return doc.getMap( 'document' ).toJSON(); + }, + }, + syncObjectType: 'root/postType', + getSyncObjectId: ( id ) => id, }, { name: 'media', @@ -263,6 +294,26 @@ async function loadPostTypeEntities() { : String( record.id ) ), __unstablePrePersist: isTemplate ? undefined : prePersistPostType, __unstable_rest_base: postType.rest_base, + syncConfig: { + fetch: async ( id ) => { + return apiFetch( { + path: `/${ namespace }/${ postType.rest_base }/${ id }?context=edit`, + } ); + }, + applyChangesToDoc: ( doc, changes ) => { + const document = doc.getMap( 'document' ); + Object.entries( changes ).forEach( ( [ key, value ] ) => { + if ( document.get( key ) !== value ) { + document.set( key, value ); + } + } ); + }, + fromCRDTDoc: ( doc ) => { + return doc.getMap( 'document' ).toJSON(); + }, + }, + syncObjectType: 'postType/' + postType.name, + getSyncObjectId: ( id ) => id, }; } ); } From 5391055070e1817288f5b9df30788ff904ae5c18 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 10 Jul 2023 14:20:50 +0100 Subject: [PATCH 3/9] Add public webrtc sync --- lib/experimental/synchronization.php | 20 ++++ lib/load.php | 1 + package-lock.json | 151 ++++++++++++++++++++++++++- packages/core-data/src/sync.js | 9 +- packages/sync/README.md | 14 +++ packages/sync/package.json | 1 + packages/sync/src/connect-webrtc.js | 28 +++++ packages/sync/src/index.js | 1 + 8 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 lib/experimental/synchronization.php create mode 100644 packages/sync/src/connect-webrtc.js diff --git a/lib/experimental/synchronization.php b/lib/experimental/synchronization.php new file mode 100644 index 00000000000000..44d51636235e9a --- /dev/null +++ b/lib/experimental/synchronization.php @@ -0,0 +1,20 @@ +=6.9.0" } }, + "node_modules/get-browser-rtc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz", + "integrity": "sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ==" + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -50482,6 +50487,25 @@ "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", "dev": true }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/quick-lru": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", @@ -50505,7 +50529,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, "dependencies": { "safe-buffer": "^5.1.0" } @@ -54278,6 +54301,96 @@ "resolved": "https://registry.npmjs.org/simple-html-tokenizer/-/simple-html-tokenizer-0.5.7.tgz", "integrity": "sha512-APW9iYbkJ5cijjX4Ljhf3VG8SwYPUJT5gZrwci/wieMabQxWFiV5VwsrP5c6GMRvXKEQMGkAB1d9dvW66dTqpg==" }, + "node_modules/simple-peer": { + "version": "9.11.1", + "resolved": "https://registry.npmjs.org/simple-peer/-/simple-peer-9.11.1.tgz", + "integrity": "sha512-D1SaWpOW8afq1CZGWB8xTfrT3FekjQmPValrqncJMX7QFl8YwhrPTZvMCANLtgBwwdS+7zURyqxDDEmY558tTw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "buffer": "^6.0.3", + "debug": "^4.3.2", + "err-code": "^3.0.1", + "get-browser-rtc": "^1.1.0", + "queue-microtask": "^1.2.3", + "randombytes": "^2.1.0", + "readable-stream": "^3.6.0" + } + }, + "node_modules/simple-peer/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/simple-peer/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/simple-peer/node_modules/err-code": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", + "integrity": "sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==" + }, + "node_modules/simple-peer/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/simple-peer/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/simple-swizzle": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", @@ -59873,6 +59986,41 @@ "yjs": "^13.0.0" } }, + "node_modules/y-protocols": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.5.tgz", + "integrity": "sha512-Wil92b7cGk712lRHDqS4T90IczF6RkcvCwAD0A2OPg+adKmOe+nOiT/N2hvpQIWS3zfjmtL4CPaH5sIW1Hkm/A==", + "dependencies": { + "lib0": "^0.2.42" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/y-webrtc": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/y-webrtc/-/y-webrtc-10.2.5.tgz", + "integrity": "sha512-ZyBNvTI5L28sQ2PQI0T/JvyWgvuTq05L21vGkIlcvNLNSJqAaLCBJRe3FHEqXoaogqWmRcEAKGfII4ErNXMnNw==", + "dependencies": { + "lib0": "^0.2.42", + "simple-peer": "^9.11.0", + "y-protocols": "^1.0.5" + }, + "bin": { + "y-webrtc-signaling": "bin/server.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "optionalDependencies": { + "ws": "^7.2.0" + } + }, "node_modules/y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", @@ -62086,6 +62234,7 @@ "dependencies": { "@babel/runtime": "^7.16.0", "y-indexeddb": "~9.0.11", + "y-webrtc": "~10.2.5", "yjs": "~13.6.6" }, "engines": { diff --git a/packages/core-data/src/sync.js b/packages/core-data/src/sync.js index e02d524abc5da7..89ebdd605208d2 100644 --- a/packages/core-data/src/sync.js +++ b/packages/core-data/src/sync.js @@ -1,12 +1,17 @@ /** * WordPress dependencies */ -import { createSyncProvider, connectIndexDb } from '@wordpress/sync'; +import { + createSyncProvider, + connectIndexDb, + connectWebRTC, +} from '@wordpress/sync'; let syncProvider; + export function getSyncProvider() { if ( ! syncProvider ) { - syncProvider = createSyncProvider( connectIndexDb ); + syncProvider = createSyncProvider( connectIndexDb, connectWebRTC ); } return syncProvider; diff --git a/packages/sync/README.md b/packages/sync/README.md index 7f661921c8ac09..20fa00d9c73863 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -28,6 +28,20 @@ _Returns_ - `Promise<() => void>`: Promise that resolves when the connection is established. +### connectWebRTC + +Connect function to the IndexedDB persistence provider. + +_Parameters_ + +- _objectId_ `ObjectID`: The object ID. +- _objectType_ `ObjectType`: The object type. +- _doc_ `CRDTDoc`: The CRDT document. + +_Returns_ + +- `Promise<() => void>`: Promise that resolves when the connection is established. + ### createSyncProvider Create a sync provider. diff --git a/packages/sync/package.json b/packages/sync/package.json index 268eceddaa1d2c..dbb78409497d39 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -29,6 +29,7 @@ "dependencies": { "@babel/runtime": "^7.16.0", "y-indexeddb": "~9.0.11", + "y-webrtc": "~10.2.5", "yjs": "~13.6.6" }, "publishConfig": { diff --git a/packages/sync/src/connect-webrtc.js b/packages/sync/src/connect-webrtc.js new file mode 100644 index 00000000000000..66005d1cd52283 --- /dev/null +++ b/packages/sync/src/connect-webrtc.js @@ -0,0 +1,28 @@ +/** + * External dependencies + */ +// @ts-ignore +import { WebrtcProvider } from 'y-webrtc'; + +/** @typedef {import('./types').ObjectType} ObjectType */ +/** @typedef {import('./types').ObjectID} ObjectID */ +/** @typedef {import('./types').CRDTDoc} CRDTDoc */ + +/** + * Connect function to the IndexedDB persistence provider. + * + * @param {ObjectID} objectId The object ID. + * @param {ObjectType} objectType The object type. + * @param {CRDTDoc} doc The CRDT document. + * + * @return {Promise<() => void>} Promise that resolves when the connection is established. + */ +export function connectWebRTC( objectId, objectType, doc ) { + const docName = `${ objectType }-${ objectId }`; + new WebrtcProvider( docName, doc, { + // @ts-ignore + password: window.__experimentalCollaborativeEditingSecret, + } ); + + return Promise.resolve( () => true ); +} diff --git a/packages/sync/src/index.js b/packages/sync/src/index.js index 8a876f85552877..975fb52989f5d2 100644 --- a/packages/sync/src/index.js +++ b/packages/sync/src/index.js @@ -1,2 +1,3 @@ export { connectIndexDb } from './connect-indexdb'; +export { connectWebRTC } from './connect-webrtc'; export { createSyncProvider } from './provider'; From ba8c194ea82d06d7f2c13dbcdac66de25ceb7730 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 17 Jul 2023 12:16:04 +0100 Subject: [PATCH 4/9] Synchronizing edits --- packages/core-data/src/actions.js | 38 ++++++++++++++++++----------- packages/core-data/src/entities.js | 8 +++++- packages/core-data/src/resolvers.js | 20 +++++++++++++++ packages/sync/src/provider.js | 32 +++++++++++++++++++++--- packages/sync/src/types.ts | 1 + packages/sync/tsconfig.json | 3 ++- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/packages/core-data/src/actions.js b/packages/core-data/src/actions.js index bfe4a4a185f3f4..6e36da317952ef 100644 --- a/packages/core-data/src/actions.js +++ b/packages/core-data/src/actions.js @@ -18,6 +18,7 @@ import { receiveItems, removeItems, receiveQueriedItems } from './queried-data'; import { getOrLoadEntitiesConfig, DEFAULT_ENTITY_KEY } from './entities'; import { createBatch } from './batch'; import { STORE_NAME } from './name'; +import { getSyncProvider } from './sync'; /** * Returns an action object used in signalling that authors have been received. @@ -382,21 +383,30 @@ export const editEntityRecord = return acc; }, {} ), }; - dispatch( { - type: 'EDIT_ENTITY_RECORD', - ...edit, - meta: { - undo: ! options.undoIgnore && { - ...edit, - // Send the current values for things like the first undo stack entry. - edits: Object.keys( edits ).reduce( ( acc, key ) => { - acc[ key ] = editedRecord[ key ]; - return acc; - }, {} ), - isCached: options.isCached, + if ( entityConfig.syncConfig ) { + const objectId = entityConfig.getSyncObjectId( recordId ); + getSyncProvider().update( + entityConfig.syncObjectType + '--edit', + objectId, + edit.edits + ); + } else { + dispatch( { + type: 'EDIT_ENTITY_RECORD', + ...edit, + meta: { + undo: ! options.undoIgnore && { + ...edit, + // Send the current values for things like the first undo stack entry. + edits: Object.keys( edits ).reduce( ( acc, key ) => { + acc[ key ] = editedRecord[ key ]; + return acc; + }, {} ), + isCached: options.isCached, + }, }, - }, - } ); + } ); + } }; /** diff --git a/packages/core-data/src/entities.js b/packages/core-data/src/entities.js index a5ae2ed1f9f289..6c1579de1ecdf1 100644 --- a/packages/core-data/src/entities.js +++ b/packages/core-data/src/entities.js @@ -303,7 +303,10 @@ async function loadPostTypeEntities() { applyChangesToDoc: ( doc, changes ) => { const document = doc.getMap( 'document' ); Object.entries( changes ).forEach( ( [ key, value ] ) => { - if ( document.get( key ) !== value ) { + if ( + document.get( key ) !== value && + typeof value !== 'function' + ) { document.set( key, value ); } } ); @@ -379,6 +382,9 @@ export const getMethodName = ( function registerSyncConfigs( configs ) { configs.forEach( ( { syncObjectType, syncConfig } ) => { getSyncProvider().register( syncObjectType, syncConfig ); + const editSyncConfig = { ...syncConfig }; + delete editSyncConfig.fetch; + getSyncProvider().register( syncObjectType + '--edit', editSyncConfig ); } ); } diff --git a/packages/core-data/src/resolvers.js b/packages/core-data/src/resolvers.js index d85663199a1360..b18f0472edca77 100644 --- a/packages/core-data/src/resolvers.js +++ b/packages/core-data/src/resolvers.js @@ -76,6 +76,8 @@ export const getEntityRecord = // use the sync algorithm instead of the old fetch behavior. if ( entityConfig.syncConfig && ! query ) { const objectId = entityConfig.getSyncObjectId( key ); + + // Loads the persisted document. await getSyncProvider().bootstrap( entityConfig.syncObjectType, objectId, @@ -88,6 +90,24 @@ export const getEntityRecord = ); } ); + + // Boostraps the edited document as well (and load from peers). + await getSyncProvider().bootstrap( + entityConfig.syncObjectType + '--edit', + objectId, + ( record ) => { + dispatch( { + type: 'EDIT_ENTITY_RECORD', + kind, + name, + recordId: key, + edits: record, + meta: { + undo: undefined, + }, + } ); + } + ); } else { if ( query !== undefined && query._fields ) { // If requesting specific fields, items and query association to said diff --git a/packages/sync/src/provider.js b/packages/sync/src/provider.js index 39d896f511d887..15d972dbcd4f09 100644 --- a/packages/sync/src/provider.js +++ b/packages/sync/src/provider.js @@ -7,6 +7,7 @@ import * as Y from 'yjs'; /** @typedef {import('./types').ObjectType} ObjectType */ /** @typedef {import('./types').ObjectID} ObjectID */ /** @typedef {import('./types').ObjectConfig} ObjectConfig */ +/** @typedef {import('./types').CRDTDoc} CRDTDoc */ /** @typedef {import('./types').ConnectDoc} ConnectDoc */ /** @typedef {import('./types').SyncProvider} SyncProvider */ @@ -28,6 +29,11 @@ export const createSyncProvider = ( connectLocal, connectRemote ) => { */ const listeners = {}; + /** + * @type {Record>} + */ + const docs = {}; + /** * Registeres an object type. * @@ -47,12 +53,14 @@ export const createSyncProvider = ( connectLocal, connectRemote ) => { */ async function bootstrap( objectType, objectId, handleChanges ) { const doc = new Y.Doc(); + docs[ objectType ] = docs[ objectType ] || {}; + docs[ objectType ][ objectId ] = doc; - const update = () => { + const updateHandler = () => { const data = config[ objectType ].fromCRDTDoc( doc ); handleChanges( data ); }; - doc.on( 'update', update ); + doc.on( 'update', updateHandler ); // connect to locally saved database. const destroyLocalConnection = await connectLocal( @@ -78,10 +86,27 @@ export const createSyncProvider = ( connectLocal, connectRemote ) => { listeners[ objectType ] = listeners[ objectType ] || {}; listeners[ objectType ][ objectId ] = () => { destroyLocalConnection(); - doc.off( 'update', update ); + doc.off( 'update', updateHandler ); }; } + /** + * Fetch data from local database or remote source. + * + * @param {ObjectType} objectType Object type to load. + * @param {ObjectID} objectId Object ID to load. + * @param {any} data Updates to make. + */ + async function update( objectType, objectId, data ) { + const doc = docs[ objectType ][ objectId ]; + if ( ! doc ) { + throw 'Error doc ' + objectType + ' ' + objectId + ' not found'; + } + doc.transact( () => { + config[ objectType ].applyChangesToDoc( doc, data ); + } ); + } + /** * Stop updating a document and discard it. * @@ -97,6 +122,7 @@ export const createSyncProvider = ( connectLocal, connectRemote ) => { return { register, bootstrap, + update, discard, }; }; diff --git a/packages/sync/src/types.ts b/packages/sync/src/types.ts index 9ceb5d2e0e33bf..03439ecf280319 100644 --- a/packages/sync/src/types.ts +++ b/packages/sync/src/types.ts @@ -22,5 +22,6 @@ export type SyncProvider = { id: ObjectID, handleChanges: ( data: any ) => void ) => Promise< CRDTDoc >; + update: ( type: ObjectType, id: ObjectID, data: any ) => void; discard: ( type: ObjectType, id: ObjectID ) => Promise< CRDTDoc >; }; diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json index 3c2c31f506f132..d2d94e16acd2ce 100644 --- a/packages/sync/tsconfig.json +++ b/packages/sync/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "rootDir": "src", - "declarationDir": "build-types" + "declarationDir": "build-types", + "types": [ "node" ] }, "include": [ "src/**/*" ] } From 16dce85bc8635192e00fda9e8811f486023bb1a3 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 17 Jul 2023 11:51:17 +0100 Subject: [PATCH 5/9] Hide behind experimental flag --- lib/experimental/editor-settings.php | 3 +++ lib/experimental/synchronization.php | 4 ++++ lib/experiments-page.php | 12 ++++++++++++ packages/core-data/src/actions.js | 2 +- packages/core-data/src/resolvers.js | 6 +++++- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/experimental/editor-settings.php b/lib/experimental/editor-settings.php index 9c7f66a587a3aa..cec1eafcf94fa0 100644 --- a/lib/experimental/editor-settings.php +++ b/lib/experimental/editor-settings.php @@ -77,6 +77,9 @@ function gutenberg_initialize_editor( $editor_name, $editor_script_handle, $sett */ function gutenberg_enable_experiments() { $gutenberg_experiments = get_option( 'gutenberg-experiments' ); + if ( $gutenberg_experiments && array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) { + wp_add_inline_script( 'wp-block-editor', 'window.__experimentalEnableSync = true', 'before' ); + } if ( $gutenberg_experiments && array_key_exists( 'gutenberg-zoomed-out-view', $gutenberg_experiments ) ) { wp_add_inline_script( 'wp-block-editor', 'window.__experimentalEnableZoomedOutView = true', 'before' ); } diff --git a/lib/experimental/synchronization.php b/lib/experimental/synchronization.php index 44d51636235e9a..e4ad7333641daa 100644 --- a/lib/experimental/synchronization.php +++ b/lib/experimental/synchronization.php @@ -9,6 +9,10 @@ * Initializes the collaborative editing secret. */ function gutenberg_rest_api_init_collaborative_editing() { + $gutenberg_experiments = get_option( 'gutenberg-experiments' ); + if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) { + return; + } $collaborative_editing_secret = get_site_option( 'collaborative_editing_secret' ); if ( ! $collaborative_editing_secret ) { $collaborative_editing_secret = wp_generate_password( 64, false ); diff --git a/lib/experiments-page.php b/lib/experiments-page.php index 3f468d0cbd12db..f49e3f7e3db712 100644 --- a/lib/experiments-page.php +++ b/lib/experiments-page.php @@ -43,6 +43,18 @@ function gutenberg_initialize_experiments_settings() { 'gutenberg-experiments' ); + add_settings_field( + 'gutenberg-sync-collaboration', + __( 'Live Collaboration and offline persistence ', 'gutenberg' ), + 'gutenberg_display_experiment_field', + 'gutenberg-experiments', + 'gutenberg_experiments_section', + array( + 'label' => __( 'Elable the live collaboration and offline persistence between peers', 'gutenberg' ), + 'id' => 'gutenberg-sync-collaboration', + ) + ); + add_settings_field( 'gutenberg-zoomed-out-view', __( 'Zoomed out view ', 'gutenberg' ), diff --git a/packages/core-data/src/actions.js b/packages/core-data/src/actions.js index 6e36da317952ef..1969d2cd717a2a 100644 --- a/packages/core-data/src/actions.js +++ b/packages/core-data/src/actions.js @@ -383,7 +383,7 @@ export const editEntityRecord = return acc; }, {} ), }; - if ( entityConfig.syncConfig ) { + if ( window.__experimentalEnableSync && entityConfig.syncConfig ) { const objectId = entityConfig.getSyncObjectId( recordId ); getSyncProvider().update( entityConfig.syncObjectType + '--edit', diff --git a/packages/core-data/src/resolvers.js b/packages/core-data/src/resolvers.js index b18f0472edca77..a9bd6adfcdbff0 100644 --- a/packages/core-data/src/resolvers.js +++ b/packages/core-data/src/resolvers.js @@ -74,7 +74,11 @@ export const getEntityRecord = try { // Entity supports configs, // use the sync algorithm instead of the old fetch behavior. - if ( entityConfig.syncConfig && ! query ) { + if ( + window.__experimentalEnableSync && + entityConfig.syncConfig && + ! query + ) { const objectId = entityConfig.getSyncObjectId( key ); // Loads the persisted document. From 792d501ffedb2a93d6657a8cb822ed2210686bc9 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Mon, 17 Jul 2023 11:51:37 +0100 Subject: [PATCH 6/9] Small README tweak --- packages/sync/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sync/README.md b/packages/sync/README.md index 20fa00d9c73863..cbe0241c457b06 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -1,6 +1,6 @@ # Sync -Sync data between frontend and backend. +Sync data between multiple peers and persist in a local database. ## Installation From f63ad1ad0951e8538ed26194b3d83ddbeb77b7e7 Mon Sep 17 00:00:00 2001 From: Nik Tsekouras Date: Thu, 20 Jul 2023 15:02:37 +0300 Subject: [PATCH 7/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pascal Birchler Co-authored-by: Fabian Kägy --- lib/experiments-page.php | 2 +- packages/sync/src/connect-webrtc.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/experiments-page.php b/lib/experiments-page.php index f49e3f7e3db712..d10be06feef19a 100644 --- a/lib/experiments-page.php +++ b/lib/experiments-page.php @@ -50,7 +50,7 @@ function gutenberg_initialize_experiments_settings() { 'gutenberg-experiments', 'gutenberg_experiments_section', array( - 'label' => __( 'Elable the live collaboration and offline persistence between peers', 'gutenberg' ), + 'label' => __( 'Enable the live collaboration and offline persistence between peers', 'gutenberg' ), 'id' => 'gutenberg-sync-collaboration', ) ); diff --git a/packages/sync/src/connect-webrtc.js b/packages/sync/src/connect-webrtc.js index 66005d1cd52283..867bba39d68927 100644 --- a/packages/sync/src/connect-webrtc.js +++ b/packages/sync/src/connect-webrtc.js @@ -9,7 +9,7 @@ import { WebrtcProvider } from 'y-webrtc'; /** @typedef {import('./types').CRDTDoc} CRDTDoc */ /** - * Connect function to the IndexedDB persistence provider. + * Connect function to the WebRTC provider. * * @param {ObjectID} objectId The object ID. * @param {ObjectType} objectType The object type. From 83a3075853c7e2c890e95623ef8be19bd6434c88 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Wed, 9 Aug 2023 10:03:24 +0100 Subject: [PATCH 8/9] Fix lock --- package-lock.json | 85 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index f82418d70536d8..3370004749667f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -62229,6 +62229,7 @@ } }, "packages/sync": { + "name": "@wordpress/sync", "version": "0.1.0", "license": "GPL-2.0-or-later", "dependencies": { @@ -77650,6 +77651,7 @@ "requires": { "@babel/runtime": "^7.16.0", "y-indexeddb": "~9.0.11", + "y-webrtc": "~10.2.5", "yjs": "~13.6.6" } }, @@ -92084,6 +92086,11 @@ "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==" }, + "get-browser-rtc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz", + "integrity": "sha512-MghbMJ61EJrRsDe7w1Bvqt3ZsBuqhce5nrn/XAwgwOXhcsz53/ltdxOse1h/8eKXj5slzxdsz56g5rzOFSGwfQ==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -103356,6 +103363,11 @@ "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", "dev": true }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + }, "quick-lru": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", @@ -103372,7 +103384,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dev": true, "requires": { "safe-buffer": "^5.1.0" } @@ -106283,6 +106294,59 @@ "resolved": "https://registry.npmjs.org/simple-html-tokenizer/-/simple-html-tokenizer-0.5.7.tgz", "integrity": "sha512-APW9iYbkJ5cijjX4Ljhf3VG8SwYPUJT5gZrwci/wieMabQxWFiV5VwsrP5c6GMRvXKEQMGkAB1d9dvW66dTqpg==" }, + "simple-peer": { + "version": "9.11.1", + "resolved": "https://registry.npmjs.org/simple-peer/-/simple-peer-9.11.1.tgz", + "integrity": "sha512-D1SaWpOW8afq1CZGWB8xTfrT3FekjQmPValrqncJMX7QFl8YwhrPTZvMCANLtgBwwdS+7zURyqxDDEmY558tTw==", + "requires": { + "buffer": "^6.0.3", + "debug": "^4.3.2", + "err-code": "^3.0.1", + "get-browser-rtc": "^1.1.0", + "queue-microtask": "^1.2.3", + "randombytes": "^2.1.0", + "readable-stream": "^3.6.0" + }, + "dependencies": { + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "err-code": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", + "integrity": "sha512-GiaH0KJUewYok+eeY05IIgjtAe4Yltygk9Wqp1V5yVWLdhf0hYZchRjNIT9bb0mSwRcIusT3cx7PJUf3zEIfUA==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "simple-swizzle": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", @@ -110549,6 +110613,25 @@ "lib0": "^0.2.74" } }, + "y-protocols": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-1.0.5.tgz", + "integrity": "sha512-Wil92b7cGk712lRHDqS4T90IczF6RkcvCwAD0A2OPg+adKmOe+nOiT/N2hvpQIWS3zfjmtL4CPaH5sIW1Hkm/A==", + "requires": { + "lib0": "^0.2.42" + } + }, + "y-webrtc": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/y-webrtc/-/y-webrtc-10.2.5.tgz", + "integrity": "sha512-ZyBNvTI5L28sQ2PQI0T/JvyWgvuTq05L21vGkIlcvNLNSJqAaLCBJRe3FHEqXoaogqWmRcEAKGfII4ErNXMnNw==", + "requires": { + "lib0": "^0.2.42", + "simple-peer": "^9.11.0", + "ws": "^7.2.0", + "y-protocols": "^1.0.5" + } + }, "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", From 7b676ea50db5ca925605939a75af8af552c89bab Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Wed, 9 Aug 2023 11:05:56 +0100 Subject: [PATCH 9/9] Fix docs linting --- packages/sync/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sync/README.md b/packages/sync/README.md index cbe0241c457b06..62fe20af4f2fbb 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -30,7 +30,7 @@ _Returns_ ### connectWebRTC -Connect function to the IndexedDB persistence provider. +Connect function to the WebRTC provider. _Parameters_