From 9895d0914b0f4704de8170fe6082a575724d003b Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 7 Nov 2019 02:55:35 +0100 Subject: [PATCH 1/2] Sync Editor React state with Yjs. I hooked Yjs to the React editor state in the /playground. When a editor block changes, it currently overwrites the complete block content, instead of applying the differences. This is basically the same syncing approach as described in #17964, therefore it should allow for a fair comparison. But Yjs also allows to apply differences to the text object and is better suited to enable multiple users to work on the same paragraph. --- package-lock.json | 45 +++++++++++++++++++++- package.json | 4 +- playground/src/index.js | 84 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index e489d2de0e3937..e848fc5e491eb0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8766,8 +8766,7 @@ "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==", - "dev": true + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, "asynckit": { "version": "0.4.0", @@ -20850,6 +20849,11 @@ "type-check": "~0.3.2" } }, + "lib0": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.1.2.tgz", + "integrity": "sha512-p64v9GZhdSA+29mGpn9S06ytaFoOSve6+rfvOV+8cFxiCqmN7nqt137q2PuSc+fuNXJ8k6fIZ+w5xEeVhRH4qA==" + }, "line-height": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/line-height/-/line-height-0.3.1.tgz", @@ -34557,6 +34561,35 @@ "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", "dev": true }, + "y-protocols": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/y-protocols/-/y-protocols-0.1.0.tgz", + "integrity": "sha512-CQCCcexfTNI7wyvVlaTuOOuaWxuD4SkmZbwyIZVGKglAuC+ruivLmbB14b1oN1CdMPhlUkKsz6524o5sWdKNnA==", + "requires": { + "lib0": "^0.1.0" + } + }, + "y-websocket": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/y-websocket/-/y-websocket-1.0.6.tgz", + "integrity": "sha512-wSomPPCt/KAgefcDID7YEOnhWSpvkNo8TVy/5LITtiblW/xoMpz/Z/Jr6KDT9tDUfUfpquD8ITD9jWe5URJV+A==", + "requires": { + "lib0": "^0.1.1", + "ws": "^6.2.1", + "y-protocols": "^0.1.0" + }, + "dependencies": { + "ws": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", + "optional": true, + "requires": { + "async-limiter": "~1.0.0" + } + } + } + }, "y18n": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", @@ -34724,6 +34757,14 @@ "fd-slicer": "~1.0.1" } }, + "yjs": { + "version": "13.0.0-102", + "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.0.0-102.tgz", + "integrity": "sha512-Uda/w8yIwX57CHXS9mmP3X9N6Q9DHVAXCEgZJG/c7e4oHtN/A6JwAsltr4fVph0p5PPdFHf7uAUVaM5XJ6ZeIg==", + "requires": { + "lib0": "^0.1.1" + } + }, "zwitch": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-1.0.4.tgz", diff --git a/package.json b/package.json index 0409deb1372b78..b735ae48580a66 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,9 @@ "@wordpress/token-list": "file:packages/token-list", "@wordpress/url": "file:packages/url", "@wordpress/viewport": "file:packages/viewport", - "@wordpress/wordcount": "file:packages/wordcount" + "@wordpress/wordcount": "file:packages/wordcount", + "y-websocket": "1.0.6", + "yjs": "13.0.0-102" }, "devDependencies": { "@actions/core": "1.0.0", diff --git a/playground/src/index.js b/playground/src/index.js index 66815d8a027256..08519e4164c77d 100644 --- a/playground/src/index.js +++ b/playground/src/index.js @@ -3,7 +3,7 @@ */ import '@wordpress/editor'; // This shouldn't be necessary -import { render, useState, Fragment } from '@wordpress/element'; +import { render, useState, useEffect, Fragment } from '@wordpress/element'; import { BlockEditorKeyboardShortcuts, BlockEditorProvider, @@ -35,8 +35,84 @@ import '@wordpress/block-library/build-style/theme.css'; import '@wordpress/format-library/build-style/style.css'; /* eslint-enable no-restricted-syntax */ +/** + * External dependencies + */ +import * as Y from 'yjs'; +import { WebsocketProvider } from 'y-websocket'; + +import _ from 'lodash'; + +// ydoc is the CRDT state +const ydoc = new Y.Doc(); +// Connect ydoc to other clients via the Websocket provider. +const provider = new WebsocketProvider( `${ window.location.protocol === 'http:' ? 'ws:' : 'wss:' }//yjs-demos.now.sh`, 'gutenberg', ydoc ); +// Define a shared type on ydoc. +// A shared type works like any other data type - but it fires synchronous +// events when it changes and is automatically synced with other clients. +const yContent = ydoc.getArray( 'gutenberg-content' ); + +window.debugY = { + ydoc, provider, yContent, +}; + +/** + * Create a diff between two editor states. + * + * @template T + * @param {Array} a The old state + * @param {Array} b The updated state + * @return {{index:number,remove:number,insert:Array}} The diff description. + */ +const simpleDiff = ( a, b ) => { + let left = 0; // number of same characters counting from left + let right = 0; // number of same characters counting from right + while ( left < a.length && left < b.length && _.isEqual( a[ left ], b[ left ] ) ) { + left++; + } + if ( left !== a.length || left !== b.length ) { + // Only check right if a !== b + while ( right + left < a.length && right + left < b.length && _.isEqual( a[ a.length - right - 1 ], b[ b.length - right - 1 ] ) ) { + right++; + } + } + return { + index: left, + remove: a.length - left - right, + insert: b.slice( left, b.length - right ), + }; +}; + function App() { - const [ blocks, updateBlocks ] = useState( [] ); + const [ blocks, updateBlocks ] = useState( yContent.toArray() ); + useEffect( () => { + // Update react state before observer is registered. + const timeout = setTimeout( () => { + updateBlocks( yContent.toArray() ); + }, 0 ); + // Register type observer + const observer = ( ) => { + // update react state + updateBlocks( yContent.toArray() ); + }; + yContent.observeDeep( observer ); + return () => { + yContent.unobserveDeep( observer ); + clearTimeout( timeout ); + }; + } ); + const updateYjsType = ( newEditorContent ) => { + // Use a very basic diff approach to calculate the differences + const currentContent = yContent.toArray(); + const d = simpleDiff( currentContent, newEditorContent ); + // Bundle all changes as a single transaction + // This transaction will trigger the observer call, which will + // trigger `updateBlocks`. + ydoc.transact( () => { + yContent.delete( d.index, d.remove ); + yContent.insert( d.index, d.insert ); + } ); + }; return ( @@ -51,8 +127,8 @@ function App() {
From 1bf98d11082ab29c493620ff4593cd29a1ea8c0d Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 13 Nov 2019 03:06:33 +0100 Subject: [PATCH 2/2] =?UTF-8?q?Yjs&Gutenberg:=20Integrate=20binding=20in?= =?UTF-8?q?=20core-data=20=E2=87=92=20enable=20shared=20editing=20in=20Wor?= =?UTF-8?q?dPress?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 5 +- package.json | 4 +- packages/core-data/package.json | 5 +- packages/core-data/src/actions.js | 15 +++- packages/core-data/src/ystore.js | 141 ++++++++++++++++++++++++++++++ playground/src/index.js | 84 +----------------- 6 files changed, 168 insertions(+), 86 deletions(-) create mode 100644 packages/core-data/src/ystore.js diff --git a/package-lock.json b/package-lock.json index e848fc5e491eb0..cc3b684918fc25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7395,8 +7395,11 @@ "@wordpress/is-shallow-equal": "file:packages/is-shallow-equal", "@wordpress/url": "file:packages/url", "equivalent-key-map": "^0.2.2", + "lib0": "^0.1.2", "lodash": "^4.17.15", - "rememo": "^3.0.0" + "rememo": "^3.0.0", + "y-websocket": "^1.0.6", + "yjs": "^13.0.0-102" } }, "@wordpress/custom-templated-path-webpack-plugin": { diff --git a/package.json b/package.json index b735ae48580a66..0409deb1372b78 100644 --- a/package.json +++ b/package.json @@ -62,9 +62,7 @@ "@wordpress/token-list": "file:packages/token-list", "@wordpress/url": "file:packages/url", "@wordpress/viewport": "file:packages/viewport", - "@wordpress/wordcount": "file:packages/wordcount", - "y-websocket": "1.0.6", - "yjs": "13.0.0-102" + "@wordpress/wordcount": "file:packages/wordcount" }, "devDependencies": { "@actions/core": "1.0.0", diff --git a/packages/core-data/package.json b/packages/core-data/package.json index 3c565d2bfc56c6..0c1598e9e5afc9 100644 --- a/packages/core-data/package.json +++ b/packages/core-data/package.json @@ -30,8 +30,11 @@ "@wordpress/is-shallow-equal": "file:../is-shallow-equal", "@wordpress/url": "file:../url", "equivalent-key-map": "^0.2.2", + "lib0": "^0.1.2", "lodash": "^4.17.15", - "rememo": "^3.0.0" + "rememo": "^3.0.0", + "y-websocket": "^1.0.6", + "yjs": "^13.0.0-102" }, "publishConfig": { "access": "public" diff --git a/packages/core-data/src/actions.js b/packages/core-data/src/actions.js index a79aaba7e33d8f..d462a5337f67ca 100644 --- a/packages/core-data/src/actions.js +++ b/packages/core-data/src/actions.js @@ -2,6 +2,10 @@ * External dependencies */ import { castArray, get, isEqual, find } from 'lodash'; +/** + * Internal dependencies + */ +import * as ystore from './ystore'; /** * Internal dependencies @@ -82,7 +86,11 @@ export function receiveEntityRecords( kind, name, records, query, invalidateCach } else { action = receiveItems( records ); } - + for ( const record of castArray( records ) ) { + if ( record.id ) { + ystore.getInstance( record.id, kind, name ); + } + } return { ...action, kind, @@ -149,6 +157,11 @@ export function* editEntityRecord( kind, name, recordId, edits, options = {} ) { recordId ); + if ( ! options.syncIgnore && edits.blocks ) { + // sync edits + ystore.updateInstance( recordId, kind, name, edits.blocks ); + } + const edit = { kind, name, diff --git a/packages/core-data/src/ystore.js b/packages/core-data/src/ystore.js new file mode 100644 index 00000000000000..eafd4b2a67f085 --- /dev/null +++ b/packages/core-data/src/ystore.js @@ -0,0 +1,141 @@ +/** + * WordPress dependencies + */ +import { dispatch } from '@wordpress/data'; + +/** + * External dependencies + */ +import * as map from 'lib0/map.js'; +import * as Y from 'yjs'; +import { WebsocketProvider } from 'y-websocket'; +import _ from 'lodash'; + +/** + * Map of Yjs instances + */ +const instances = new Map(); + +/** + * If two clients join a session at the same time, both with fill the shared + * document with the initial data, resulting in duplicate blocks. + * Before transforming the Yjs document to a Gutenberg blocks, we remove all + * duplicate blocks. + * + * @param {Y.Array} ycontent + */ +const checkDuplicateBlocks = ( ycontent ) => { + const found = new Set(); + for ( let i = ycontent.length - 1; i >= 0; i-- ) { + const c = ycontent.get( i ); + const clientId = c.clientId; + if ( clientId ) { + if ( found.has( clientId ) ) { + ycontent.delete( i, 1 ); + } else { + found.add( c.clientId ); + } + } + } +}; + +const gutenbergSourcedChange = Symbol( 'gutenberg-sourced-change' ); + +/** + * @param {number} id document id + * @param {string} kind Kind of the edited entity record. + * @param {string} name Name of the edited entity record. + * @return {{ydoc:Y.Doc,provider:any,type:Y.Array}} Instance description + */ +export const getInstance = ( id, kind, name ) => map.setIfUndefined( instances, id, () => { + // ydoc is the CRDT state + const ydoc = new Y.Doc(); + // Connect ydoc to other clients via the Websocket provider. + const location = window.location; + const provider = new WebsocketProvider( `${ location.protocol === 'http:' ? 'ws:' : 'wss:' }//yjs-demos.now.sh`, `gutenberg-${ location.host }-${ id }`, ydoc ); + // Define a shared type on ydoc. + // A shared type works like any other data type - but it fires synchronous + // events when it changes and is automatically synced with other clients. + const type = ydoc.getArray( 'gutenberg-blocks' ); + provider.on( 'sync', () => { + const updateEditor = () => { + checkDuplicateBlocks( type ); + // Type is synced and has content ⇒ Overwrite current state. + dispatch( 'core' ).editEntityRecord( + kind, + name, + id, + { blocks: type.toArray() }, + { undoIgnore: true, syncIgnore: true } + ); + }; + type.observeDeep( ( event, transaction ) => { + if ( transaction.origin !== gutenbergSourcedChange ) { + updateEditor(); + } + } ); + if ( type.length > 0 ) { + updateEditor(); + } else { + // Type is synced and has no content yet. + dispatch( 'core' ).receiveEntityRecords( kind, name, id ).then( ( record ) => { + if ( type.length === 0 && record.blocks ) { + type.insert( 0, record.blocks ); + } + } ); + } + } ); + + return { + ydoc, provider, type, + }; +} ); + +/** + * Create a diff between two editor states. + * + * @template T + * @param {Array} a The old state + * @param {Array} b The updated state + * @return {{index:number,remove:number,insert:Array}} The diff description. + */ +const simpleDiff = ( a, b ) => { + let left = 0; // number of same characters counting from left + let right = 0; // number of same characters counting from right + while ( left < a.length && left < b.length && _.isEqual( a[ left ], b[ left ] ) ) { + left++; + } + if ( left !== a.length || left !== b.length ) { + // Only check right if a !== b + while ( right + left < a.length && right + left < b.length && _.isEqual( a[ a.length - right - 1 ], b[ b.length - right - 1 ] ) ) { + right++; + } + } + return { + index: left, + remove: a.length - left - right, + insert: b.slice( left, b.length - right ), + }; +}; + +/** + * @param {number} id document id + * @param {string} kind Kind of the edited entity record. + * @param {string} name Name of the edited entity record. + * @param {Array} newEditorContent + */ +export const updateInstance = ( id, kind, name, newEditorContent ) => { + const { type, provider, ydoc } = getInstance( id, kind, name ); + if ( provider.synced ) { + // Use a very basic diff approach to calculate the differences + const currentContent = type.toArray(); + const d = simpleDiff( currentContent, newEditorContent ); + // Bundle all changes as a single transaction + // This transaction will trigger the observer call, which will + // trigger updateBlocks. + ydoc.transact( () => { + type.delete( d.index, d.remove ); + type.insert( d.index, d.insert ); + }, gutenbergSourcedChange ); + } +}; diff --git a/playground/src/index.js b/playground/src/index.js index 08519e4164c77d..66815d8a027256 100644 --- a/playground/src/index.js +++ b/playground/src/index.js @@ -3,7 +3,7 @@ */ import '@wordpress/editor'; // This shouldn't be necessary -import { render, useState, useEffect, Fragment } from '@wordpress/element'; +import { render, useState, Fragment } from '@wordpress/element'; import { BlockEditorKeyboardShortcuts, BlockEditorProvider, @@ -35,84 +35,8 @@ import '@wordpress/block-library/build-style/theme.css'; import '@wordpress/format-library/build-style/style.css'; /* eslint-enable no-restricted-syntax */ -/** - * External dependencies - */ -import * as Y from 'yjs'; -import { WebsocketProvider } from 'y-websocket'; - -import _ from 'lodash'; - -// ydoc is the CRDT state -const ydoc = new Y.Doc(); -// Connect ydoc to other clients via the Websocket provider. -const provider = new WebsocketProvider( `${ window.location.protocol === 'http:' ? 'ws:' : 'wss:' }//yjs-demos.now.sh`, 'gutenberg', ydoc ); -// Define a shared type on ydoc. -// A shared type works like any other data type - but it fires synchronous -// events when it changes and is automatically synced with other clients. -const yContent = ydoc.getArray( 'gutenberg-content' ); - -window.debugY = { - ydoc, provider, yContent, -}; - -/** - * Create a diff between two editor states. - * - * @template T - * @param {Array} a The old state - * @param {Array} b The updated state - * @return {{index:number,remove:number,insert:Array}} The diff description. - */ -const simpleDiff = ( a, b ) => { - let left = 0; // number of same characters counting from left - let right = 0; // number of same characters counting from right - while ( left < a.length && left < b.length && _.isEqual( a[ left ], b[ left ] ) ) { - left++; - } - if ( left !== a.length || left !== b.length ) { - // Only check right if a !== b - while ( right + left < a.length && right + left < b.length && _.isEqual( a[ a.length - right - 1 ], b[ b.length - right - 1 ] ) ) { - right++; - } - } - return { - index: left, - remove: a.length - left - right, - insert: b.slice( left, b.length - right ), - }; -}; - function App() { - const [ blocks, updateBlocks ] = useState( yContent.toArray() ); - useEffect( () => { - // Update react state before observer is registered. - const timeout = setTimeout( () => { - updateBlocks( yContent.toArray() ); - }, 0 ); - // Register type observer - const observer = ( ) => { - // update react state - updateBlocks( yContent.toArray() ); - }; - yContent.observeDeep( observer ); - return () => { - yContent.unobserveDeep( observer ); - clearTimeout( timeout ); - }; - } ); - const updateYjsType = ( newEditorContent ) => { - // Use a very basic diff approach to calculate the differences - const currentContent = yContent.toArray(); - const d = simpleDiff( currentContent, newEditorContent ); - // Bundle all changes as a single transaction - // This transaction will trigger the observer call, which will - // trigger `updateBlocks`. - ydoc.transact( () => { - yContent.delete( d.index, d.remove ); - yContent.insert( d.index, d.insert ); - } ); - }; + const [ blocks, updateBlocks ] = useState( [] ); return ( @@ -127,8 +51,8 @@ function App() {