Use lastKnownHash to handle checkpoints in the realtime channel

www/common/onlyoffice/inner.js

@@ -49,6 +49,8 @@ define([
         $: $
     };
 
+    var CHECKPOINT_INTERVAL = 50;
+
     var stringify = function (obj) {
         return JSONSortify(obj);
     };
@@ -58,11 +60,66 @@ define([
     var andThen = function (common) {
         var sframeChan = common.getSframeChannel();
         var metadataMgr = common.getMetadataMgr();
+        var privateData = metadataMgr.getPrivateData();
         var readOnly = false;
-        var locked = false;
+        //var locked = false;
         var config = {};
-        var hashes = [];
+        var content = {
-        var channel;
+            hashes: {},
+            ids: {}
+        };
+        var myOOId;
+
+        var deleteOffline = function () {
+            var ids = content.ids;
+            var users = Object.keys(metadataMgr.getMetadata().users);
+            Object.keys(ids).forEach(function (id) {
+                var nId = id.slice(0,32);
+                if (users.indexOf(nId) === -1) {
+                    delete ids[id];
+                }
+            });
+            APP.onLocal();
+        };
+
+        var isUserOnline = function (ooid) {
+            // Remove ids for users that have left the channel
+            deleteOffline();
+            var ids = content.ids;
+            // Check if the provided id is in the ID list
+            return Object.keys(ids).some(function (id) {
+                return ooid === ids[id];
+            });
+        };
+
+        var setMyId = function (netfluxId) {
+            // Remove ids for users that have left the channel
+            deleteOffline();
+            var ids = content.ids;
+            if (!myOOId) {
+                myOOId = Util.createRandomInteger();
+                while (Object.keys(ids).some(function (id) {
+                    return ids[id] === myOOId;
+                })) {
+                    myOOId = Util.createRandomInteger();
+                }
+            }
+            var myId = (netfluxId || metadataMgr.getNetfluxId()) + '-' + privateData.clientId;
+            ids[myId] = myOOId;
+            APP.onLocal();
+        };
+
+        // Another tab from our worker has left: remove its id from the list
+        var removeClient = function (obj) {
+            var tabId = metadataMgr.getNetfluxId() + '-' + obj.id;
+            console.log(tabId);
+            if (content.ids[tabId]) {
+                console.log('delete');
+                delete content.ids[tabId];
+                APP.onLocal();
+                console.log(content.ids);
+            }
+        };
 
         var getFileType = function () {
             var type = common.getMetadataMgr().getPrivateData().ooType;
@@ -91,12 +148,12 @@ define([
         var now = function () { return +new Date(); };
 
         var getLastCp = function () {
-            if (!hashes || !hashes.length) { return; }
+            var hashes = content.hashes;
-            var last = hashes.slice().pop();
+            if (!hashes || !Object.keys(hashes).length) { return {}; }
-            var parsed = Hash.parsePadUrl(last);
+            var lastIndex = Math.max.apply(null, Object.keys(hashes).map(Number));
-            var secret = Hash.getSecrets('file', parsed.hash);
+            // TODO check if hashes[lastIndex] is undefined?
-            if (!secret || !secret.channel) { return; }
+            var last = JSON.parse(JSON.stringify(hashes[lastIndex]));
-            return 'cp|' + secret.channel.slice(0,8);
+            return last;
         };
 
         var rtChannel = {
@@ -119,32 +176,112 @@ define([
         var ooChannel = {
             ready: false,
             queue: [],
-            send: function () {}
+            send: function () {},
+            cpIndex: 0
+        };
+
+        var getContent = APP.getContent = function () {
+            try {
+                return window.frames[0].editor.asc_nativeGetFile();
+            } catch (e) {
+                console.error(e);
+                return;
+            }
+        };
+
+        var fmConfig = {
+            noHandlers: true,
+            noStore: true,
+            body: $('body'),
+            onUploaded: function (ev, data) {
+                if (!data || !data.url) { return; }
+                sframeChan.query('Q_OO_SAVE', data, function (err) {
+                    if (err) {
+                        console.error(err);
+                        return void UI.alert(Messages.oo_saveError);
+                    }
+                    var i = Math.floor(ev.index / CHECKPOINT_INTERVAL);
+                    // XXX check if content.hashes[i] already exists?
+                    content.hashes[i] = {
+                        file: data.url,
+                        hash: ev.hash,
+                        index: ev.index
+                    };
+                    content.saveLock = undefined;
+                    APP.onLocal();
+                    sframeChan.query('Q_OO_COMMAND', {
+                        cmd: 'UPDATE_HASH',
+                        data: ev.hash
+                    }, function (err, obj) {
+                        if (err || (obj && obj.error)) { console.error(err || obj.error); }
+                    });
+                    UI.log(Messages.saved);
+                });
+            }
+        };
+        APP.FM = common.createFileManager(fmConfig);
+
+        var saveToServer = function () {
+            var text = getContent();
+            var blob = new Blob([text], {type: 'plain/text'});
+            var file = getFileType();
+            blob.name = (metadataMgr.getMetadataLazy().title || file.doc) + '.' + file.type;
+            var data = {
+                hash: ooChannel.lastHash,
+                index: ooChannel.cpIndex
+            };
+            APP.FM.handleFile(blob, data);
+        };
+        var makeCheckpoint = function (force) {
+            var locked = content.saveLock;
+            if (!locked || !isUserOnline(locked) || force) {
+                content.saveLock = myOOId;
+                APP.onLocal();
+                APP.realtime.onSettle(function () {
+                    saveToServer();
+                });
+                return;
+            }
+            // The save is locked by someone else. If no new checkpoint is created
+            // in the next 20 to 40 secondes and the lock is kept by the same user,
+            // force the lock and make a checkpoint.
+            var saved = stringify(content.hashes);
+            var to = 20000 + (Math.random() * 20000)
+            setTimeout(function () {
+                if (stringify(content.hashes) === saved && locked === content.saveLock) {
+                    makeCheckpoint(force);
+                }
+            }, to);
         };
 
         var openRtChannel = function (cb) {
-            if (rtChannel.ready) { return; }
+            if (rtChannel.ready) { return void cb(); }
-            var chan = channel || Hash.createChannelId();
+            var chan = content.channel || Hash.createChannelId();
-            if (!channel) {
+            if (!content.channel) {
-                channel = chan;
+                content.channel = chan;
                 APP.onLocal();
             }
             sframeChan.query('Q_OO_OPENCHANNEL', {
-                channel: channel,
+                channel: content.channel,
-                lastCp: getLastCp()
+                lastCpHash: getLastCp().hash
             }, function (err, obj) {
                 if (err || (obj && obj.error)) { console.error(err || (obj && obj.error)); }
             });
-            sframeChan.on('EV_OO_EVENT', function (data) {
+            sframeChan.on('EV_OO_EVENT', function (obj) {
-                switch (data.ev) {
+                switch (obj.ev) {
                     case 'READY':
                         rtChannel.ready = true;
                         break;
+                    case 'LEAVE':
+                        removeClient(obj.data);
+                        break;
                     case 'MESSAGE':
                         if (ooChannel.ready) {
-                            ooChannel.send(data.data);
+                            ooChannel.send(obj.data.msg);
+                            ooChannel.lastHash = obj.data.hash;
+                            ooChannel.cpIndex++;
                         } else {
-                            ooChannel.queue.push(data.data);
+                            ooChannel.queue.push(obj.data);
                         }
                         break;
                 }
@@ -168,7 +305,7 @@ define([
                 };
             });
         };
-        var mkChannel = function () {
+        var makeChannel = function () {
             var msgEv = Util.mkEvent();
             var iframe = $('#cp-app-oo-container > iframe')[0].contentWindow;
             window.addEventListener('message', function (msg) {
@@ -213,8 +350,10 @@ define([
                             });
                             setTimeout(function () {
                                 if (ooChannel.queue) {
-                                    ooChannel.queue.forEach(function (msg) {
+                                    ooChannel.queue.forEach(function (data) {
-                                        send(msg);
+                                        send(data.msg);
+                                        ooChannel.lastHash = data.hash;
+                                        ooChannel.cpIndex++;
                                     });
                                 }
                             }, 2000);
@@ -253,7 +392,12 @@ define([
                                 changesIndex: 2,
                                 locks: [], // XXX take from userdoc?
                                 excelAdditionalInfo: null
-                            }, null, function () {
+                            }, null, function (err, hash) {
+                                ooChannel.cpIndex++;
+                                ooChannel.lastHash = hash;
+                                if (ooChannel.cpIndex % CHECKPOINT_INTERVAL === 0) {
+                                    makeCheckpoint();
+                                }
                             });
                             break;
                     }
@@ -261,11 +405,11 @@ define([
             });
         };
 
+        var ooLoaded = false;
         var startOO = function (blob, file) {
             if (APP.ooconfig) { return void console.error('already started'); }
             var url = URL.createObjectURL(blob);
-            var lock = /*locked !== common.getMetadataMgr().getNetfluxId() ||*/
+            var lock = readOnly || !common.isLoggedIn();
-                       !common.isLoggedIn();
 
             // Config
             APP.ooconfig = {
@@ -313,51 +457,15 @@ define([
                 if (ifr) { ifr.remove(); }
             };
             APP.docEditor = new DocsAPI.DocEditor("cp-app-oo-placeholder", APP.ooconfig);
-            mkChannel();
+            ooLoaded = true;
-        };
+            makeChannel();
-
-        var getContent = APP.getContent = function () {
-            try {
-                return window.frames[0].editor.asc_nativeGetFile();
-            } catch (e) {
-                console.error(e);
-                return;
-            }
-        };
-
-        var fmConfig = {
-            noHandlers: true,
-            noStore: true,
-            body: $('body'),
-            onUploaded: function (ev, data) {
-                if (!data || !data.url) { return; }
-                common.getSframeChannel().query('Q_OO_SAVE', data, function (err) {
-                    if (err) {
-                        console.error(err);
-                        return void UI.alert(Messages.oo_saveError);
-                    }
-                    hashes.push(data.url);
-                    APP.onLocal();
-                    rtChannel.sendMsg(null, getLastCp(), function () {
-                        UI.log(Messages.saved);
-                    });
-                });
-            }
-        };
-        APP.FM = common.createFileManager(fmConfig);
-
-        var saveToServer = function () {
-            var text = getContent();
-            var blob = new Blob([text], {type: 'plain/text'});
-            var file = getFileType();
-            blob.name = (metadataMgr.getMetadataLazy().title || file.doc) + '.' + file.type;
-            APP.FM.handleFile(blob);
         };
 
         var loadLastDocument = function () {
-            if (!hashes || !hashes.length) { return; }
+            var lastCp = getLastCp();
-            var last = hashes.slice().pop();
+            if (!lastCp) { return; }
-            var parsed = Hash.parsePadUrl(last);
+            ooChannel.cpIndex = lastCp.index || 0;
+            var parsed = Hash.parsePadUrl(lastCp.file);
             var secret = Hash.getSecrets('file', parsed.hash);
             if (!secret || !secret.channel) { return; }
             var hexFileName = secret.channel;
@@ -383,6 +491,7 @@ define([
             xhr.send(null);
         };
         var loadDocument = function (newPad) {
+            if (ooLoaded) { return; }
             var type = common.getMetadataMgr().getPrivateData().ooType;
             var file = getFileType();
             if (!newPad) {
@@ -432,17 +541,15 @@ define([
 
         var stringifyInner = function () {
             var obj = {
-                content: {
+                content: content,
-                    channel: channel,
-                    hashes: hashes || [],
-                    //locked: locked
-                },
                 metadata: metadataMgr.getMetadataLazy()
             };
             // stringify the json and send it into chainpad
             return stringify(obj);
         };
 
+        APP.getContent = function () { return content; };
+
         APP.onLocal = config.onLocal = function () {
             if (initializing) { return; }
             if (readOnly) { return; }
@@ -482,7 +589,9 @@ define([
 
             var $rightside = toolbar.$rightside;
 
-            var $save = common.createButton('save', true, {}, saveToServer);
+            var $save = common.createButton('save', true, {}, function () {
+                saveToServer();
+            });
             $save.appendTo($rightside);
 
             if (common.isLoggedIn()) {
@@ -520,10 +629,8 @@ define([
                     UI.errorLoadingScreen(errorText);
                     throw new Error(errorText);
                 }
-                hashes = hjson.content && hjson.content.hashes;
+                content = hjson.content || content;
-                //locked = hjson.content && hjson.content.locked;
+                newDoc = !content.hashes || Object.keys(content.hashes).length === 0;
-                channel = hjson.content && hjson.content.channel;
-                newDoc = !hashes || hashes.length === 0;
             } else {
                 Title.updateTitle(Title.defaultTitle);
             }
@@ -551,15 +658,14 @@ define([
 
             openRtChannel(function () {
                 loadDocument(newDoc);
-
                 initializing = false;
+                setMyId();
                 setEditable(!readOnly);
                 UI.removeLoadingScreen();
             });
 
         };
 
-        var reloadDisplayed = false;
         config.onRemote = function () {
             if (initializing) { return; }
             var userDoc = APP.realtime.getUserDoc();
@@ -567,20 +673,7 @@ define([
             if (json.metadata) {
                 metadataMgr.updateMetadata(json.metadata);
             }
-            /*
+            content = json.content;
-            var newHashes = (json.content && json.content.hashes) || [];
-            if (newHashes.length !== hashes.length ||
-                stringify(newHashes) !== stringify(hashes)) {
-                hashes = newHashes;
-                if (reloadDisplayed) { return; }
-                reloadDisplayed = true;
-                UI.confirm(Messages.oo_newVersion, function (yes) {
-                    reloadDisplayed = false;
-                    if (!yes) { return; }
-                    common.gotoURL();
-                });
-            }
-            */
         };
 
         config.onAbort = function () {

www/common/onlyoffice/main.js

@@ -63,7 +63,7 @@ define([
                         // XXX add owners?
                         // owners: something...
                         channel: data.channel,
-                        lastCp: data.lastCp,
+                        lastCpHash: data.lastCpHash,
                         padChan: Utils.secret.channel,
                         validateKey: Utils.secret.keys.validateKey
                     }
@@ -71,18 +71,22 @@ define([
             });
             sframeChan.on('Q_OO_COMMAND', function (obj, cb) {
                 if (obj.cmd === 'SEND_MESSAGE') {
-                    if (obj.data.isCp) {
-                        obj.data.isCp += '|' + crypto.encrypt('cp');
-                    } else {
                     obj.data.msg = crypto.encrypt(JSON.stringify(obj.data.msg));
-                    }
+                    var hash = obj.data.msg.slice(0,64);
+                    var _cb = cb;
+                    cb = function () {
+                        _cb(hash);
+                    };
                 }
                 Cryptpad.onlyoffice.execCommand(obj, cb);
             });
             Cryptpad.onlyoffice.onEvent.reg(function (obj) {
-                if (obj.ev === 'MESSAGE') {
+                if (obj.ev === 'MESSAGE' && !/^cp\|/.test(obj.data)) {
                     try {
-                        obj.data = JSON.parse(crypto.decrypt(obj.data, Utils.secret.keys.validateKey));
+                        obj.data = {
+                            msg: JSON.parse(crypto.decrypt(obj.data, Utils.secret.keys.validateKey)),
+                            hash: obj.data.slice(0,64)
+                        };
                     } catch (e) {
                         console.error(e);
                     }

www/common/outer/async-store.js

@@ -454,6 +454,7 @@ define([
                 },
                 // "priv" is not shared with other users but is needed by the apps
                 priv: {
+                    clientId: clientId,
                     edPublic: store.proxy.edPublic,
                     friends: store.proxy.friends || {},
                     settings: store.proxy.settings,

www/common/outer/onlyoffice.js

@@ -1,11 +1,7 @@
 define([
-    '/common/common-util.js',
+], function () {
-], function (Util) {
     var OO = {};
 
-    var getHistory = function (ctx, data, clientId, cb) {
-    };
-
     var openChannel = function (ctx, obj, client, cb) {
         var channel = obj.channel;
         var padChan = obj.padChan;
@@ -29,7 +25,6 @@ define([
             // ==> Use our netflux ID to create our client ID
             if (!c.id) { c.id = chan.wc.myID + '-' + client; }
 
-            /// XXX send chan.history to client
             chan.history.forEach(function (msg) {
                 ctx.emit('MESSAGE', msg, [client]);
             });
@@ -61,17 +56,11 @@ define([
             }
 
             wc.on('join', function () {
-                // XXX
             });
-            wc.on('leave', function (peer) {
+            wc.on('leave', function () {
-                // XXX
             });
             wc.on('message', function (msg) {
-                if (/^cp\|/.test(msg)) {
-                    chan.history = [];
-                } else {
                 chan.history.push(msg);
-                }
                 ctx.emit('MESSAGE', msg, chan.clients);
             });
 
@@ -79,11 +68,7 @@ define([
             chan.sendMsg = function (msg, cb) {
                 cb = cb || function () {};
                 wc.bcast(msg).then(function () {
-                    if (/^cp\|/.test(msg)) {
-                        chan.history = [];
-                    } else {
                     chan.history.push(msg);
-                    }
                     cb();
                 }, function (err) {
                     cb({error: err});
@@ -92,7 +77,7 @@ define([
 
             if (first) {
                 chan.clients = [client];
-                chan.lastCp = obj.lastCp;
+                chan.lastCpHash = obj.lastCpHash;
                 first = false;
                 cb();
             }
@@ -100,7 +85,7 @@ define([
             var hk = network.historyKeeper;
             var cfg = {
                 validateKey: obj.validateKey,
-                lastKnownHash: chan.lastKnownHash,
+                lastKnownHash: chan.lastKnownHash || chan.lastCpHash,
                 owners: obj.owners,
             };
             var msg = ['GET_HISTORY', wc.id, cfg];
@@ -137,24 +122,11 @@ define([
             }
             if (parsed.error && parsed.channel) { return; }
 
-            var msg = parsed[4];
+            msg = parsed[4];
 
             // Keep only the history for our channel
             if (parsed[3] !== channel) { return; }
 
-            if (chan.lastCp) {
-                if (chan.lastCp === msg.slice(0, 11)) {
-                    delete chan.lastCp;
-                }
-                return;
-            }
-
-            var isCp = /^cp\|/.test(msg);
-            if (isCp) {
-                chan.history = [];
-                return;
-            }
-
             chan.lastKnownHash = msg.slice(0,64);
             ctx.emit('MESSAGE', msg, chan.clients);
             chan.history.push(msg);
@@ -172,6 +144,25 @@ define([
         });
     };
 
+    var updateHash = function (ctx, data, clientId, cb) {
+        var c = ctx.clients[clientId];
+        if (!c) { return void cb({ error: 'NOT_IN_CHANNEL' }); }
+        var chan = ctx.channels[c.channel];
+        if (!chan) { return void cb({ error: 'INVALID_CHANNEL' }); }
+        var hash = data;
+        var index = -1;
+        chan.history.some(function (msg, idx) {
+            if (msg.slice(0,64) === hash) {
+                index = idx + 1;
+                return true;
+            }
+        });
+        if (index !== -1) {
+            chan.history = chan.history.slice(index);
+        }
+        cb();
+    };
+
     var sendMessage = function (ctx, data, clientId, cb) {
         var c = ctx.clients[clientId];
         if (!c) { return void cb({ error: 'NOT_IN_CHANNEL' }); }
@@ -214,6 +205,11 @@ define([
             }
         }
 
+        var oldChannel = ctx.clients[clientId].channel;
+        var oldChan = ctx.channels[oldChannel];
+        if (oldChan) {
+            ctx.emit('LEAVE', {id: clientId}, [oldChan.clients[0]]);
+        }
         delete ctx.clients[clientId];
     };
 
@@ -240,6 +236,9 @@ define([
             if (cmd === 'SEND_MESSAGE') {
                 return void sendMessage(ctx, data, clientId, cb);
             }
+            if (cmd === 'UPDATE_HASH') {
+                return void updateHash(ctx, data, clientId, cb);
+            }
             if (cmd === 'OPEN_CHANNEL') {
                 return void openChannel(ctx, data, clientId, cb);
             }

www/common/sframe-common-outer.js

@@ -95,7 +95,6 @@ define([
                 // sframe-boot.js. Then we can start the channel.
                 var msgEv = _Util.mkEvent();
                 var iframe = $('#sbox-iframe')[0].contentWindow;
-                var iframeReady = false;
                 var postMsg = function (data) {
                     iframe.postMessage(data, '*');
                 };