Improve history to load it chunk by chunk

customize.dist/src/less2/include/toolbar-history.less

@@ -5,23 +5,47 @@
         display: none;
         text-align: center;
         width: 100%;
+        padding: 10px 0;
+        align-items: center;
+        justify-content: center;
         * {
             font: @colortheme_app-font;
         }
-        .cp-toolbar-history-next {
+        .cp-history-filler {
-            display: inline-block;
+            flex: 1;
-            vertical-align: middle;
-            margin: 20px;
         }
-        .cp-toolbar-history-previous {
+        .cp-toolbar-history-close,
-            display: inline-block;
+        .cp-toolbar-history-revert {
-            vertical-align: middle;
+            background: white;
-            margin: 20px;
+            color: black;
+            //margin-top: 5px;
+            &:hover {
+                background-color: #e6e6e6;
+            }
+        }
+        .cp-toolbar-history-loadmore {
+            height: 100%;
+            color: black;
+            width: 25px;
+            position: absolute;
+            left: 0;
+            padding: 0;
+        }
+        .cp-toolbar-history-version {
+            position: absolute;
+            height: 25px;
+            line-height: 25px;
+            width: 100%;
+            text-align: center;
         }
         .cp-toolbar-history-goto {
             display: inline-block;
             vertical-align: middle;
             text-align: center;
+            flex: 1;
+            flex-basis: 80%;
+            min-width: 0;
+            max-width: 600px;
             input { width: 75px; }
         }
         .cp-toolbar-history-goto-input {
@@ -29,6 +53,30 @@
             margin-left: 5px;
             vertical-align: middle;
         }
+        .cp-toolbar-history-bar {
+            width: 100%;
+            background: white;
+            height: 25px;
+            margin: auto;
+            position: relative;
+        }
+        .cp-toolbar-history-pos-container {
+            width: ~"calc(100% - 2px)";
+            height: 25px;
+            position: relative;
+        }
+        @pos-color: #55FF55;
+        .cp-toolbar-history-pos {
+            width: 2px;
+            height: 25px;
+            background: @pos-color;
+            &:after {
+                content: '';
+                border: 6px solid transparent;
+                border-top-color: @pos-color;
+                margin-left: -5px;
+            }
+        }
         button {
             color: inherit;
             background-color: rgba(0,0,0,0.2);
@@ -36,14 +84,6 @@
                 background-color: rgba(0,0,0,0.4);
             }
         }
-        .cp-toolbar-history-close {
-            background: white;
-            color: black;
-            margin-top: 5px;
-            &:hover {
-                background-color: #e6e6e6;
-            }
-        }
         .fa-spinner {
             font-size: 66px;
         }

customize.dist/translations/messages.fr.js

@@ -232,12 +232,11 @@ define(function () {
 
     out.historyText = "Historique";
     out.historyButton = "Afficher l'historique du document";
-    out.history_next = "Voir la version suivante";
+    out.history_next = "Version plus récente";
-    out.history_prev = "Voir la version précédente";
+    out.history_prev = "Version plus ancienne";
-    out.history_goTo = "Voir la version sélectionnée";
+    out.history_loadMore = "Charger davantage d'historique";
     out.history_close = "Retour";
     out.history_closeTitle = "Fermer l'historique";
-    out.history_restore = "Restaurer";
     out.history_restoreTitle = "Restaurer la version du document sélectionnée";
     out.history_restorePrompt = "Êtes-vous sûr de vouloir remplacer la version actuelle du document par la version affichée ?";
     out.history_restoreDone = "Document restauré";

customize.dist/translations/messages.js

@@ -234,12 +234,10 @@ define(function () {
 
     out.historyText = "History";
     out.historyButton = "Display the document history";
-    out.history_next = "Go to the next version";
+    out.history_next = "Newer version";
-    out.history_prev = "Go to the previous version";
+    out.history_prev = "Older version";
-    out.history_goTo = "Go to the selected version";
+    out.history_loadMore = "Load more history";
-    out.history_close = "Back";
     out.history_closeTitle = "Close the history";
-    out.history_restore = "Restore";
     out.history_restoreTitle = "Restore the selected version of the document";
     out.history_restorePrompt = "Are you sure you want to replace the current version of the document by the displayed one?";
     out.history_restoreDone = "Document restored";

www/common/cryptpad-common.js

@@ -692,6 +692,9 @@ define([
     common.getFullHistory = function (data, cb) {
         postMessage("GET_FULL_HISTORY", data, cb);
     };
+    common.getHistoryRange = function (data, cb) {
+        postMessage("GET_HISTORY_RANGE", data, cb);
+    };
 
     common.getShareHashes = function (secret, cb) {
         var hashes;

www/common/outer/async-store.js

@@ -1101,10 +1101,13 @@ define([
                 }
             };
             var msgs = [];
+            var completed = false;
             var onMsg = function (msg) {
+                if (completed) { return; }
                 var parsed = parse(msg);
                 if (parsed[0] === 'FULL_HISTORY_END') {
                     cb(msgs);
+                    completed = true;
                     return;
                 }
                 if (parsed[0] !== 'FULL_HISTORY') { return; }
@@ -1123,6 +1126,63 @@ define([
             network.sendto(hkn, JSON.stringify(['GET_FULL_HISTORY', data.channel, data.validateKey]));
         };
 
+        Store.getHistoryRange = function (clientId, data, cb) {
+            var network = store.network;
+            var hkn = network.historyKeeper;
+            var parse = function (msg) {
+                try {
+                    return JSON.parse(msg);
+                } catch (e) {
+                    return null;
+                }
+            };
+            var msgs = [];
+            var first = true;
+            var fullHistory = false;
+            var completed = false;
+            var lastKnownHash;
+            var txid = Util.uid();
+
+            var onMsg = function (msg) {
+                if (completed) { return; }
+                var parsed = parse(msg);
+                if (parsed[1] !== txid) { console.log('bad txid'); return; }
+                if (parsed[0] === 'HISTORY_RANGE_END') {
+                    cb({
+                        messages: msgs,
+                        isFull: fullHistory,
+                        lastKnownHash: lastKnownHash
+                    });
+                    completed = true;
+                    return;
+                }
+                if (parsed[0] !== 'HISTORY_RANGE') { return; }
+                if (parsed[2] && parsed[1].validateKey) { // Metadata
+                    return;
+                }
+                if (parsed[2][3] !== data.channel) { return; }
+                msg = parsed[2][4];
+                if (msg) {
+                    if (first) {
+                        // If the first message if not a checkpoint, it means it is the first
+                        // message of the pad, so we have the full history!
+                        if (!/^cp\|/.test(msg)) { fullHistory = true; }
+                        lastKnownHash = msg.slice(0,64);
+                        first = false;
+                    }
+                    msg = msg.replace(/cp\|(([A-Za-z0-9+\/=]+)\|)?/, '');
+                    msgs.push(msg);
+                }
+            };
+
+            network.on('message', onMsg);
+            network.sendto(hkn, JSON.stringify(['GET_HISTORY_RANGE', data.channel, {
+                from: data.lastKnownHash,
+                cpCount: 2,
+                txid: txid
+            }]));
+        };
+
         // Drive
         Store.userObjectCommand = function (clientId, cmdData, cb) {
             if (!cmdData || !cmdData.cmd) { return; }

www/common/outer/store-rpc.js

@@ -70,6 +70,7 @@ define([
             JOIN_PAD: Store.joinPad,
             LEAVE_PAD: Store.leavePad,
             GET_FULL_HISTORY: Store.getFullHistory,
+            GET_HISTORY_RANGE: Store.getHistoryRange,
             IS_NEW_CHANNEL: Store.isNewChannel,
             // Drive
             DRIVE_USEROBJECT: Store.userObjectCommand,

www/common/sframe-common-history.js

@@ -1,26 +1,36 @@
 define([
     'jquery',
     '/common/common-interface.js',
+    '/bower_components/nthen/index.js',
     //'/bower_components/chainpad-json-validator/json-ot.js',
 
     '/bower_components/chainpad/chainpad.dist.js',
-], function ($, UI, ChainPad /* JsonOT */) {
+], function ($, UI, nThen, ChainPad /* JsonOT */) {
     //var ChainPad = window.ChainPad;
     var History = {};
 
-    var getStates = function (rt) {
+    History.create = function (common, config) {
-        var states = [];
+        if (!config.$toolbar) { return void console.error("config.$toolbar is undefined");}
-        var b = rt.getAuthBlock();
+        if (History.loading) { return void console.error("History is already being loaded..."); }
-        if (b) { states.unshift(b); }
+        History.loading = true;
-        while (b.getParent()) {
+        var $toolbar = config.$toolbar;
-            b = b.getParent();
-            states.unshift(b);
-        }
-        return states;
-    };
 
-    var loadHistory = function (config, common, cb) {
+        if (!config.applyVal || !config.setHistory || !config.onLocal || !config.onRemote) {
-        var createRealtime = function () {
+            throw new Error("Missing config element: applyVal, onLocal, onRemote, setHistory");
+        }
+
+        var getStates = function (rt) {
+            var states = [];
+            var b = rt.getAuthBlock();
+            if (b) { states.unshift(b); }
+            while (b.getParent()) {
+                b = b.getParent();
+                states.unshift(b);
+            }
+            return states;
+        };
+
+        var createRealtime = function (config) {
             return ChainPad.create({
                 userName: 'history',
                 validateContent: function (content) {
@@ -33,36 +43,45 @@ define([
                     }
                 },
                 initialState: '',
-                //patchTransformer: ChainPad.NaiveJSONTransformer,
-                //logLevel: 0,
-                //transformFunction: JsonOT.validate,
                 logLevel: config.debug ? 2 : 0,
                 noPrune: true
             });
         };
-        var realtime = createRealtime();
 
-        History.readOnly = common.getMetadataMgr().getPrivateData().readOnly;
+        var loadFullHistory = function (config, common, cb) {
+            var realtime = createRealtime(config);
+            common.getFullHistory(realtime, function () {
+                cb(null, realtime);
+            });
+        };
+        loadFullHistory = loadFullHistory;
 
-        /*var to = window.setTimeout(function () {
+        var fillChainPad = function (realtime, messages) {
-            cb('[GET_FULL_HISTORY_TIMEOUT]');
+            messages.forEach(function (m) {
-        }, 30000);*/
+                realtime.message(m);
+            });
+        };
 
-        common.getFullHistory(realtime, function () {
+        var allMessages = [];
-            //window.clearTimeout(to);
+        var lastKnownHash;
-            cb(null, realtime);
+        var isComplete = false;
-        });
+        var loadMoreHistory = function (config, common, cb) {
-    };
+            if (isComplete) { return void cb ('EFULL'); }
+            var realtime = createRealtime(config);
+            var sframeChan = common.getSframeChannel();
 
-    History.create = function (common, config) {
+            sframeChan.query('Q_GET_HISTORY_RANGE', {
-        if (!config.$toolbar) { return void console.error("config.$toolbar is undefined");}
+                lastKnownHash: lastKnownHash
-        if (History.loading) { return void console.error("History is already being loaded..."); }
+            }, function (err, data) {
-        History.loading = true;
+                if (err) { return void console.error(err); }
-        var $toolbar = config.$toolbar;
+                if (!Array.isArray(data.messages)) { return void console.error('Not an array!'); }
-
+                lastKnownHash = data.lastKnownHash;
-        if (!config.applyVal || !config.setHistory || !config.onLocal || !config.onRemote) {
+                isComplete = data.isFull;
-            throw new Error("Missing config element: applyVal, onLocal, onRemote, setHistory");
+                Array.prototype.unshift.apply(allMessages, data.messages); // Destructive concat
-        }
+                fillChainPad(realtime, allMessages);
+                cb (null, realtime);
+            });
+        };
 
         // config.setHistory(bool, bool)
         // - bool1: history value
@@ -84,21 +103,20 @@ define([
         };
 
         config.setHistory(true);
-        var onReady = function () { };
 
         var Messages = common.Messages;
 
         var realtime;
 
         var states = [];
-        var c = states.length - 1;
+        var c = 0;//states.length - 1;
 
         var $hist = $toolbar.find('.cp-toolbar-history');
         var $left = $toolbar.find('.cp-toolbar-leftside');
         var $right = $toolbar.find('.cp-toolbar-rightside');
         var $cke = $toolbar.find('.cke_toolbox_main');
 
-        $hist.html('').show();
+        $hist.html('').css('display', 'flex');
         $left.hide();
         $right.hide();
         $cke.hide();
@@ -107,29 +125,73 @@ define([
 
         var onUpdate;
 
-        var update = function () {
+        var update = function (newRt) {
+            realtime = newRt;
             if (!realtime) { return []; }
             states = getStates(realtime);
             if (typeof onUpdate === "function") { onUpdate(); }
             return states;
         };
 
+        var $loadMore, $version, get;
+
         // Get the content of the selected version, and change the version number
-        var get = function (i) {
+        var loading = false;
+        var loadMore = function (cb) {
+            if (loading) { return; }
+            loading = true;
+            $loadMore.removeClass('fa fa-ellipsis-h')
+                .append($('<span>', {'class': 'fa fa-refresh fa-spin fa-3x fa-fw'}));
+            loadMoreHistory(config, common, function (err, newRt) {
+                if (err === 'EFULL') {
+                    $loadMore.off('click').hide();
+                    get(c);
+                    $version.show();
+                    return;
+                }
+                loading = false;
+                if (err) { return void console.error(err); }
+                update(newRt);
+                $loadMore.addClass('fa fa-ellipsis-h').html('');
+                get(c);
+                if (cb) { cb(); }
+            });
+        };
+        get = function (i) {
             i = parseInt(i);
             if (isNaN(i)) { return; }
-            if (i < 0) { i = 0; }
+            if (i > 0) { i = 0; }
-            if (i > states.length - 1) { i = states.length - 1; }
+            if (i < -(states.length - 2)) { i = -(states.length - 2); }
-            var val = states[i].getContent().doc;
+            if (i <= -(states.length - 11)) {
+                loadMore();
+            }
+            var idx = states.length - 1 + i;
+            var val = states[idx].getContent().doc;
             c = i;
             if (typeof onUpdate === "function") { onUpdate(); }
-            $hist.find('.cp-toolbar-history-next, .cp-toolbar-history-previous').css('visibility', '');
+            $hist.find('.cp-toolbar-history-next, .cp-toolbar-history-previous, ' +
-            if (c === states.length - 1) { $hist.find('.cp-toolbar-history-next').css('visibility', 'hidden'); }
+                       '.cp-toolbar-history-fast-next, .cp-toolbar-history-fast-previous')
-            if (c === 0) { $hist.find('.cp-toolbar-history-previous').css('visibility', 'hidden'); }
+                .css('visibility', '');
+            if (c === -(states.length-1)) {
+                $hist.find('.cp-toolbar-history-previous').css('visibility', 'hidden');
+                $hist.find('.cp-toolbar-history-fast-previous').css('visibility', 'hidden');
+            }
+            if (c === 0) {
+                $hist.find('.cp-toolbar-history-next').css('visibility', 'hidden');
+                $hist.find('.cp-toolbar-history-fast-next').css('visibility', 'hidden');
+            }
+            var $pos = $hist.find('.cp-toolbar-history-pos');
+            var p = 100 * (1 - (-c / (states.length-1)));
+            $pos.css('margin-left', p+'%');
+
+            // Display the version when the full history is loaded
+            // Note: the first version is always empty and probably can't be displayed, so
+            // we can consider we have only states.length - 1 versions
+            $version.text(idx + ' / ' + (states.length-1));
 
             if (config.debug) {
-                console.log(states[i]);
+                console.log(states[idx]);
-                var ops = states[i] && states[i].getPatch() && states[i].getPatch().operations;
+                var ops = states[idx] && states[idx].getPatch() && states[idx].getPatch().operations;
                 if (Array.isArray(ops)) {
                     ops.forEach(function (op) { console.log(op); });
                 }
@@ -148,6 +210,17 @@ define([
         // Create the history toolbar
         var display = function () {
             $hist.html('');
+
+            var $rev = $('<button>', {
+                'class':'cp-toolbar-history-revert buttonSuccess fa fa-check-circle-o',
+                title: Messages.history_restoreTitle
+            }).appendTo($hist);//.text(Messages.history_restore);
+            if (History.readOnly) { $rev.css('visibility', 'hidden'); }
+            $('<span>', {'class': 'cp-history-filler'}).appendTo($hist);
+            var $fastPrev = $('<button>', {
+                'class': 'cp-toolbar-history-fast-previous fa fa-fast-backward buttonPrimary',
+                title: Messages.history_prev
+            }).appendTo($hist);
             var $prev =$('<button>', {
                 'class': 'cp-toolbar-history-previous fa fa-step-backward buttonPrimary',
                 title: Messages.history_prev
@@ -157,58 +230,73 @@ define([
                 'class': 'cp-toolbar-history-next fa fa-step-forward buttonPrimary',
                 title: Messages.history_next
             }).appendTo($hist);
-
+            var $fastNext = $('<button>', {
-            $('<label>').text(Messages.history_version).appendTo($nav);
+                'class': 'cp-toolbar-history-fast-next fa fa-fast-forward buttonPrimary',
-            var $cur = $('<input>', {
+                title: Messages.history_next
-                'class' : 'cp-toolbar-history-goto-input',
+            }).appendTo($hist);
-                'type' : 'number',
+            $('<span>', {'class': 'cp-history-filler'}).appendTo($hist);
-                'min' : '1',
-                'max' : states.length
-            }).val(c + 1).appendTo($nav).mousedown(function (e) {
-                // stopPropagation because the event would be cancelled by the dropdown menus
-                e.stopPropagation();
-            });
-            var $label2 = $('<label>').text(' / '+ states.length).appendTo($nav);
-            $('<br>').appendTo($nav);
             var $close = $('<button>', {
-                'class':'cp-toolbar-history-close',
+                'class':'cp-toolbar-history-close fa fa-window-close',
                 title: Messages.history_closeTitle
-            }).text(Messages.history_closeTitle).appendTo($nav);
+            }).appendTo($hist);
-            var $rev = $('<button>', {
+
-                'class':'cp-toolbar-history-revert buttonSuccess',
+            var $bar = $('<div>', {'class': 'cp-toolbar-history-bar'}).appendTo($nav);
-                title: Messages.history_restoreTitle
+            var $container = $('<div>', {'class':'cp-toolbar-history-pos-container'}).appendTo($bar);
-            }).text(Messages.history_restore).appendTo($nav);
+            $('<div>', {'class': 'cp-toolbar-history-pos'}).appendTo($container);
-            if (History.readOnly) { $rev.hide(); }
+
+            $version = $('<span>', {
+                'class': 'cp-toolbar-history-version'
+            }).prependTo($bar).hide();
+            $loadMore = $('<button>', {
+                'class':'cp-toolbar-history-loadmore fa fa-ellipsis-h',
+                title: Messages.history_loadMore
+            }).click(function () {
+                loadMore(function () {
+                    get(c);
+                });
+            }).prependTo($container);
+
+            // Load a version when clicking on the bar
+            $container.click(function (e) {
+                e.stopPropagation();
+                if (!$(e.target).is('.cp-toolbar-history-pos-container')) { return; }
+                var p = e.offsetX / $container.width();
+                var v = -Math.round((states.length - 1) * (1 - p));
+                render(get(v));
+            });
 
             onUpdate = function () {
-                $cur.attr('max', states.length);
+                // Called when a new version is loaded
-                $cur.val(c+1);
-                $label2.text(' / ' + states.length);
             };
 
+            var onKeyDown, onKeyUp;
             var close = function () {
                 $hist.hide();
                 $left.show();
                 $right.show();
                 $cke.show();
                 $(window).trigger('resize');
+                $(window).off('keydown', onKeyDown);
+                $(window).off('keyup', onKeyUp);
             };
 
-            // Buttons actions
+            // Version buttons
             $prev.click(function () { render(getPrevious()); });
             $next.click(function () { render(getNext()); });
-            $cur.keydown(function (e) {
+            $fastPrev.click(function () { render(getPrevious(10)); });
+            $fastNext.click(function () { render(getNext(10)); });
+            onKeyDown = function (e) {
                 var p = function () { e.preventDefault(); };
-                if (e.which === 13) { p(); return render( get($cur.val() - 1) ); } // Enter
                 if ([37, 40].indexOf(e.which) >= 0) { p(); return render(getPrevious()); } // Left
                 if ([38, 39].indexOf(e.which) >= 0) { p(); return render(getNext()); } // Right
                 if (e.which === 33) { p(); return render(getNext(10)); } // PageUp
                 if (e.which === 34) { p(); return render(getPrevious(10)); } // PageUp
                 if (e.which === 27) { p(); $close.click(); }
-            }).keyup(function (e) { e.stopPropagation(); }).focus();
+            };
-            $cur.on('change', function () {
+            onKeyUp = function (e) { e.stopPropagation(); };
-                render( get($cur.val() - 1) );
+            $(window).on('keydown', onKeyDown).on('keyup', onKeyUp).focus();
-            });
+
+            // Close & restore buttons
             $close.click(function () {
                 states = [];
                 close();
@@ -229,14 +317,13 @@ define([
         };
 
         // Load all the history messages into a new chainpad object
-        loadHistory(config, common, function (err, newRt) {
+        loadMoreHistory(config, common, function (err, newRt) {
+            History.readOnly = common.getMetadataMgr().getPrivateData().readOnly;
             History.loading = false;
             if (err) { throw new Error(err); }
-            realtime = newRt;
+            update(newRt);
-            update();
             c = states.length - 1;
             display();
-            onReady();
         });
     };
 

www/common/sframe-common-outer.js

@@ -420,6 +420,24 @@ define([
                     }));
                 });
             });
+            sframeChan.on('Q_GET_HISTORY_RANGE', function (data, cb) {
+                var crypto = Crypto.createEncryptor(secret.keys);
+                Cryptpad.getHistoryRange({
+                    channel: secret.channel,
+                    validateKey: secret.keys.validateKey,
+                    lastKnownHash: data.lastKnownHash
+                }, function (data) {
+                    cb({
+                        isFull: data.isFull,
+                        messages: data.messages.map(function (msg) {
+                            // The 3rd parameter "true" means we're going to skip signature validation.
+                            // We don't need it since the message is already validated serverside by hk
+                            return crypto.decrypt(msg, true, true);
+                        }),
+                        lastKnownHash: data.lastKnownHash
+                    });
+                });
+            });
 
             sframeChan.on('Q_GET_PAD_ATTRIBUTE', function (data, cb) {
                 var href;

www/common/sframe-protocol.js

@@ -87,6 +87,7 @@ define({
     // Request the full history from the server when the users clicks on the history button.
     // Callback is called when the FULL_HISTORY_END message is received in the outside.
     'Q_GET_FULL_HISTORY': true,
+    'Q_GET_HISTORY_RANGE': true,
     // When a (full) history message is received from the server.
     'EV_RT_HIST_MESSAGE': true,