Add the latest changes from _socket into the netflux pad

www/common/realtime-input.js

@@ -20,10 +20,10 @@ define([
     '/common/netflux.js',
     '/common/crypto.js',
     '/common/toolbar.js',
-    '/common/sharejs_textarea.js',
+    '/_socket/text-patcher.js',
     '/common/chainpad.js',
     '/bower_components/jquery/dist/jquery.min.js',
-], function (Messages, Netflux, Crypto, Toolbar, sharejs) {
+], function (Messages, Netflux, Crypto, Toolbar, TextPatcher) {
     var $ = window.jQuery;
     var ChainPad = window.ChainPad;
     var PARANOIA = true;
@@ -61,25 +61,6 @@ define([
         }
     };
 
-    var bindAllEvents = function (textarea, docBody, onEvent, unbind)
-    {
-        /*
-            we use docBody for the purposes of CKEditor.
-            because otherwise special keybindings like ctrl-b and ctrl-i
-            would open bookmarks and info instead of applying bold/italic styles
-        */
-        if (docBody) {
-            bindEvents(docBody,
-               ['textInput', 'keydown', 'keyup', 'select', 'cut', 'paste'],
-               onEvent,
-               unbind);
-        }
-        bindEvents(textarea,
-                   ['mousedown','mouseup','click','change'],
-                   onEvent,
-                   unbind);
-    };
-
     var getParameterByName = function (name, url) {
         if (!url) { url = window.location.href; }
         name = name.replace(/[\[\]]/g, "\\$&");
@@ -93,7 +74,6 @@ define([
     var start = module.exports.start =
         function (config)
     {
-        var textarea = config.textarea;
         var websocketUrl = config.websocketURL;
         var webrtcUrl = config.webrtcURL;
         var userName = config.userName;
@@ -106,20 +86,17 @@ define([
 
         var doc = config.doc || null;
 
-        // trying to deprecate onRemote, prefer loading it via the conf
-        var onRemote = config.onRemote || null;
-
-        // define this in case it gets called before the rest of our stuff is ready.
-        var onEvent = function () { };
-
         var allMessages = [];
         var initializing = true;
         var recoverableErrorCount = 0;
-        var bump = function () {};
+        var toReturn = {};
         var messagesHistory = [];
         var chainpadAdapter = {};
         var realtime;
         
+        // define this in case it gets called before the rest of our stuff is ready.
+        var onEvent = toReturn.onEvent = function (newText) { };
+
         var parseMessage = function (msg) {
             var res ={};
             // two or more? use a for
@@ -167,11 +144,11 @@ define([
 
             verbose(message);
             allMessages.push(message);
-            if (!initializing) {
+            // if (!initializing) {
-                if (PARANOIA) {
+                // if (toReturn.onLocal) {
-                    onEvent();
+                    // toReturn.onLocal();
-                }
+                // }
-            }
+            // }
             realtime.message(message);
             if (/\[5,/.test(message)) { verbose("pong"); }
 
@@ -183,7 +160,11 @@ define([
                     } else {
                         //verbose("Received remote message");
                         // obviously this is only going to get called if
-                        if (onRemote) { onRemote(realtime.getUserDoc()); }
+                        if (config.onRemote) { 
+                            config.onRemote({
+                                realtime: realtime
+                            }); 
+                        }
                     }
                 }
             }
@@ -263,7 +244,7 @@ define([
             return ChainPad.create(userName,
                                         passwd,
                                         channel,
-                                        $(textarea).val(),
+                                        config.initialState || {},
                                         {
                                         transformFunction: config.transformFunction
                                         });
@@ -286,7 +267,9 @@ define([
 
             // execute an onReady callback if one was supplied
             if (config.onReady) {
-                config.onReady();
+                config.onReady({
+                    realtime: realtime
+                });
             }
         }
 
@@ -334,16 +317,10 @@ define([
               hc.send(JSON.stringify(['GET_HISTORY', wc.id]));
             }
 
-            // Check the connection to the channel
-            if(!rtc) {
-              // TODO
-              // checkConnection(wc);
-            }
             
-            bindAllEvents(textarea, doc, onEvent, false);
+            toReturn.patchText = TextPatcher.create({
-
+                realtime: realtime
-            sharejs.attach(textarea, realtime);
+            });
-            bump = realtime.bumpSharejs;
 
             realtime.start();
         };
@@ -401,12 +378,7 @@ define([
             }
         };
 
-        return {
+        return toReturn;
-            onEvent: function () {
-                onEvent();
-            },
-            bumpSharejs: function () { bump(); }
-        };
     };
     return module.exports;
 });

www/pad/index.html

@@ -38,6 +38,12 @@
             right: 0px;
             top: 70px;
         }
+        #debug button {
+            visibility: hidden;
+        }
+        #debug:hover button {
+            visibility: visible;
+        }
     </style>
 </head>
 <body>

www/pad/main.js

@@ -3,24 +3,46 @@ define([
     '/common/messages.js',
     '/common/crypto.js',
     '/common/realtime-input.js',
-    '/common/convert.js',
+    '/common/hyperjson.js',
+    '/common/hyperscript.js',
     '/common/toolbar.js',
     '/common/cursor.js',
     '/common/json-ot.js',
     '/bower_components/diff-dom/diffDOM.js',
     '/bower_components/jquery/dist/jquery.min.js',
     '/customize/pad.js'
-], function (Config, Messages, Crypto, realtimeInput, Convert, Toolbar, Cursor, JsonOT) {
+], function (Config, Messages, Crypto, realtimeInput, Hyperjson, Hyperscript, Toolbar, Cursor, JsonOT) {
     var $ = window.jQuery;
     var ifrw = $('#pad-iframe')[0].contentWindow;
     var Ckeditor; // to be initialized later...
     var DiffDom = window.diffDOM;
 
     window.Toolbar = Toolbar;
+    window.Hyperjson = Hyperjson;
+
+    var hjsonToDom = function (H) {
+        return Hyperjson.callOn(H, Hyperscript);
+    };
+
+    var module = window.REALTIME_MODULE = {
+        localChangeInProgress: 0
+    };
 
     var userName = Crypto.rand64(8),
         toolbar;
 
+    var isNotMagicLine = function (el) {
+        // factor as:
+        // return !(el.tagName === 'SPAN' && el.contentEditable === 'false');
+        var filter = (el.tagName === 'SPAN' && el.contentEditable === 'false');
+        if (filter) {
+            console.log("[hyperjson.serializer] prevented an element" +
+                "from being serialized:", el);
+            return false;
+        }
+        return true;
+    };
+
     var andThen = function (Ckeditor) {
         $(window).on('hashchange', function() {
             window.location.reload();
@@ -39,7 +61,7 @@ define([
             removeButtons: 'Source,Maximize',
             // magicline plugin inserts html crap into the document which is not part of the
             // document itself and causes problems when it's sent across the wire and reflected back
-            removePlugins: 'magicline,resize'
+            removePlugins: 'resize'
         });
 
         editor.on('instanceReady', function (Ckeditor) {
@@ -51,8 +73,6 @@ define([
             var inner = window.inner = documentBody;
             var cursor = window.cursor = Cursor(inner);
 
-            var $textarea = $('#feedback');
-
             var setEditable = function (bool) {
                 inner.setAttribute('contenteditable',
                     (typeof (bool) !== 'undefined'? bool : true));
@@ -63,6 +83,31 @@ define([
 
             var diffOptions = {
                 preDiffApply: function (info) {
+                    /* DiffDOM will filter out magicline plugin elements
+                        in practice this will make it impossible to use it
+                        while someone else is typing, which could be annoying.
+
+                        we should check when such an element is going to be
+                        removed, and prevent that from happening. */
+                    if (info.node && info.node.tagName === 'SPAN' &&
+                        info.node.contentEditable === "true") {
+                        // it seems to be a magicline plugin element...
+                        if (info.diff.action === 'removeElement') {
+                            // and you're about to remove it...
+                            // this probably isn't what you want
+
+                            /*
+                                I have never seen this in the console, but the
+                                magic line is still getting removed on remote
+                                edits. This suggests that it's getting removed
+                                by something other than diffDom.
+                            */
+                            console.log("preventing removal of the magic line!");
+
+                            // return true to prevent diff application
+                            return true;
+                        }
+                    }
 
                     // no use trying to recover the cursor if it doesn't exist
                     if (!cursor.exists()) { return; }
@@ -74,21 +119,8 @@ define([
 
                     if (!frame) { return; }
 
-                    var debug = info.debug = {
-                        frame: frame,
-                        action: info.diff.action,
-                        cursorLength: cursor.getLength(),
-                        node: info.node
-                    };
-
-                    if (info.diff.oldValue) { debug.oldValue = info.diff.oldValue; }
-                    if (info.diff.newValue) { debug.newValue = info.diff.newValue; }
                     if (typeof info.diff.oldValue === 'string' && typeof info.diff.newValue === 'string') {
                         var pushes = cursor.pushDelta(info.diff.oldValue, info.diff.newValue);
-                        debug.commonStart = pushes.commonStart;
-                        debug.commonEnd = pushes.commonEnd;
-                        debug.insert = pushes.insert;
-                        debug.remove = pushes.remove;
 
                         if (frame & 1) {
                             // push cursor start if necessary
@@ -103,8 +135,6 @@ define([
                             }
                         }
                     }
-                    console.log("###################################");
-                    console.log(debug);
                 },
                 postDiffApply: function (info) {
                     if (info.frame) {
@@ -121,6 +151,8 @@ define([
                 }
             };
 
+            var now = function () { return new Date().getTime(); };
+
             var initializing = true;
             var userList = {}; // List of pretty name of all users (mapped with their server ID)
             var toolbarList; // List of users still connected to the channel (server IDs)
@@ -147,7 +179,7 @@ define([
                    if (newName && newName.trim()) {
                        var myUserNameTemp = newName.trim();
                        if(newName.trim().length > 32) {
-                         myUserNameTemp = myUserNameTemp.substr(0, 31);
+                         myUserNameTemp = myUserNameTemp.substr(0, 32);
                        }
                        myUserName = myUserNameTemp;
                        myData[myID] = {
@@ -159,65 +191,26 @@ define([
                 });
             };
 
+            var DD = new DiffDom(diffOptions);
+
             // apply patches, and try not to lose the cursor in the process!
             var applyHjson = function (shjson) {
-                var hjson = JSON.parse(shjson);
+                // var hjson = JSON.parse(shjson);
-                var peerUserList = hjson[hjson.length-1];
+                // var peerUserList = hjson[hjson.length-1];
-                if(peerUserList.metadata) {
+                // if(peerUserList.metadata) {
-                  var userData = peerUserList.metadata;
+                  // var userData = peerUserList.metadata;
-                  addToUserList(userData);
+                  // addToUserList(userData);
-                  delete hjson[hjson.length-1];
+                  // delete hjson[hjson.length-1];
-                }
+                // }
-                var userDocStateDom = Convert.hjson.to.dom(hjson);
+                var userDocStateDom = hjsonToDom(JSON.parse(shjson));
                 userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf
-                var DD = new DiffDom(diffOptions);
                 var patch = (DD).diff(inner, userDocStateDom);
                 (DD).apply(inner, patch);
             };
 
-            var onRemote = function (shjson) {
-                if (initializing) { return; }
-
-                // remember where the cursor is
-                cursor.update();
-
-                // build a dom from HJSON, diff, and patch the editor
-                applyHjson(shjson);
-            };
-
-            var onInit = function (info) {
-                var $bar = $('#pad-iframe')[0].contentWindow.$('#cke_1_toolbox');
-                toolbarList = info.userList;
-                var config = {
-                    userData: userList,
-                    changeNameID: 'cryptpad-changeName'
-                };
-                toolbar = info.realtime.toolbar = Toolbar.create($bar, info.myID, info.realtime, info.webChannel, info.userList, config);
-                createChangeName('cryptpad-changeName', $bar);
-                /* TODO handle disconnects and such*/
-            };
-
-            var onReady = function (info) {
-                console.log("Unlocking editor");
-                initializing = false;
-                setEditable(true);
-                applyHjson($textarea.val());
-                $textarea.trigger('keyup');
-            };
-
-            var onAbort = function (info) {
-                console.log("Aborting the session!");
-                // stop the user from continuing to edit
-                setEditable(false);
-                // TODO inform them that the session was torn down
-                toolbar.failed();
-            };
-
-            
-            
             var realtimeOptions = {
-                // the textarea that we will sync
+                // provide initialstate...
-                textarea: $textarea[0],
+                initialState: JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine)),
 
                 // the websocket URL (deprecated?)
                 websocketURL: Config.websocketURL,
@@ -234,36 +227,136 @@ define([
 
                 // configuration :D
                 doc: inner,
-                // first thing called
-                onInit: onInit,
-
-                onReady: onReady,
 
                 setMyID: setMyID,
 
-                // when remote changes occur
-                onRemote: onRemote,
-
-                // handle aborts
-                onAbort: onAbort,
-
                 // really basic operational transform
                 transformFunction : JsonOT.validate
                 // pass in websocket/netflux object TODO
             };
 
-            var rti = window.rti = realtimeInput.start(realtimeOptions);
+            var onRemote = realtimeOptions.onRemote = function (info) {
+                if (initializing) { return; }
 
-            $textarea.val(JSON.stringify(Convert.dom.to.hjson(inner)));
+                var shjson = info.realtime.getUserDoc();
 
-            editor.on('change', function () {
+                // remember where the cursor is
-                var hjson = Convert.core.hyperjson.fromDOM(inner);
+                cursor.update();
-                if(myData !== {}) {
+
+                // Extract the user list (metadata) from the hyperjson
+                var hjson = JSON.parse(shjson);
+                var peerUserList = hjson[hjson.length-1];
+                if(peerUserList.metadata) {
+                  var userData = peerUserList.metadata;
+                  // Update the local user data
+                  userList = userData;
+                  // Send the new data to the toolbar
+                  if(toolbarList && typeof toolbarList.onChange === "function") {
+                    toolbarList.onChange(userList);
+                  }
+                  hjson.pop();
+                }
+
+                // build a dom from HJSON, diff, and patch the editor
+                applyHjson(shjson);
+
+                // Build a new stringified Chainpad hyperjson without metadata to compare with the one build from the dom
+                shjson = JSON.stringify(hjson);
+
+                var hjson2 = Hyperjson.fromDOM(inner);
+                var shjson2 = JSON.stringify(hjson2);
+                if (shjson2 !== shjson) {
+                    console.error("shjson2 !== shjson");
+                    module.realtimeInput.patchText(shjson2);
+                }
+            };
+
+            var onInit = realtimeOptions.onInit = function (info) {
+                var $bar = $('#pad-iframe')[0].contentWindow.$('#cke_1_toolbox');
+                toolbarList = info.userList;
+                var config = {
+                    userData: userList,
+                    changeNameID: 'cryptpad-changeName'
+                };
+                toolbar = info.realtime.toolbar = Toolbar.create($bar, info.myID, info.realtime, info.webChannel, info.userList, config);
+                createChangeName('cryptpad-changeName', $bar);
+                /* TODO handle disconnects and such*/
+            };
+
+            var onReady = realtimeOptions.onReady = function (info) {
+                console.log("Unlocking editor");
+                initializing = false;
+                setEditable(true);
+                var shjson = info.realtime.getUserDoc();
+                applyHjson(shjson);
+            };
+
+            var onAbort = realtimeOptions.onAbort = function (info) {
+                console.log("Aborting the session!");
+                // stop the user from continuing to edit
+                setEditable(false);
+                // TODO inform them that the session was torn down
+                toolbar.failed();
+            };
+
+
+
+
+
+            var rti = module.realtimeInput = realtimeInput.start(realtimeOptions);
+
+            /* catch `type="_moz"` before it goes over the wire */
+            var brFilter = function (hj) {
+                if (hj[1].type === '_moz') { hj[1].type = undefined; }
+                return hj;
+            };
+
+            // $textarea.val(JSON.stringify(Convert.dom.to.hjson(inner)));
+
+            /*  It's incredibly important that you assign 'rti.onLocal'
+                It's used inside of realtimeInput to make sure that all changes
+                make it into chainpad.
+
+                It's being assigned this way because it can't be passed in, and
+                and can't be easily returned from realtime input without making
+                the code less extensible.
+            */
+            var propogate = rti.onLocal = function () {
+                /*  if the problem were a matter of external patches being
+                    applied while a local patch were in progress, then we would
+                    expect to be able to check and find
+                    'module.localChangeInProgress' with a non-zero value while
+                    we were applying a remote change.
+                */
+                var hjson = Hyperjson.fromDOM(inner, isNotMagicLine, brFilter);
+                if(Object.keys(myData).length > 0) {
                     hjson[hjson.length] = {metadata: userList};
                 }
-                $textarea.val(JSON.stringify(hjson));
+                var shjson = JSON.stringify(hjson);
-                rti.bumpSharejs();
+                if (!rti.patchText(shjson)) {
-            });
+                    return;
+                }
+                rti.onEvent(shjson);
+            };
+
+            /* hitting enter makes a new line, but places the cursor inside
+                of the <br> instead of the <p>. This makes it such that you
+                cannot type until you click, which is rather unnacceptable.
+                If the cursor is ever inside such a <br>, you probably want
+                to push it out to the parent element, which ought to be a
+                paragraph tag. This needs to be done on keydown, otherwise
+                the first such keypress will not be inserted into the P. */
+            inner.addEventListener('keydown', cursor.brFix);
+
+            editor.on('change', propogate);
+            // editor.on('change', function () {
+                // var hjson = Convert.core.hyperjson.fromDOM(inner);
+                // if(myData !== {}) {
+                    // hjson[hjson.length] = {metadata: userList};
+                // }
+                // $textarea.val(JSON.stringify(hjson));
+                // rti.bumpSharejs();
+            // });
         });
     };