New drive structure without file names

www/assert/main.js

@@ -4,7 +4,8 @@ define([
     '/bower_components/textpatcher/TextPatcher.amd.js',
     'json.sortify',
     '/common/cryptpad-common.js',
-], function ($, Hyperjson, TextPatcher, Sortify, Cryptpad) {
+    '/drive/tests.js'
+], function ($, Hyperjson, TextPatcher, Sortify, Cryptpad, Drive) {
     window.Hyperjson = Hyperjson;
     window.TextPatcher = TextPatcher;
     window.Sortify = Sortify;
@@ -204,6 +205,9 @@ define([
         return cb(true);
     }, "version 2 hash failed to parse correctly");
 
+
+    Drive.test(assert);
+
     var swap = function (str, dict) {
         return str.replace(/\{\{(.*?)\}\}/g, function (all, key) {
             return typeof dict[key] !== 'undefined'? dict[key] : all;

www/common/userObject.js

@@ -8,6 +8,7 @@ define([
     var UNSORTED = module.UNSORTED = "unsorted";
     var TRASH = module.TRASH = "trash";
     var TEMPLATE = module.TEMPLATE = "template";
+    var NEW_FILES_DATA = 'filesData';
 
     module.init = function (files, config) {
         var exp = {};
@@ -54,7 +55,7 @@ define([
         var compareFiles = function (fileA, fileB) { return fileA === fileB; };
 
         var isFile = exp.isFile = function (element) {
-            return typeof(element) === "string";
+            return typeof(element) === "number" || typeof(element) === "string";
         };
 
         exp.isReadOnlyFile = function (element) {
@@ -247,6 +248,7 @@ define([
         };
         _getFiles[FILES_DATA] = function () {
             var ret = [];
+            if (!files[FILES_DATA]) { return ret; }
             files[FILES_DATA].forEach(function (el) {
                 if (el.href && ret.indexOf(el.href) === -1) {
                     ret.push(el.href);
@@ -254,10 +256,21 @@ define([
             });
             return ret;
         };
+        _getFiles[NEW_FILES_DATA] = function () {
+            var ret = [];
+            if (!files[NEW_FILES_DATA]) { return ret; }
+            for (var k in files[NEW_FILES_DATA]) {
+                var el = files[NEW_FILES_DATA][k];
+                if (el.href && ret.indexOf(el.href) === -1) {
+                    ret.push(el.href);
+                }
+            };
+            return ret;
+        };
         var getFiles = exp.getFiles = function (categories) {
             var ret = [];
             if (!categories || !categories.length) {
-                categories = [ROOT, 'hrefArray', TRASH, FILES_DATA];
+                categories = [ROOT, 'hrefArray', TRASH, FILES_DATA, NEW_FILES_DATA];
             }
             categories.forEach(function (c) {
                 if (typeof _getFiles[c] === "function") {
@@ -298,6 +311,7 @@ define([
             return _findFileInRoot([ROOT], href);
         };
         var _findFileInHrefArray = function (rootName, href) {
+            if (!files[rootName]) { return []; }
             var unsorted = files[rootName].slice();
             var ret = [];
             var i = -1;
@@ -833,8 +847,19 @@ define([
                         debug("An element in ROOT was not a folder nor a file. ", element[el]);
                         element[el] = undefined;
                         delete element[el];
-                    } else if (isFolder(element[el])) {
+                        continue;
+                    }
+                    if (isFolder(element[el])) {
                         fixRoot(element[el]);
+                        continue;
+                    }
+                    if (typeof element[el] === "string") {
+                        // We have an old file (href) which is not in filesData: add it
+                        var id = Cryptpad.createRandomInteger();
+                        var key = Cryptpad.createChannelId();
+                        files[NEW_FILES_DATA][id] = {href: element[el], filename: el};
+                        element[key] = id;
+                        delete element[el];
                     }
                 }
             };
@@ -842,10 +867,17 @@ define([
                 if (typeof(files[TRASH]) !== "object") { debug("TRASH was not an object"); files[TRASH] = {}; }
                 var tr = files[TRASH];
                 var toClean;
-                var addToClean = function (obj, idx) {
+                var addToClean = function (obj, idx, el) {
                     if (typeof(obj) !== "object") { toClean.push(idx); return; }
                     if (!isFile(obj.element) && !isFolder(obj.element)) { toClean.push(idx); return; }
                     if (!$.isArray(obj.path)) { toClean.push(idx); return; }
+                    if (typeof obj.element === "string") {
+                        // We have an old file (href) which is not in filesData: add it
+                        var id = Cryptpad.createRandomInteger();
+                        files[NEW_FILES_DATA][id] = {href: obj.element, filename: el};
+                        obj.element = id;
+                    }
+                    if (isFolder(obj.element)) { fixRoot(obj.element); }
                 };
                 for (var el in tr) {
                     if (!$.isArray(tr[el])) {
@@ -854,35 +886,13 @@ define([
                         delete tr[el];
                     } else {
                         toClean = [];
-                        tr[el].forEach(addToClean);
+                        tr[el].forEach(function (obj, idx) { addToClean(obj, idx, el); });
                         for (var i = toClean.length-1; i>=0; i--) {
                             tr[el].splice(toClean[i], 1);
                         }
                     }
                 }
             };
-            // Make sure unsorted doesn't exist anymore
-            var fixUnsorted = function () {
-                if (!files[UNSORTED]) { return; }
-                debug("UNSORTED still exists in the object, removing it...");
-                var us = files[UNSORTED];
-                if (us.length === 0) {
-                    delete files[UNSORTED];
-                    return;
-                }
-                var rootFiles = getFiles([ROOT, TEMPLATE]).slice();
-                var root = find([ROOT]);
-                us.forEach(function (el) {
-                    if (!isFile(el) || rootFiles.indexOf(el) !== -1) {
-                        return;
-                    }
-                    var data = getFileData(el);
-                    var name = data ? data.title : NEW_FILE_NAME;
-                    var newName = getAvailableName(root, name);
-                    root[newName] = el;
-                });
-                delete files[UNSORTED];
-            };
             var fixTemplate = function () {
                 if (!Array.isArray(files[TEMPLATE])) { debug("TEMPLATE was not an array"); files[TEMPLATE] = []; }
                 files[TEMPLATE] = Cryptpad.deduplicateString(files[TEMPLATE].slice());
@@ -893,36 +903,43 @@ define([
                     if (!isFile(el) || rootFiles.indexOf(el) !== -1) {
                         toClean.push(idx);
                     }
+                    if (typeof el === "string") {
+                        // We have an old file (href) which is not in filesData: add it
+                        var id = Cryptpad.createRandomInteger();
+                        files[NEW_FILES_DATA][id] = {href: el};
+                        us[idx] = id;
+                    }
                 });
                 toClean.forEach(function (idx) {
                     us.splice(idx, 1);
                 });
             };
             var fixFilesData = function () {
-                if (!$.isArray(files[FILES_DATA])) { debug("FILES_DATA was not an array"); files[FILES_DATA] = []; }
+                if (typeof files[NEW_FILES_DATA] !== "object") { debug("FILES_DATA was not an object"); files[NEW_FILES_DATA] = {}; }
-                var fd = files[FILES_DATA];
+                var fd = files[NEW_FILES_DATA];
                 var rootFiles = getFiles([ROOT, TRASH, 'hrefArray']);
                 var root = find([ROOT]);
                 var toClean = [];
-                fd.forEach(function (el) {
+                for (var id in fd) {
+                    id = Number(id);
+                    var el = fd[id];
                     if (!el || typeof(el) !== "object") {
                         debug("An element in filesData was not an object.", el);
                         toClean.push(el);
-                        return;
+                        continue;
                     }
                     if (!el.href) {
-                        debug("Rmoving an element in filesData with a missing href.", el);
+                        debug("Removing an element in filesData with a missing href.", el);
                         toClean.push(el);
-                        return;
+                        continue;
                     }
-                    if (Cryptpad.isLoggedIn() && rootFiles.indexOf(el.href) === -1) {
+                    if (Cryptpad.isLoggedIn() && rootFiles.indexOf(id) === -1) {
-                        debug("An element in filesData was not in ROOT, TEMPLATE or TRASH.", el);
+                        debug("An element in filesData was not in ROOT, TEMPLATE or TRASH.", id, el);
-                        var name = el.title || NEW_FILE_NAME;
+                        var newName = Cryptpad.createChannelId();
-                        var newName = getAvailableName(root, name);
+                        root[newName] = id;
-                        root[newName] = el.href;
+                        continue;
-                        return;
                     }
-                });
+                };
                 toClean.forEach(function (el) {
                     var idx = fd.indexOf(el);
                     if (idx !== -1) {
@@ -931,10 +948,90 @@ define([
                 });
             };
 
+            // Make sure unsorted doesn't exist anymore
+            // Note: Unsorted only works with ld structure where pads are href
+            // It should be called before the migration code
+            var fixUnsorted = function () {
+                if (!files[UNSORTED] || !files[FILES_DATA]) { return; }
+                debug("UNSORTED still exists in the object, removing it...");
+                var us = files[UNSORTED];
+                if (us.length === 0) {
+                    delete files[UNSORTED];
+                    return;
+                }
+                var root = find([ROOT]);
+                us.forEach(function (el) {
+                    if (typeof el !== "string") {
+                        return;
+                    }
+                    var data = files[FILES_DATA].filter(function (x) {
+                        return x.href === el;
+                    });
+                    if (data.length === 0) {
+                        files[FILES_DATA].push({
+                            href: el
+                        });
+                    }
+                    return;
+                    /*
+                    TODO remove
+                    var name = data.length !== 0 ? data[0].title : NEW_FILE_NAME;
+                    var newName = getAvailableName(root, name);
+                    root[newName] = el;
+                    */
+                });
+                delete files[UNSORTED];
+            };
+            // TODO: getRecentPads in cryptpad-common
+            // TODO: in fixRoot, trash and template, make sure we have a filedata associated to the id
+            var migrateToNewFormat = function () {
+                if (!files[FILES_DATA]) { return; }
+                try {
+                    var oldData = files[FILES_DATA].slice();
+                    var newData = files[NEW_FILES_DATA] = {};
+                    //var oldFiles = oldData.map(function (o) { return o.href; });
+                    oldData.forEach(function (obj) {
+                        if (!obj || !obj.href) { return; }
+                        var href = obj.href;
+                        var id = Cryptpad.createRandomInteger();
+                        var paths = findFile(href);
+                        var data = getFileData(href);
+                        var key = Cryptpad.createChannelId();
+                        if (data) {
+                            newData[id] = data;
+                        } else {
+                            newData[id] = {href: href};
+                        }
+                        paths.forEach(function (p) {
+                            var parentPath = p.slice();
+                            var okey = parentPath.pop(); // get the parent
+                            var parent = find(parentPath);
+                            if (isInTrashRoot(p)) {
+                                parent.element = id;
+                                newData[id].filename = p[1];
+                                return;
+                            }
+                            if (isPathIn(p, ['hrefArray'])) {
+                                parent[okey] = id;
+                                return;
+                            }
+                            // else root or trash (not trashroot)
+                            parent[key] = id;
+                            newData[id].filename = okey;
+                            delete parent[okey];
+                        });
+                    });
+                    delete files[FILES_DATA];
+                } catch(e) {
+                    console.error(e);
+                }
+            };
+
+            fixUnsorted();
+            migrateToNewFormat();
             fixRoot();
             fixTrashRoot();
             if (!workgroup) {
-                fixUnsorted();
                 fixTemplate();
                 fixFilesData();
             }
@@ -948,6 +1045,5 @@ define([
 
         return exp;
     };
-
     return module;
 });

www/drive/tests.js

@@ -0,0 +1,178 @@
+define([
+    '/common/cryptpad-common.js',
+    '/common/userObject.js',
+    'json.sortify',
+],function (Cryptpad, FO, sortify) {
+    var module = {};
+
+    var href1 = "/pad/#/1/edit/a798u+miu2tg5b-QaP9SvA/UIPoGUPewZscBUFhNIi+eBBM/";
+    var href2 = "/poll/#/1/edit/uFJTXjQUEwV2bl-y3cKVpP/LJ-4qPnpR5iY0HVdwLcnjLsx/";
+    var href3 = "/code/#/1/edit/R1kZC1mY9khSsrLCyJT+CA/jtQrCxbTiqQJ4HyUxbFBnmG8/";
+    var href4 = "/slide/#/1/edit/R2bZC1mY9khSsrLCyJT+CA/mlQrCxbTiqQJ4HyUxbFBnmG8/";
+
+    module.test = function (assert) {
+        var config = {Cryptpad: Cryptpad, workgroup: false};
+        assert(function (cb) {
+            var files = {
+              "root": {
+                "Folder": {},
+                "Folder2": {
+                  "FileName": href1
+                }
+              },
+              "template": [href3],
+              "trash": {
+                "DeletedF": [{
+                  "path": ["root"],
+                  "element": {}
+                }, {
+                  "path": ["root", "Folder"],
+                  "element": href2
+                }]
+              },
+              "CryptPad_RECENTPADS": [{
+                "atime": 23456783456,
+                "ctime": 12345678901,
+                "href": href3,
+                "title": "pewcode"
+              }, {
+                "atime": 23456789012,
+                "ctime": 12345789235,
+                "href": href2,
+                "title": "pewpoll"
+              }, {
+                "atime": 23456789012,
+                "ctime": 12345789235,
+                "href": href1,
+                "title": "pewpad"
+              }]
+            };
+            var fo = FO.init(files, config);
+            fo.fixFiles();
+            if (files['CryptPad_RECENTPADS'] || !files.filesData) {
+                console.log("DRIVE1: migration from RECENTPADS to filesData failed");
+                return cb();
+            }
+            var fileKey = Object.keys(files.root.Folder2)[0];
+            if (!fileKey) { return cb(); }
+            var fileId = files.root.Folder2[fileKey];
+            var res = typeof fileId === "number"
+                    && typeof files.filesData[fileId] === "object"
+                    && files.filesData[fileId].filename === "FileName"
+                    && typeof files.trash.DeletedF[1].element === "number"
+                    && typeof files.filesData[files.trash.DeletedF[1].element] === "object"
+                    && files.filesData[files.trash.DeletedF[1].element].filename === "DeletedF"
+                    && typeof files.template[0] === "number"
+                    && typeof files.filesData[files.template[0]] === "object"
+                    && !files.filesData[files.template[0]].filename
+            return cb(res);
+        }, "DRIVE1: migration and fixFiles without unsorted");
+
+        assert(function (cb) {
+            var files = {
+              "root": {
+                "Folder": {},
+                "Folder2": {
+                  "FileName": "/pad/#/1/edit/a798u+miu2tg5b-QaP9SvA/UIPoGUPewZscBUFhNIi+eBBM/"
+                }
+              },
+              "unsorted": ["/code/#/1/edit/R1kZC1mY9khSsrLCyJT+CA/jtQrCxbTiqQJ4HyUxbFBnmG8/"],
+              "trash": {},
+              "CryptPad_RECENTPADS": [{
+                "atime": 23456783456,
+                "ctime": 12345678901,
+                "href": "/code/#/1/edit/R1kZC1mY9khSsrLCyJT+CA/jtQrCxbTiqQJ4HyUxbFBnmG8/",
+                "title": "pewcode"
+              }, {
+                "atime": 23456789012,
+                "ctime": 12345789235,
+                "href": "/pad/#/1/edit/a798u+miu2tg5b-QaP9SvA/UIPoGUPewZscBUFhNIi+eBBM/",
+                "title": "pewpad"
+              }]
+            };
+            var fo = FO.init(files, config);
+            fo.fixFiles();
+            if (files['CryptPad_RECENTPADS'] || !files.filesData) {
+                console.log("DRIVE2: migration from RECENTPADS to filesData failed");
+                return cb();
+            }
+            if (!files.template) {
+                console.log("DRIVE2: template is missing");
+                return cb();
+            }
+            if (files.unsorted) {
+                console.log("DRIVE2: unsorted not removed");
+                return cb();
+            }
+            var fileKey = Object.keys(files.root.Folder2)[0];
+            var fileKey2 = Object.keys(files.root).filter(function (x) {
+                return typeof files.root[x] === "number"
+            })[0];
+            if (!fileKey || !fileKey2) { return cb(); }
+            var fileId = files.root.Folder2[fileKey];
+            var fileId2 = files.root[fileKey2];
+            var res = typeof fileId === "number"
+                    && typeof files.filesData[fileId] === "object"
+                    && files.filesData[fileId].filename === "FileName"
+                    && typeof fileId2 === "number"
+                    && typeof files.filesData[fileId2] === "object"
+                    && !files.filesData[fileId2].filename
+            return cb(res);
+        }, "DRIVE2: migration and fixFiles with unsorted");
+
+        assert(function (cb) {
+            var files = {
+              "root": {
+                "Folder": {},
+                "Folder2": {
+                  "FileName": href1
+                }
+              },
+              "template": [href3],
+              "trash": {
+                "DeletedF": [{
+                  "path": ["root"],
+                  "element": { "Trash": href4 }
+                }, {
+                  "path": ["root", "Folder"],
+                  "element": href2
+                }]
+              },
+              "CryptPad_RECENTPADS": []
+            };
+            var fo = FO.init(files, config);
+            fo.fixFiles();
+            if (files['CryptPad_RECENTPADS'] || !files.filesData) {
+                console.log("DRIVE2: migration from RECENTPADS to filesData failed");
+                return cb();
+            }
+            var fileKey = Object.keys(files.root.Folder2)[0];
+            var fileKey2 = Object.keys(files.trash.DeletedF[0].element)[0];
+            if (!fileKey || !fileKey2) { return cb(); }
+            var fileId = files.root.Folder2[fileKey];
+            var fileId2 = files.trash.DeletedF[0].element[fileKey2];
+            var res = typeof fileId === "number"
+                    && typeof files.filesData[fileId] === "object"
+                    && files.filesData[fileId].filename === "FileName"
+                    && files.filesData[fileId].href === href1
+                    && typeof files.trash.DeletedF[1].element === "number"
+                    && typeof files.filesData[files.trash.DeletedF[1].element] === "object"
+                    && files.filesData[files.trash.DeletedF[1].element].filename === "DeletedF"
+                    && files.filesData[files.trash.DeletedF[1].element].href === href2
+                    && typeof files.template[0] === "number"
+                    && typeof files.filesData[files.template[0]] === "object"
+                    && !files.filesData[files.template[0]].filename
+                    && files.filesData[files.template[0]].href === href3
+                    && typeof fileId2 === "number"
+                    && typeof files.filesData[fileId2] === "object"
+                    && files.filesData[fileId2].filename === "Trash"
+                    && files.filesData[fileId2].href === href4;
+            return cb(res);
+        }, "DRIVE4: migration and fixFiles with a pad in trash not root");
+
+    };
+
+    //TODO test with a file not in RECENTPADS 
+
+    return module;
+});