move more database reads into the database worker

lib/workers/compute-index.js

@@ -3,16 +3,20 @@
 
 const HK = require("../hk-util");
 const Store = require("../storage/file");
+const BlobStore = require("../storage/blob");
 const Util = require("../common-util");
 const nThen = require("nthen");
 const Meta = require("../metadata");
 const Pins = require("../pins");
+const Core = require("../commands/core");
+const Saferphore = require("saferphore");
 
 const Env = {};
 
 var ready = false;
 var store;
 var pinStore;
+var blobStore;
 const init = function (config, _cb) {
     const cb = Util.once(Util.mkAsync(_cb));
     if (!config) {
@@ -36,6 +40,18 @@ const init = function (config, _cb) {
             }
             pinStore = _pinStore;
         }));
+        BlobStore.create({
+            blobPath: config.blobPath,
+            blobStagingPath: config.blobStagingPath,
+            archivePath: config.archivePath,
+            getSession: function () {},
+        }, w(function (err, blob) {
+            if (err) {
+                w.abort();
+                return void cb(err);
+            }
+            blobStore = blob;
+        }));
     }).nThen(function () {
         cb();
     });
@@ -218,11 +234,124 @@ const getPinState = function (data, cb, errorHandler) {
     });
 };
 
+const _getFileSize = function (channel, _cb) {
+    var cb = Util.once(Util.mkAsync(_cb));
+    if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); }
+    if (channel.length === 32) {
+        return void store.getChannelSize(channel, function (e, size) {
+            if (e) {
+                if (e.code === 'ENOENT') { return void cb(void 0, 0); }
+                return void cb(e.code);
+            }
+            cb(void 0, size);
+        });
+    }
+
+    // 'channel' refers to a file, so you need another API
+    blobStore.size(channel, function (e, size) {
+        if (typeof(size) === 'undefined') { return void cb(e); }
+        cb(void 0, size);
+    });
+};
+
+const getFileSize = function (data, cb) {
+    _getFileSize(data.channel, cb);
+};
+
+const _iterateFiles = function (channels, handler, cb) {
+    if (!Array.isArray(channels)) { return cb('INVALID_LIST'); }
+    var L = channels.length;
+    var sem = Saferphore.create(10);
+
+    // (channel, next) => { ??? }
+    var job = function (channel, wait) {
+        return function (give) {
+            handler(channel, wait(give()));
+        };
+    };
+
+    nThen(function (w) {
+        for (var i = 0; i < L; i++) {
+            sem.take(job(channels[i], w));
+        }
+    }).nThen(function () {
+        cb();
+    });
+};
+
+const getTotalSize = function (data, cb) {
+    var bytes = 0;
+    _iterateFiles(data.channels, function (channel, next) {
+        _getFileSize(channel, function (err, size) {
+            if (!err) { bytes += size; }
+            next();
+        });
+    }, function (err) {
+        if (err) { return cb(err); }
+        cb(void 0, bytes);
+    });
+};
+
+const getDeletedPads = function (data, cb) {
+    var absentees = [];
+    _iterateFiles(data.channels, function (channel, next) {
+        _getFileSize(channel, function (err, size) {
+            if (err) { return next(); }
+            if (size === 0) { absentees.push(channel); }
+            next();
+        });
+    }, function (err) {
+        if (err) { return void cb(err); }
+        cb(void 0, absentees);
+    });
+};
+
+const getMultipleFileSize = function (data, cb) {
+    const counts = {};
+    _iterateFiles(data.channels, function (channel, next) {
+        _getFileSize(channel, function (err, size) {
+            counts[channel] = err? 0: size;
+            next();
+        });
+    }, function (err) {
+        if (err) {
+            return void cb(err);
+        }
+        cb(void 0, counts);
+    });
+};
+
+const getHashOffset = function (data, cb) {
+    const channelName = data.channel;
+    const lastKnownHash = data.lastKnownHash;
+
+    var offset = -1;
+    store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
+        // tryParse return a parsed message or undefined
+        const msg = tryParse(Env, msgObj.buff.toString('utf8'));
+        // if it was undefined then go onto the next message
+        if (typeof msg === "undefined") { return readMore(); }
+        if (typeof(msg[4]) !== 'string' || lastKnownHash !== HK.getHash(msg[4])) {
+            return void readMore();
+        }
+        offset = msgObj.offset;
+        abort();
+    }, function (err) {
+        if (err) { return void cb(err); }
+        cb(void 0, offset);
+    });
+};
+
 const COMMANDS = {
     COMPUTE_INDEX: computeIndex,
     COMPUTE_METADATA: computeMetadata,
     GET_OLDER_HISTORY: getOlderHistory,
     GET_PIN_STATE: getPinState,
+    GET_FILE_SIZE: getFileSize,
+    GET_TOTAL_SIZE: getTotalSize,
+    GET_DELETED_PADS: getDeletedPads,
+    GET_MULTIPLE_FILE_SIZE: getMultipleFileSize,
+    GET_HASH_OFFSET: getHashOffset,
 };
 
 process.on('message', function (data) {