web/cryptpad
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

compute metadata in the same child process that builds indexes

Browse Source
This commit is contained in:
ansuz
2020-03-24 17:43:15 -04:00
parent 479b76f848
commit 471e374533
4 changed files with 95 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
lib/hk-util.js
View File

@@ -337,7 +337,7 @@ const storeMessage = function (Env, channel, msg, isCp, optionalMessageHash) {
* -1 if you didn't find it
*/
const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => {
const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => { // XXX child process
const cb = Util.once(Util.mkAsync(_cb));
const store = Env.store;
const Log = Env.Log;
@@ -454,7 +454,7 @@ const getHistoryAsync = (Env, channelName, lastKnownHash, beforeHash, handler, c
Used by:
* GET_HISTORY_RANGE
*/
const getOlderHistory = function (Env, channelName, oldestKnownHash, cb) {
const getOlderHistory = function (Env, channelName, oldestKnownHash, cb) { // XXX child process
const store = Env.store;
const Log = Env.Log;
var messageBuffer = [];
@@ -833,7 +833,14 @@ HK.initializeIndexWorkers = function (Env, config, _cb) {
});
worker.on('message', function (res) {
if (!res || !res.txid) { return; }
if (!res) { return; }
if (!res.txid) {
// !report errors...
if (res.error) {
Env.Log.error(res.error, res.value);
}
return;
}
//console.log(res);
try {
response.handle(res.txid, [res.error, res.value]);
@@ -860,20 +867,18 @@ HK.initializeIndexWorkers = function (Env, config, _cb) {
};
var workerIndex = 0;
var sendCommand = function (Env, channel, cb) {
var sendCommand = function (msg, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
workerIndex = (workerIndex + 1) % workers.length;
if (workers.length === 0 ||
typeof(workers[workerIndex].send) !== 'function') {
return void cb("NO_WORKERS");
}
Env.store.getWeakLock(channel, function (next) {
const txid = Util.uid();
response.expect(txid, Util.both(next, cb), 45000);
workers[workerIndex].send({
txid: txid,
args: channel,
});
});
const txid = Util.uid();
msg.txid = txid;
response.expect(txid, cb, 45000);
workers[workerIndex].send(msg);
};
nThen(function (w) {
@@ -885,6 +890,30 @@ HK.initializeIndexWorkers = function (Env, config, _cb) {
}));
});
}).nThen(function () {
Env.computeIndex = function (Env, channel, cb) {
Env.store.getWeakLock(channel, function (next) {
sendCommand({
channel: channel,
command: 'COMPUTE_INDEX',
}, function (err, index) {
next();
cb(err, index);
});
});
};
Env.computeMetadata = function (channel, cb) {
Env.store.getWeakLock(channel, function (next) {
sendCommand({
channel: channel,
command: 'COMPUTE_METADATA',
}, function (err, metadata) {
next();
cb(err, metadata);
});
});
};
//console.log("index workers ready");
cb(void 0, sendCommand);
});
@@ -925,14 +954,13 @@ HK.initializeValidationWorkers = function (Env) {
var nextWorker = 0;
const send = function (msg, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
// let's be paranoid about asynchrony and only calling back once..
nextWorker = (nextWorker + 1) % workers.length;
if (workers.length === 0 || typeof(workers[nextWorker].send) !== 'function') {
console.error(workers);
throw new Error("INVALID_WORKERS");
return void cb("INVALID_WORKERS");
}
var cb = Util.once(Util.mkAsync(_cb));
var txid = msg.txid = Util.uid();
// expect a response within 15s