Merge branch 'staging' of github.com:xwiki-labs/cryptpad into staging
This commit is contained in:
yflory
commit
eaa389fcb8
@ -219,6 +219,29 @@ module.exports = {
|
||||
*/
|
||||
inactiveTime: 90, // days
|
||||
|
||||
/* CryptPad can be configured to remove inactive data which has not been pinned.
|
||||
* Deletion of data is always risky and as an operator you have the choice to
|
||||
* archive data instead of deleting it outright. Set this value to true if
|
||||
* you want your server to archive files and false if you want to keep using
|
||||
* the old behaviour of simply removing files.
|
||||
*
|
||||
* WARNING: this is not implemented universally, so at the moment this will
|
||||
* only apply to the removal of 'channels' due to inactivity.
|
||||
*/
|
||||
retainData: true,
|
||||
|
||||
/* As described above, CryptPad offers the ability to archive some data
|
||||
* instead of deleting it outright. This archived data still takes up space
|
||||
* and so you'll probably still want to remove these files after a brief period.
|
||||
* The intent with this feature is to provide a safety net in case of accidental
|
||||
* deletion. Set this value to the number of days you'd like to retain
|
||||
* archived data before it's removed permanently.
|
||||
*
|
||||
* If 'retainData' is set to false, there will never be any archived data
|
||||
* to remove.
|
||||
*/
|
||||
archiveRetentionTime: 15,
|
||||
|
||||
/* Max Upload Size (bytes)
|
||||
* this sets the maximum size of any one file uploaded to the server.
|
||||
* anything larger than this size will be rejected
|
||||
@ -245,12 +268,21 @@ module.exports = {
|
||||
* ===================== */
|
||||
|
||||
/*
|
||||
CryptPad stores each document in an individual file on your hard drive.
|
||||
Specify a directory where files should be stored.
|
||||
It will be created automatically if it does not already exist.
|
||||
*/
|
||||
* CryptPad stores each document in an individual file on your hard drive.
|
||||
* Specify a directory where files should be stored.
|
||||
* It will be created automatically if it does not already exist.
|
||||
*/
|
||||
filePath: './datastore/',
|
||||
|
||||
/* CryptPad offers the ability to archive data for a configurable period
|
||||
* before deleting it, allowing a means of recovering data in the event
|
||||
* that it was deleted accidentally.
|
||||
*
|
||||
* To set the location of this archive directory to a custom value, change
|
||||
* the path below:
|
||||
*/
|
||||
archivePath: './data/archive',
|
||||
|
||||
/* CryptPad allows logged in users to request that particular documents be
|
||||
* stored by the server indefinitely. This is called 'pinning'.
|
||||
* Pin requests are stored in a pin-store. The location of this store is
|
||||
|
||||
37
package-lock.json
generated
37
package-lock.json
generated
@ -99,26 +99,15 @@
|
||||
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
|
||||
},
|
||||
"chainpad-server": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-3.0.1.tgz",
|
||||
"integrity": "sha512-1r53gYvPlrnZg0vf91gP3pqHILfi67oSo3cnj7kcvC4Y/n4t6wS3QCCjXeNArOuZv/sIByuKkeo1929osr1/KA==",
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-3.0.2.tgz",
|
||||
"integrity": "sha512-c5aEljVAapDKKs0+Rt2jymKAszm8X4ZeLFNJj1yxflwBqoh0jr8OANYvbfjtNaYFe2Wdflp/1i4gibYX4IMc+g==",
|
||||
"requires": {
|
||||
"nthen": "^0.1.8",
|
||||
"pull-stream": "^3.6.9",
|
||||
"stream-to-pull-stream": "^1.7.3",
|
||||
"tweetnacl": "~0.12.2",
|
||||
"ws": "^1.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"ws": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-1.1.5.tgz",
|
||||
"integrity": "sha512-o3KqipXNUdS7wpQzBHSe180lBGO60SoK0yVo3CYJgb2MkobuWuBX6dhkYP5ORCLd55y+SaflMOV5fqAB53ux4w==",
|
||||
"requires": {
|
||||
"options": ">=0.0.5",
|
||||
"ultron": "1.0.x"
|
||||
}
|
||||
}
|
||||
"ws": "^3.3.1"
|
||||
}
|
||||
},
|
||||
"chalk": {
|
||||
@ -827,11 +816,6 @@
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"options": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz",
|
||||
"integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8="
|
||||
},
|
||||
"os-tmpdir": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
|
||||
@ -1248,9 +1232,9 @@
|
||||
}
|
||||
},
|
||||
"ultron": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.0.2.tgz",
|
||||
"integrity": "sha1-rOEWq1V80Zc4ak6I9GhTeMiy5Po="
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
|
||||
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
|
||||
},
|
||||
"uniq": {
|
||||
"version": "1.0.1",
|
||||
@ -1298,13 +1282,6 @@
|
||||
"async-limiter": "~1.0.0",
|
||||
"safe-buffer": "~5.1.0",
|
||||
"ultron": "~1.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ultron": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
|
||||
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"xml2js": {
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
"url": "git://github.com/xwiki-labs/cryptpad.git"
|
||||
},
|
||||
"dependencies": {
|
||||
"chainpad-server": "~3.0.0",
|
||||
"chainpad-server": "~3.0.2",
|
||||
"express": "~4.16.0",
|
||||
"fs-extra": "^7.0.0",
|
||||
"nthen": "~0.1.0",
|
||||
|
||||
17
rpc.js
17
rpc.js
@ -814,6 +814,7 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) {
|
||||
return void cb('INSUFFICIENT_PERMISSIONS');
|
||||
}
|
||||
|
||||
// FIXME COLDSTORAGE
|
||||
return void Env.msgStore.clearChannel(channelId, function (e) {
|
||||
cb(e);
|
||||
});
|
||||
@ -900,6 +901,20 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) {
|
||||
if (metadata.owners.indexOf(unsafeKey) === -1) {
|
||||
return void cb('INSUFFICIENT_PERMISSIONS');
|
||||
}
|
||||
|
||||
// if the admin has configured data retention...
|
||||
// temporarily archive the file instead of removing it
|
||||
if (Env.retainData) {
|
||||
return void Env.msgStore.archiveChannel(channelId, function (e) {
|
||||
Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', {
|
||||
unsafeKey: unsafeKey,
|
||||
channelId: channelId,
|
||||
status: e? String(e): 'SUCCESS',
|
||||
});
|
||||
cb(e);
|
||||
});
|
||||
}
|
||||
|
||||
return void Env.msgStore.removeChannel(channelId, function (e) {
|
||||
Log.info('DELETION_CHANNEL_BY_OWNER_RPC', {
|
||||
unsafeKey: unsafeKey,
|
||||
@ -1430,6 +1445,7 @@ var removeLoginBlock = function (Env, msg, cb) {
|
||||
return void cb('E_INVALID_BLOCK_PATH');
|
||||
}
|
||||
|
||||
// FIXME COLDSTORAGE
|
||||
Fs.unlink(path, function (err) {
|
||||
Log.info('DELETION_BLOCK_BY_OWNER_RPC', {
|
||||
publicKey: publicKey,
|
||||
@ -1645,6 +1661,7 @@ RPC.create = function (
|
||||
};
|
||||
|
||||
var Env = {
|
||||
retainData: config.retainData || false,
|
||||
defaultStorageLimit: config.defaultStorageLimit,
|
||||
maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024),
|
||||
Sessions: {},
|
||||
|
||||
@ -7,6 +7,14 @@ const config = require("../lib/load-config");
|
||||
|
||||
if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") { return; }
|
||||
|
||||
/* Instead of this script you should probably use
|
||||
evict-inactive.js which moves things to an archive directory
|
||||
in case the data that would have been deleted turns out to be important.
|
||||
it also handles removing that archived data after a set period of time
|
||||
|
||||
it only works for channels at the moment, though, and nothing else.
|
||||
*/
|
||||
|
||||
let inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
|
||||
let inactiveConfig = {
|
||||
unpinned: true,
|
||||
|
||||
63
scripts/diagnose-archive-conflicts.js
Normal file
63
scripts/diagnose-archive-conflicts.js
Normal file
@ -0,0 +1,63 @@
|
||||
var nThen = require("nthen");
|
||||
|
||||
var Store = require("../storage/file");
|
||||
var config = require("../lib/load-config");
|
||||
|
||||
var store;
|
||||
var Log;
|
||||
nThen(function (w) {
|
||||
// load the store which will be used for iterating over channels
|
||||
// and performing operations like archival and deletion
|
||||
Store.create(config, w(function (_) {
|
||||
store = _;
|
||||
}));
|
||||
|
||||
// load the logging module so that you have a record of which
|
||||
// files were archived or deleted at what time
|
||||
var Logger = require("../lib/log");
|
||||
Logger.create(config, w(function (_) {
|
||||
Log = _;
|
||||
}));
|
||||
}).nThen(function (w) {
|
||||
// count the number of files which have been restored in this run
|
||||
var conflicts = 0;
|
||||
|
||||
var handler = function (err, item, cb) {
|
||||
if (err) {
|
||||
Log.error('DIAGNOSE_ARCHIVE_CONFLICTS_ITERATION', err);
|
||||
return void cb();
|
||||
}
|
||||
|
||||
// check if such a file exists on the server
|
||||
store.isChannelAvailable(item.channel, function (err, available) {
|
||||
// weird edge case?
|
||||
if (err) { return void cb(); }
|
||||
|
||||
// the channel doesn't exist in the database
|
||||
if (!available) { return void cb(); }
|
||||
|
||||
// the channel is available
|
||||
// that means it's a duplicate of something in the archive
|
||||
conflicts++;
|
||||
Log.info('DIAGNOSE_ARCHIVE_CONFLICT_DETECTED', item.channel);
|
||||
cb();
|
||||
});
|
||||
};
|
||||
|
||||
// if you hit an error, log it
|
||||
// otherwise, when there are no more channels to process
|
||||
// log some stats about how many were removed
|
||||
var done = function (err) {
|
||||
if (err) {
|
||||
return Log.error('DIAGNOSE_ARCHIVE_CONFLICTS_FINAL_ERROR', err);
|
||||
}
|
||||
Log.info('DIAGNOSE_ARCHIVE_CONFLICTS_COUNT', conflicts);
|
||||
};
|
||||
|
||||
store.listArchivedChannels(handler, w(done));
|
||||
}).nThen(function () {
|
||||
// the store will keep this script running if you don't shut it down
|
||||
store.shutdown();
|
||||
Log.shutdown();
|
||||
});
|
||||
|
||||
169
scripts/evict-inactive.js
Normal file
169
scripts/evict-inactive.js
Normal file
@ -0,0 +1,169 @@
|
||||
var nThen = require("nthen");
|
||||
|
||||
var Store = require("../storage/file");
|
||||
var Pinned = require("./pinned");
|
||||
var config = require("../lib/load-config");
|
||||
|
||||
// the administrator should have set an 'inactiveTime' in their config
|
||||
// if they didn't, just exit.
|
||||
if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") { return; }
|
||||
|
||||
// files which have not been changed since before this date can be considered inactive
|
||||
var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
|
||||
|
||||
// files which were archived before this date can be considered safe to remove
|
||||
var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
|
||||
|
||||
var store;
|
||||
var pins;
|
||||
var Log;
|
||||
nThen(function (w) {
|
||||
// load the store which will be used for iterating over channels
|
||||
// and performing operations like archival and deletion
|
||||
Store.create(config, w(function (_) {
|
||||
store = _;
|
||||
})); // load the list of pinned files so you know which files
|
||||
// should not be archived or deleted
|
||||
Pinned.load(w(function (err, _) {
|
||||
if (err) {
|
||||
w.abort();
|
||||
return void console.error(err);
|
||||
}
|
||||
pins = _;
|
||||
}), {
|
||||
pinPath: config.pinPath,
|
||||
});
|
||||
|
||||
// load the logging module so that you have a record of which
|
||||
// files were archived or deleted at what time
|
||||
var Logger = require("../lib/log");
|
||||
Logger.create(config, w(function (_) {
|
||||
Log = _;
|
||||
}));
|
||||
}).nThen(function (w) {
|
||||
// this block will iterate over archived channels and remove them
|
||||
// if they've been in cold storage for longer than your configured archive time
|
||||
|
||||
// if the admin has not set an 'archiveRetentionTime', this block makes no sense
|
||||
// so just skip it
|
||||
if (typeof(config.archiveRetentionTime) !== "number") { return; }
|
||||
|
||||
// count the number of files which have been removed in this run
|
||||
var removed = 0;
|
||||
|
||||
var handler = function (err, item, cb) {
|
||||
if (err) {
|
||||
Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err);
|
||||
return void cb();
|
||||
}
|
||||
// don't mess with files that are freshly stored in cold storage
|
||||
// based on ctime because that's changed when the file is moved...
|
||||
if (+new Date(item.ctime) > retentionTime) {
|
||||
return void cb();
|
||||
}
|
||||
|
||||
// but if it's been stored for the configured time...
|
||||
// expire it
|
||||
store.removeArchivedChannel(item.channel, w(function (err) {
|
||||
if (err) {
|
||||
Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', {
|
||||
error: err,
|
||||
channel: item.channel,
|
||||
});
|
||||
return void cb();
|
||||
}
|
||||
Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel);
|
||||
removed++;
|
||||
cb();
|
||||
}));
|
||||
};
|
||||
|
||||
// if you hit an error, log it
|
||||
// otherwise, when there are no more channels to process
|
||||
// log some stats about how many were removed
|
||||
var done = function (err) {
|
||||
if (err) {
|
||||
return Log.error('EVICT_ARCHIVED_FINAL_ERROR', err);
|
||||
}
|
||||
Log.info('EVICT_ARCHIVED_CHANNELS_REMOVED', removed);
|
||||
};
|
||||
|
||||
store.listArchivedChannels(handler, w(done));
|
||||
}).nThen(function (w) {
|
||||
var removed = 0;
|
||||
var channels = 0;
|
||||
var archived = 0;
|
||||
|
||||
var handler = function (err, item, cb) {
|
||||
channels++;
|
||||
if (err) {
|
||||
Log.error('EVICT_CHANNEL_ITERATION', err);
|
||||
return void cb();
|
||||
}
|
||||
// check if the database has any ephemeral channels
|
||||
// if it does it's because of a bug, and they should be removed
|
||||
if (item.channel.length === 34) {
|
||||
return void store.removeChannel(item.channel, w(function (err) {
|
||||
if (err) {
|
||||
Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', {
|
||||
error: err,
|
||||
channel: item.channel,
|
||||
});
|
||||
return void cb();
|
||||
}
|
||||
Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel);
|
||||
cb();
|
||||
}));
|
||||
}
|
||||
|
||||
// bail out if the channel was modified recently
|
||||
if (+new Date(item.mtime) > inactiveTime) { return void cb(); }
|
||||
|
||||
// ignore the channel if it's pinned
|
||||
if (pins[item.channel]) { return void cb(); }
|
||||
|
||||
// if the server is configured to retain data, archive the channel
|
||||
if (config.retainData) {
|
||||
return void store.archiveChannel(item.channel, w(function (err) {
|
||||
if (err) {
|
||||
Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
|
||||
error: err,
|
||||
channel: item.channel,
|
||||
});
|
||||
return void cb();
|
||||
}
|
||||
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel);
|
||||
archived++;
|
||||
cb();
|
||||
}));
|
||||
}
|
||||
|
||||
// otherwise remove it
|
||||
store.removeChannel(item.channel, w(function (err) {
|
||||
if (err) {
|
||||
Log.error('EVICT_CHANNEL_REMOVAL_ERROR', {
|
||||
error: err,
|
||||
channel: item.channel,
|
||||
});
|
||||
return void cb();
|
||||
}
|
||||
Log.info('EVICT_CHANNEL_REMOVAL', item.channel);
|
||||
removed++;
|
||||
cb();
|
||||
}));
|
||||
};
|
||||
|
||||
var done = function () {
|
||||
if (config.retainData) {
|
||||
return void Log.info('EVICT_CHANNELS_ARCHIVED', archived);
|
||||
}
|
||||
return void Log.info('EVICT_CHANNELS_REMOVED', removed);
|
||||
};
|
||||
|
||||
store.listChannels(handler, w(done));
|
||||
}).nThen(function () {
|
||||
// the store will keep this script running if you don't shut it down
|
||||
store.shutdown();
|
||||
Log.shutdown();
|
||||
});
|
||||
|
||||
61
scripts/restore-archived.js
Normal file
61
scripts/restore-archived.js
Normal file
@ -0,0 +1,61 @@
|
||||
var nThen = require("nthen");
|
||||
|
||||
var Store = require("../storage/file");
|
||||
var config = require("../lib/load-config");
|
||||
|
||||
var store;
|
||||
var Log;
|
||||
nThen(function (w) {
|
||||
// load the store which will be used for iterating over channels
|
||||
// and performing operations like archival and deletion
|
||||
Store.create(config, w(function (_) {
|
||||
store = _;
|
||||
}));
|
||||
|
||||
// load the logging module so that you have a record of which
|
||||
// files were archived or deleted at what time
|
||||
var Logger = require("../lib/log");
|
||||
Logger.create(config, w(function (_) {
|
||||
Log = _;
|
||||
}));
|
||||
}).nThen(function (w) {
|
||||
// count the number of files which have been restored in this run
|
||||
var restored = 0;
|
||||
|
||||
var handler = function (err, item, cb) {
|
||||
if (err) {
|
||||
Log.error('RESTORE_ARCHIVED_CHANNEL_ITERATION', err);
|
||||
return void cb();
|
||||
}
|
||||
|
||||
store.restoreArchivedChannel(item.channel, w(function (err) {
|
||||
if (err) {
|
||||
Log.error('RESTORE_ARCHIVED_CHANNEL_RESTORATION_ERROR', {
|
||||
error: err,
|
||||
channel: item.channel,
|
||||
});
|
||||
return void cb();
|
||||
}
|
||||
Log.info('RESTORE_ARCHIVED_CHANNEL_RESTORATION', item.channel);
|
||||
restored++;
|
||||
cb();
|
||||
}));
|
||||
};
|
||||
|
||||
// if you hit an error, log it
|
||||
// otherwise, when there are no more channels to process
|
||||
// log some stats about how many were removed
|
||||
var done = function (err) {
|
||||
if (err) {
|
||||
return Log.error('RESTORE_ARCHIVED_FINAL_ERROR', err);
|
||||
}
|
||||
Log.info('RESTORE_ARCHIVED_CHANNELS_RESTORED', restored);
|
||||
};
|
||||
|
||||
store.listArchivedChannels(handler, w(done));
|
||||
}).nThen(function () {
|
||||
// the store will keep this script running if you don't shut it down
|
||||
store.shutdown();
|
||||
Log.shutdown();
|
||||
});
|
||||
|
||||
186
storage/file.js
186
storage/file.js
@ -5,6 +5,7 @@ var Fs = require("fs");
|
||||
var Fse = require("fs-extra");
|
||||
var Path = require("path");
|
||||
var nThen = require("nthen");
|
||||
var Semaphore = require("saferphore");
|
||||
const ToPull = require('stream-to-pull-stream');
|
||||
const Pull = require('pull-stream');
|
||||
|
||||
@ -14,10 +15,18 @@ const isValidChannelId = function (id) {
|
||||
/^[a-zA-Z0-9=+-]*$/.test(id);
|
||||
};
|
||||
|
||||
// 511 -> octal 777
|
||||
// read, write, execute permissions flag
|
||||
const PERMISSIVE = 511;
|
||||
|
||||
var mkPath = function (env, channelId) {
|
||||
return Path.join(env.root, channelId.slice(0, 2), channelId) + '.ndjson';
|
||||
};
|
||||
|
||||
var mkArchivePath = function (env, channelId) {
|
||||
return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.ndjson';
|
||||
};
|
||||
|
||||
var getMetadataAtPath = function (Env, path, cb) {
|
||||
var remainder = '';
|
||||
var stream = Fs.createReadStream(path, { encoding: 'utf8' });
|
||||
@ -68,7 +77,7 @@ var closeChannel = function (env, channelName, cb) {
|
||||
}
|
||||
};
|
||||
|
||||
var clearChannel = function (env, channelId, cb) { // FIXME deletion
|
||||
var clearChannel = function (env, channelId, cb) {
|
||||
var path = mkPath(env, channelId);
|
||||
getMetadataAtPath(env, path, function (e, metadata) {
|
||||
if (e) { return cb(new Error(e)); }
|
||||
@ -189,8 +198,7 @@ var checkPath = function (path, callback) {
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
// 511 -> octal 777
|
||||
Fse.mkdirp(Path.dirname(path), 511, function (err) {
|
||||
Fse.mkdirp(Path.dirname(path), PERMISSIVE, function (err) {
|
||||
if (err && err.code !== 'EEXIST') {
|
||||
callback(err);
|
||||
return;
|
||||
@ -200,11 +208,128 @@ var checkPath = function (path, callback) {
|
||||
});
|
||||
};
|
||||
|
||||
var removeChannel = function (env, channelName, cb) { // FIXME deletion
|
||||
var removeChannel = function (env, channelName, cb) {
|
||||
var filename = mkPath(env, channelName);
|
||||
Fs.unlink(filename, cb);
|
||||
};
|
||||
|
||||
// pass in the path so we can reuse the same function for archived files
|
||||
var channelExists = function (filepath, channelName, cb) {
|
||||
Fs.stat(filepath, function (err, stat) {
|
||||
if (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
// no, the file doesn't exist
|
||||
return void cb(void 0, false);
|
||||
}
|
||||
return void cb(err);
|
||||
}
|
||||
if (!stat.isFile()) { return void cb("E_NOT_FILE"); }
|
||||
return void cb(void 0, true);
|
||||
});
|
||||
};
|
||||
|
||||
var removeArchivedChannel = function (env, channelName, cb) {
|
||||
var filename = mkArchivePath(env, channelName);
|
||||
Fs.unlink(filename, cb);
|
||||
};
|
||||
|
||||
var listChannels = function (root, handler, cb) {
|
||||
// do twenty things at a time
|
||||
var sema = Semaphore.create(20);
|
||||
|
||||
var dirList = [];
|
||||
|
||||
nThen(function (w) {
|
||||
// the root of your datastore contains nested directories...
|
||||
Fs.readdir(root, w(function (err, list) {
|
||||
if (err) {
|
||||
w.abort();
|
||||
// TODO check if we normally return strings or errors
|
||||
return void cb(err);
|
||||
}
|
||||
dirList = list;
|
||||
}));
|
||||
}).nThen(function (w) {
|
||||
// search inside the nested directories
|
||||
// stream it so you don't put unnecessary data in memory
|
||||
var wait = w();
|
||||
dirList.forEach(function (dir) {
|
||||
sema.take(function (give) {
|
||||
var nestedDirPath = Path.join(root, dir);
|
||||
Fs.readdir(nestedDirPath, w(give(function (err, list) {
|
||||
if (err) { return void handler(err); } // Is this correct?
|
||||
|
||||
list.forEach(function (item) {
|
||||
// ignore things that don't match the naming pattern
|
||||
if (/^\./.test(item) || !/[0-9a-fA-F]{32,}\.ndjson$/.test(item)) { return; }
|
||||
var filepath = Path.join(nestedDirPath, item);
|
||||
var channel = filepath.replace(/\.ndjson$/, '').replace(/.*\//, '');
|
||||
if ([32, 34].indexOf(channel.length) === -1) { return; }
|
||||
|
||||
// otherwise throw it on the pile
|
||||
sema.take(function (give) {
|
||||
var next = w(give());
|
||||
Fs.stat(filepath, w(function (err, stats) {
|
||||
if (err) {
|
||||
return void handler(err);
|
||||
}
|
||||
|
||||
handler(void 0, {
|
||||
channel: channel,
|
||||
atime: stats.atime,
|
||||
mtime: stats.mtime,
|
||||
ctime: stats.ctime,
|
||||
size: stats.size,
|
||||
}, next);
|
||||
}));
|
||||
});
|
||||
});
|
||||
})));
|
||||
});
|
||||
});
|
||||
wait();
|
||||
}).nThen(function () {
|
||||
cb();
|
||||
});
|
||||
};
|
||||
|
||||
// move a channel's log file from its current location
|
||||
// to an equivalent location in the cold storage directory
|
||||
var archiveChannel = function (env, channelName, cb) {
|
||||
if (!env.retainData) {
|
||||
return void cb("ARCHIVES_DISABLED");
|
||||
}
|
||||
|
||||
// ctime is the most reliable indicator of when a file was archived
|
||||
// because it is used to indicate changes to the files metadata
|
||||
// and not its contents
|
||||
// if we find that this is not reliable in production, we can update it manually
|
||||
// https://nodejs.org/api/fs.html#fs_fs_utimes_path_atime_mtime_callback
|
||||
|
||||
// check what the channel's path should be (in its current location)
|
||||
var currentPath = mkPath(env, channelName);
|
||||
|
||||
// construct a parallel path in the new location
|
||||
var archivePath = mkArchivePath(env, channelName);
|
||||
|
||||
// use Fse.move to move it, Fse makes paths to the directory when you use it.
|
||||
// https://github.com/jprichardson/node-fs-extra/blob/HEAD/docs/move.md
|
||||
Fse.move(currentPath, archivePath, { overwrite: true }, cb);
|
||||
};
|
||||
|
||||
var unarchiveChannel = function (env, channelName, cb) {
|
||||
// very much like 'archiveChannel' but in the opposite direction
|
||||
|
||||
// the file is currently archived
|
||||
var currentPath = mkArchivePath(env, channelName);
|
||||
var unarchivedPath = mkPath(env, channelName);
|
||||
|
||||
// if a file exists in the unarchived path, you probably don't want to clobber its data
|
||||
// so unlike 'archiveChannel' we won't overwrite.
|
||||
// Fse.move will call back with EEXIST in such a situation
|
||||
Fse.move(currentPath, unarchivedPath, cb);
|
||||
};
|
||||
|
||||
var flushUnusedChannels = function (env, cb, frame) {
|
||||
var currentTime = +new Date();
|
||||
|
||||
@ -413,19 +538,30 @@ module.exports.create = function (
|
||||
) {
|
||||
var env = {
|
||||
root: conf.filePath || './datastore',
|
||||
archiveRoot: conf.archivePath || './data/archive',
|
||||
retainData: conf.retainData,
|
||||
channels: { },
|
||||
channelExpirationMs: conf.channelExpirationMs || 30000,
|
||||
verbose: conf.verbose,
|
||||
openFiles: 0,
|
||||
openFileLimit: conf.openFileLimit || 2048,
|
||||
};
|
||||
// 0x1ff -> 777
|
||||
var it;
|
||||
Fse.mkdirp(env.root, 0x1ff, function (err) {
|
||||
if (err && err.code !== 'EEXIST') {
|
||||
// TODO: somehow return a nice error
|
||||
throw err;
|
||||
}
|
||||
|
||||
nThen(function (w) {
|
||||
// make sure the store's directory exists
|
||||
Fse.mkdirp(env.root, PERMISSIVE, w(function (err) {
|
||||
if (err && err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
// make sure the cold storage directory exists
|
||||
Fse.mkdirp(env.archiveRoot, PERMISSIVE, w(function (err) {
|
||||
if (err && err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
}).nThen(function () {
|
||||
cb({
|
||||
readMessagesBin: (channelName, start, asyncMsgHandler, cb) => {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
@ -449,6 +585,10 @@ module.exports.create = function (
|
||||
cb(err);
|
||||
});
|
||||
},
|
||||
removeArchivedChannel: function (channelName, cb) {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
removeArchivedChannel(env, channelName, cb);
|
||||
},
|
||||
closeChannel: function (channelName, cb) {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
closeChannel(env, channelName, cb);
|
||||
@ -468,6 +608,32 @@ module.exports.create = function (
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
clearChannel(env, channelName, cb);
|
||||
},
|
||||
listChannels: function (handler, cb) {
|
||||
listChannels(env.root, handler, cb);
|
||||
},
|
||||
isChannelAvailable: function (channelName, cb) {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
// construct the path
|
||||
var filepath = mkPath(env, channelName);
|
||||
channelExists(filepath, channelName, cb);
|
||||
},
|
||||
isChannelArchived: function (channelName, cb) {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
// construct the path
|
||||
var filepath = mkArchivePath(env, channelName);
|
||||
channelExists(filepath, channelName, cb);
|
||||
},
|
||||
listArchivedChannels: function (handler, cb) {
|
||||
listChannels(Path.join(env.archiveRoot, 'datastore'), handler, cb);
|
||||
},
|
||||
archiveChannel: function (channelName, cb) {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
archiveChannel(env, channelName, cb);
|
||||
},
|
||||
restoreArchivedChannel: function (channelName, cb) {
|
||||
if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); }
|
||||
unarchiveChannel(env, channelName, cb);
|
||||
},
|
||||
log: function (channelName, content, cb) {
|
||||
message(env, channelName, content, cb);
|
||||
},
|
||||
|
||||
@ -83,6 +83,7 @@ var write = function (env, task, cb) {
|
||||
};
|
||||
|
||||
var remove = function (env, path, cb) {
|
||||
// FIXME COLDSTORAGE?
|
||||
Fs.unlink(path, cb);
|
||||
};
|
||||
|
||||
|
||||
@ -327,15 +327,6 @@ define([
|
||||
account.note = obj.note;
|
||||
cb(obj);
|
||||
});
|
||||
|
||||
arePinsSynced(function (err, yes) {
|
||||
if (!yes) {
|
||||
resetPins(function (err) {
|
||||
if (err) { return console.error(err); }
|
||||
console.log('RESET DONE');
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
@ -1690,6 +1681,15 @@ define([
|
||||
loadUniversal(Profile, 'profile', waitFor);
|
||||
cleanFriendRequests();
|
||||
}).nThen(function () {
|
||||
arePinsSynced(function (err, yes) {
|
||||
if (!yes) {
|
||||
resetPins(function (err) {
|
||||
if (err) { return console.error(err); }
|
||||
console.log('RESET DONE');
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
var requestLogin = function () {
|
||||
broadcast([], "REQUEST_LOGIN");
|
||||
};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user