Move the various scripts into a dedicated folder

This commit is contained in:
yflory
2019-04-08 16:31:36 +02:00
parent 05f464ce4d
commit 0870a0651f
6 changed files with 21 additions and 21 deletions

View File

@@ -0,0 +1,76 @@
/* jshint esversion: 6, node: true */
const Fs = require('fs');
const nThen = require('nthen');
const Pinned = require('./pinned');
const Nacl = require('tweetnacl');
const hashesFromPinFile = (pinFile, fileName) => {
var pins = {};
pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => {
switch (l[0]) {
case 'RESET': {
pins = {};
if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); }
//jshint -W086
// fallthrough
}
case 'PIN': {
l[1].forEach((x) => { pins[x] = 1; });
break;
}
case 'UNPIN': {
l[1].forEach((x) => { delete pins[x]; });
break;
}
default: throw new Error(JSON.stringify(l) + ' ' + fileName);
}
});
return Object.keys(pins);
};
var escapeKeyCharacters = function (key) {
return key && key.replace && key.replace(/\//g, '-');
};
const dataIdx = process.argv.indexOf('--data');
let edPublic;
if (dataIdx === -1) {
const hasEdPublic = process.argv.indexOf('--ed');
if (hasEdPublic === -1) { return void console.error("Missing ed argument"); }
edPublic = escapeKeyCharacters(process.argv[hasEdPublic+1]);
} else {
const deleteData = JSON.parse(process.argv[dataIdx+1]);
if (!deleteData.toSign || !deleteData.proof) { return void console.error("Invalid arguments"); }
// Check sig
const ed = Nacl.util.decodeBase64(deleteData.toSign.edPublic);
const signed = Nacl.util.decodeUTF8(JSON.stringify(deleteData.toSign));
const proof = Nacl.util.decodeBase64(deleteData.proof);
if (!Nacl.sign.detached.verify(signed, proof, ed)) { return void console.error("Invalid signature"); }
edPublic = escapeKeyCharacters(deleteData.toSign.edPublic);
}
let data = [];
let pinned = [];
nThen((waitFor) => {
let f = '../pins/' + edPublic.slice(0, 2) + '/' + edPublic + '.ndjson';
Fs.readFile(f, waitFor((err, content) => {
if (err) { throw err; }
pinned = hashesFromPinFile(content.toString('utf8'), f);
}));
}).nThen((waitFor) => {
Pinned.load(waitFor((d) => {
data = Object.keys(d);
}), {
exclude: [edPublic + '.ndjson']
});
}).nThen(() => {
console.log('Pads pinned by this user and not pinned by anybody else:');
pinned.forEach((p) => {
if (data.indexOf(p) === -1) {
console.log(p);
}
});
});

40
scripts/check-accounts.js Normal file
View File

@@ -0,0 +1,40 @@
/* globals Buffer */
var Https = require('https');
var Config = require("../config/config.js");
var Package = require("../package.json");
var body = JSON.stringify({
domain: Config.myDomain,
adminEmail: Config.adminEmail,
version: Package.version,
});
var options = {
host: 'accounts.cryptpad.fr',
path: '/api/getauthorized',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(body)
}
};
Https.request(options, function (response) {
if (!('' + response.statusCode).match(/^2\d\d$/)) {
throw new Error('SERVER ERROR ' + response.statusCode);
}
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
try {
var json = JSON.parse(str);
console.log(json);
} catch (e) {
throw new Error(e);
}
});
}).on('error', function (e) {
console.error(e);
}).end(body);

View File

@@ -0,0 +1,40 @@
/* jshint esversion: 6, node: true */
const Fs = require("fs");
const nThen = require("nthen");
const Saferphore = require("saferphore");
const PinnedData = require('./pinneddata');
let config;
try {
config = require('../config/config');
} catch (e) {
config = require('../config/config.example');
}
if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") { return; }
let inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
let inactiveConfig = {
unpinned: true,
olderthan: inactiveTime,
blobsolderthan: inactiveTime
};
let toDelete;
nThen(function (waitFor) {
PinnedData.load(inactiveConfig, waitFor(function (err, data) {
if (err) {
waitFor.abort();
throw new Error(err);
}
toDelete = data;
}));
}).nThen(function () {
var sem = Saferphore.create(10);
toDelete.forEach(function (f) {
sem.take(function (give) {
Fs.unlink(f.filename, give(function (err) {
if (err) { return void console.error(err + " " + f.filename); }
console.log(f.filename + " " + f.size + " " + (+f.mtime) + " " + (+new Date()));
}));
});
});
});

114
scripts/expire-channels.js Normal file
View File

@@ -0,0 +1,114 @@
var Fs = require("fs");
var Path = require("path");
var nThen = require("nthen");
var config;
try {
config = require('../config/config');
} catch (e) {
config = require('../config/config.example');
}
var FileStorage = require('../' + config.storage || './storage/file');
var root = Path.resolve('../' + config.taskPath || './tasks');
var dirs;
var nt;
var store;
var queue = function (f) {
nt = nt.nThen(f);
};
var tryParse = function (s) {
try { return JSON.parse(s); }
catch (e) { return null; }
};
var CURRENT = +new Date();
var handleTask = function (str, path, cb) {
var task = tryParse(str);
if (!Array.isArray(task)) {
console.error('invalid task: not array');
return cb();
}
if (task.length < 2) {
console.error('invalid task: too small');
return cb();
}
var time = task[0];
var command = task[1];
var args = task.slice(2);
if (time > CURRENT) {
// not time for this task yet
console.log('not yet time');
return cb();
}
nThen(function (waitFor) {
switch (command) {
case 'EXPIRE':
console.log("expiring: %s", args[0]);
store.removeChannel(args[0], waitFor());
break;
default:
console.log("unknown command", command);
}
}).nThen(function () {
// remove the task file...
Fs.unlink(path, function (err) { // FIXME deletion
if (err) { console.error(err); }
cb();
});
});
};
nt = nThen(function (w) {
Fs.readdir(root, w(function (e, list) {
if (e) { throw e; }
dirs = list;
if (dirs.length === 0) {
w.abort();
return;
}
}));
}).nThen(function (waitFor) {
FileStorage.create(config, waitFor(function (_store) {
store = _store;
}));
}).nThen(function () {
dirs.forEach(function (dir, dIdx) {
queue(function (w) {
console.log('recursing into %s', dir);
Fs.readdir(Path.join(root, dir), w(function (e, list) {
list.forEach(function (fn) {
queue(function (w) {
var filePath = Path.join(root, dir, fn);
var cb = w();
console.log("processing file at %s", filePath);
Fs.readFile(filePath, 'utf8', function (e, str) {
if (e) {
console.error(e);
return void cb();
}
handleTask(str, filePath, cb);
});
});
});
if (dIdx === (dirs.length - 1)) {
queue(function () {
store.shutdown();
});
}
}));
});
});
});

83
scripts/pinned.js Normal file
View File

@@ -0,0 +1,83 @@
/* jshint esversion: 6, node: true */
const Fs = require('fs');
const Semaphore = require('saferphore');
const nThen = require('nthen');
const sema = Semaphore.create(20);
let dirList;
const fileList = [];
const pinned = {};
const hashesFromPinFile = (pinFile, fileName) => {
var pins = {};
pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => {
switch (l[0]) {
case 'RESET': {
pins = {};
if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); }
//jshint -W086
// fallthrough
}
case 'PIN': {
l[1].forEach((x) => { pins[x] = 1; });
break;
}
case 'UNPIN': {
l[1].forEach((x) => { delete pins[x]; });
break;
}
default: throw new Error(JSON.stringify(l) + ' ' + fileName);
}
});
return Object.keys(pins);
};
module.exports.load = function (cb, config) {
nThen((waitFor) => {
Fs.readdir('../pins', waitFor((err, list) => {
if (err) {
if (err.code === 'ENOENT') {
dirList = [];
return;
}
throw err;
}
dirList = list;
}));
}).nThen((waitFor) => {
dirList.forEach((f) => {
sema.take((returnAfter) => {
Fs.readdir('../pins/' + f, waitFor(returnAfter((err, list2) => {
if (err) { throw err; }
list2.forEach((ff) => {
if (config && config.exclude && config.exclude.indexOf(ff) > -1) { return; }
fileList.push('../pins/' + f + '/' + ff);
});
})));
});
});
}).nThen((waitFor) => {
fileList.forEach((f) => {
sema.take((returnAfter) => {
Fs.readFile(f, waitFor(returnAfter((err, content) => {
if (err) { throw err; }
const hashes = hashesFromPinFile(content.toString('utf8'), f);
hashes.forEach((x) => {
(pinned[x] = pinned[x] || {})[f.replace(/.*\/([^/]*).ndjson$/, (x, y)=>y)] = 1;
});
})));
});
});
}).nThen(() => {
cb(pinned);
});
};
if (!module.parent) {
module.exports.load(function (data) {
Object.keys(data).forEach(function (x) {
console.log(x + ' ' + JSON.stringify(data[x]));
});
});
}

284
scripts/pinneddata.js Normal file
View File

@@ -0,0 +1,284 @@
/* jshint esversion: 6, node: true */
const Fs = require('fs');
const Semaphore = require('saferphore');
const nThen = require('nthen');
/*
takes contents of a pinFile (UTF8 string)
and the pin file's name
returns an array of of channel ids which are pinned
throw errors on pin logs with invalid pin data
*/
const hashesFromPinFile = (pinFile, fileName) => {
var pins = {};
pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => {
switch (l[0]) {
case 'RESET': {
pins = {};
if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); }
break;
}
case 'PIN': {
l[1].forEach((x) => { pins[x] = 1; });
break;
}
case 'UNPIN': {
l[1].forEach((x) => { delete pins[x]; });
break;
}
default: throw new Error(JSON.stringify(l) + ' ' + fileName);
}
});
return Object.keys(pins);
};
/*
takes an array of pinned file names
and a global map of stats indexed by public keys
returns the sum of the size of those pinned files
*/
const sizeForHashes = (hashes, dsFileStats) => {
let sum = 0;
hashes.forEach((h) => {
const s = dsFileStats[h];
if (typeof(s) !== 'object' || typeof(s.size) !== 'number') {
//console.log('missing ' + h + ' ' + typeof(s));
} else {
sum += s.size;
}
});
return sum;
};
// do twenty things at a time
const sema = Semaphore.create(20);
let dirList;
const fileList = []; // array which we reuse for a lot of things
const dsFileStats = {}; // map of stats
const out = []; // what we return at the end
const pinned = {}; // map of pinned files
// define a function: 'load' which takes a config
// and a callback
module.exports.load = function (config, cb) {
nThen((waitFor) => {
// read the subdirectories in the datastore
Fs.readdir('../datastore', waitFor((err, list) => {
if (err) { throw err; }
dirList = list;
}));
}).nThen((waitFor) => {
// iterate over all subdirectories
dirList.forEach((f) => {
// process twenty subdirectories simultaneously
sema.take((returnAfter) => {
// get the list of files in every subdirectory
// and push them to 'fileList'
Fs.readdir('../datastore/' + f, waitFor(returnAfter((err, list2) => {
if (err) { throw err; }
list2.forEach((ff) => { fileList.push('../datastore/' + f + '/' + ff); });
})));
});
});
}).nThen((waitFor) => {
// read the subdirectories in 'blob'
Fs.readdir('../blob', waitFor((err, list) => {
if (err) { throw err; }
// overwrite dirList
dirList = list;
}));
}).nThen((waitFor) => {
// iterate over all subdirectories
dirList.forEach((f) => {
// process twenty subdirectories simultaneously
sema.take((returnAfter) => {
// get the list of files in every subdirectory
// and push them to 'fileList'
Fs.readdir('../blob/' + f, waitFor(returnAfter((err, list2) => {
if (err) { throw err; }
list2.forEach((ff) => { fileList.push('../blob/' + f + '/' + ff); });
})));
});
});
}).nThen((waitFor) => {
// iterate over the fileList
fileList.forEach((f) => {
// process twenty files simultaneously
sema.take((returnAfter) => {
// get the stats of each files
Fs.stat(f, waitFor(returnAfter((err, st) => {
if (err) { throw err; }
st.filename = f;
// push them to a big map of stats
dsFileStats[f.replace(/^.*\/([^\/\.]*)(\.ndjson)?$/, (all, a) => (a))] = st;
})));
});
});
}).nThen((waitFor) => {
// read the subdirectories in the pinstore
Fs.readdir('../pins', waitFor((err, list) => {
if (err) { throw err; }
dirList = list;
}));
}).nThen((waitFor) => {
// set file list to an empty array
// fileList = [] ??
fileList.splice(0, fileList.length);
dirList.forEach((f) => {
// process twenty directories at a time
sema.take((returnAfter) => {
// get the list of files in every subdirectory
// and push them to 'fileList' (which is empty because we keep reusing it)
Fs.readdir('../pins/' + f, waitFor(returnAfter((err, list2) => {
if (err) { throw err; }
list2.forEach((ff) => { fileList.push('../pins/' + f + '/' + ff); });
})));
});
});
}).nThen((waitFor) => {
// iterate over the list of pin logs
fileList.forEach((f) => {
// twenty at a time
sema.take((returnAfter) => {
// read the full content
Fs.readFile(f, waitFor(returnAfter((err, content) => {
if (err) { throw err; }
// get the list of channels pinned by this log
const hashes = hashesFromPinFile(content.toString('utf8'), f);
if (config.unpinned) {
hashes.forEach((x) => { pinned[x] = 1; });
} else {
// get the size of files pinned by this log
// but only if we're gonna use it
let size = sizeForHashes(hashes, dsFileStats);
// we will return a list of values
// [user_public_key, size_of_files_they_have_pinned]
out.push([f, Math.floor(size / (1024 * 1024))]);
}
})));
});
});
}).nThen(() => {
// handle all the information you've processed so far
if (config.unpinned) {
// the user wants data about what has not been pinned
// by default we concern ourselves with pads and files older than infinity (everything)
let before = Infinity;
// but you can override this with config
if (config.olderthan) {
before = config.olderthan;
// FIXME validate inputs before doing the heavy lifting
if (isNaN(before)) { // make sure the supplied value is a number
return void cb('--olderthan error [' + config.olderthan + '] not a valid date');
}
}
// you can specify a different time for blobs...
let blobsbefore = before;
if (config.blobsolderthan) {
// use the supplied date if it exists
blobsbefore = config.blobsolderthan;
if (isNaN(blobsbefore)) {
return void cb('--blobsolderthan error [' + config.blobsolderthan + '] not a valid date');
}
}
let files = [];
// iterate over all the stats that you've saved
Object.keys(dsFileStats).forEach((f) => {
// we only care about files which are not in the pin map
if (!(f in pinned)) {
// check if it's a blob or a 'pad'
const isBlob = dsFileStats[f].filename.indexOf('.ndjson') === -1;
// if the mtime is newer than the specified value for its file type, ignore this file
if ((+dsFileStats[f].mtime) >= ((isBlob) ? blobsbefore : before)) { return; }
// otherwise push it to the list of files, with its filename, size, and mtime
files.push({
filename: dsFileStats[f].filename,
size: dsFileStats[f].size,
mtime: dsFileStats[f].mtime
});
}
});
// return the list of files
cb(null, files);
} else {
// if you're not in 'unpinned' mode, sort by size (ascending)
out.sort((a,b) => (a[1] - b[1]));
// and return the sorted data
cb(null, out.slice());
}
});
};
// This script can be called directly on its own
// or required as part of another script
if (!module.parent) {
// if no parent, it is being invoked directly
let config = {}; // build the config from command line arguments...
// --unpinned gets the list of unpinned files
// if you don't pass this, it will list the size of pinned data per user
if (process.argv.indexOf('--unpinned') > -1) { config.unpinned = true; }
// '--olderthan' must be used in conjunction with '--unpinned'
// if you pass '--olderthan' with a string date or number, it will limit
// results only to pads older than the supplied time
// it defaults to 'infinity', or no filter at all
const ot = process.argv.indexOf('--olderthan');
if (ot > -1) {
config.olderthan = Number(process.argv[ot+1]) ? new Date(Number(process.argv[ot+1]))
: new Date(process.argv[ot+1]);
}
// '--blobsolderthan' must be used in conjunction with '--unpinned'
// if you pass '--blobsolderthan with a string date or number, it will limit
// results only to blobs older than the supplied time
// it defaults to using the same value passed '--olderthan'
const bot = process.argv.indexOf('--blobsolderthan');
if (bot > -1) {
config.blobsolderthan = Number(process.argv[bot+1]) ? new Date(Number(process.argv[bot+1]))
: new Date(process.argv[bot+1]);
}
// call our big function directly
// pass our constructed configuration and a callback
module.exports.load(config, function (err, data) {
if (err) { throw new Error(err); } // throw errors
if (!Array.isArray(data)) { return; } // if the returned value is not an array, you're done
if (config.unpinned) {
// display the list of unpinned files with their size and mtime
data.forEach((f) => { console.log(f.filename + " " + f.size + " " + (+f.mtime)); });
} else {
// display the list of public keys and the size of the data they have pinned in megabytes
data.forEach((x) => { console.log(x[0] + ' ' + x[1] + ' MB'); });
}
});
}
/* Example usage of this script...
# display the list of public keys and the size of the data the have pinned in megabytes
node pinneddata.js
# display the list of unpinned pads and blobs with their size and mtime
node pinneddata.js --unpinned
# display the list of unpinned pads and blobs older than 12345 with their size and mtime
node pinneddata.js --unpinned --olderthan 12345
# display the list of unpinned pads older than 12345 and unpinned blobs older than 123
# each with their size and mtime
node pinneddata.js --unpinned --olderthan 12345 --blobsolderthan 123
*/