decaffeinate: rename app/coffee dir to app/js
This commit is contained in:
mserranom
20
app/js/CommandRunner.js
Normal file
20
app/js/CommandRunner.js
Normal file
@@ -0,0 +1,20 @@
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let commandRunnerPath;
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) {
|
||||
commandRunnerPath = "./DockerRunner";
|
||||
} else {
|
||||
commandRunnerPath = "./LocalCommandRunner";
|
||||
}
|
||||
logger.info({commandRunnerPath}, "selecting command runner for clsi");
|
||||
const CommandRunner = require(commandRunnerPath);
|
||||
|
||||
module.exports = CommandRunner;
|
||||
170
app/js/CompileController.js
Normal file
170
app/js/CompileController.js
Normal file
@@ -0,0 +1,170 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CompileController;
|
||||
const RequestParser = require("./RequestParser");
|
||||
const CompileManager = require("./CompileManager");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const ProjectPersistenceManager = require("./ProjectPersistenceManager");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Errors = require("./Errors");
|
||||
|
||||
module.exports = (CompileController = {
|
||||
compile(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const timer = new Metrics.Timer("compile-request");
|
||||
return RequestParser.parse(req.body, function(error, request) {
|
||||
if (error != null) { return next(error); }
|
||||
request.project_id = req.params.project_id;
|
||||
if (req.params.user_id != null) { request.user_id = req.params.user_id; }
|
||||
return ProjectPersistenceManager.markProjectAsJustAccessed(request.project_id, function(error) {
|
||||
if (error != null) { return next(error); }
|
||||
return CompileManager.doCompileWithLock(request, function(error, outputFiles) {
|
||||
let code, status;
|
||||
if (outputFiles == null) { outputFiles = []; }
|
||||
if (error instanceof Errors.AlreadyCompilingError) {
|
||||
code = 423; // Http 423 Locked
|
||||
status = "compile-in-progress";
|
||||
} else if (error instanceof Errors.FilesOutOfSyncError) {
|
||||
code = 409; // Http 409 Conflict
|
||||
status = "retry";
|
||||
} else if (error != null ? error.terminated : undefined) {
|
||||
status = "terminated";
|
||||
} else if (error != null ? error.validate : undefined) {
|
||||
status = `validation-${error.validate}`;
|
||||
} else if (error != null ? error.timedout : undefined) {
|
||||
status = "timedout";
|
||||
logger.log({err: error, project_id: request.project_id}, "timeout running compile");
|
||||
} else if (error != null) {
|
||||
status = "error";
|
||||
code = 500;
|
||||
logger.warn({err: error, project_id: request.project_id}, "error running compile");
|
||||
} else {
|
||||
let file;
|
||||
status = "failure";
|
||||
for (file of Array.from(outputFiles)) {
|
||||
if (file.path != null ? file.path.match(/output\.pdf$/) : undefined) {
|
||||
status = "success";
|
||||
}
|
||||
}
|
||||
|
||||
if (status === "failure") {
|
||||
logger.warn({project_id: request.project_id, outputFiles}, "project failed to compile successfully, no output.pdf generated");
|
||||
}
|
||||
|
||||
// log an error if any core files are found
|
||||
for (file of Array.from(outputFiles)) {
|
||||
if (file.path === "core") {
|
||||
logger.error({project_id:request.project_id, req, outputFiles}, "core file found in output");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (error != null) {
|
||||
outputFiles = error.outputFiles || [];
|
||||
}
|
||||
|
||||
timer.done();
|
||||
return res.status(code || 200).send({
|
||||
compile: {
|
||||
status,
|
||||
error: (error != null ? error.message : undefined) || error,
|
||||
outputFiles: outputFiles.map(file =>
|
||||
({
|
||||
url:
|
||||
`${Settings.apis.clsi.url}/project/${request.project_id}` +
|
||||
((request.user_id != null) ? `/user/${request.user_id}` : "") +
|
||||
((file.build != null) ? `/build/${file.build}` : "") +
|
||||
`/output/${file.path}`,
|
||||
path: file.path,
|
||||
type: file.type,
|
||||
build: file.build
|
||||
})
|
||||
)
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
stopCompile(req, res, next) {
|
||||
const {project_id, user_id} = req.params;
|
||||
return CompileManager.stopCompile(project_id, user_id, function(error) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.sendStatus(204);
|
||||
});
|
||||
},
|
||||
|
||||
clearCache(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
return ProjectPersistenceManager.clearProject(req.params.project_id, req.params.user_id, function(error) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.sendStatus(204);
|
||||
});
|
||||
}, // No content
|
||||
|
||||
syncFromCode(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const { file } = req.query;
|
||||
const line = parseInt(req.query.line, 10);
|
||||
const column = parseInt(req.query.column, 10);
|
||||
const { project_id } = req.params;
|
||||
const { user_id } = req.params;
|
||||
return CompileManager.syncFromCode(project_id, user_id, file, line, column, function(error, pdfPositions) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.json({
|
||||
pdf: pdfPositions
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
syncFromPdf(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const page = parseInt(req.query.page, 10);
|
||||
const h = parseFloat(req.query.h);
|
||||
const v = parseFloat(req.query.v);
|
||||
const { project_id } = req.params;
|
||||
const { user_id } = req.params;
|
||||
return CompileManager.syncFromPdf(project_id, user_id, page, h, v, function(error, codePositions) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.json({
|
||||
code: codePositions
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
wordcount(req, res, next) {
|
||||
if (next == null) { next = function(error) {}; }
|
||||
const file = req.query.file || "main.tex";
|
||||
const { project_id } = req.params;
|
||||
const { user_id } = req.params;
|
||||
const { image } = req.query;
|
||||
logger.log({image, file, project_id}, "word count request");
|
||||
|
||||
return CompileManager.wordcount(project_id, user_id, file, image, function(error, result) {
|
||||
if (error != null) { return next(error); }
|
||||
return res.json({
|
||||
texcount: result
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
status(req, res, next ){
|
||||
if (next == null) { next = function(error){}; }
|
||||
return res.send("OK");
|
||||
}
|
||||
});
|
||||
|
||||
463
app/js/CompileManager.js
Normal file
463
app/js/CompileManager.js
Normal file
@@ -0,0 +1,463 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-undef,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CompileManager;
|
||||
const ResourceWriter = require("./ResourceWriter");
|
||||
const LatexRunner = require("./LatexRunner");
|
||||
const OutputFileFinder = require("./OutputFileFinder");
|
||||
const OutputCacheManager = require("./OutputCacheManager");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const Path = require("path");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const child_process = require("child_process");
|
||||
const DraftModeManager = require("./DraftModeManager");
|
||||
const TikzManager = require("./TikzManager");
|
||||
const LockManager = require("./LockManager");
|
||||
const fs = require("fs");
|
||||
const fse = require("fs-extra");
|
||||
const os = require("os");
|
||||
const async = require("async");
|
||||
const Errors = require('./Errors');
|
||||
const CommandRunner = require("./CommandRunner");
|
||||
|
||||
const getCompileName = function(project_id, user_id) {
|
||||
if (user_id != null) { return `${project_id}-${user_id}`; } else { return project_id; }
|
||||
};
|
||||
|
||||
const getCompileDir = (project_id, user_id) => Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id));
|
||||
|
||||
module.exports = (CompileManager = {
|
||||
|
||||
doCompileWithLock(request, callback) {
|
||||
if (callback == null) { callback = function(error, outputFiles) {}; }
|
||||
const compileDir = getCompileDir(request.project_id, request.user_id);
|
||||
const lockFile = Path.join(compileDir, ".project-lock");
|
||||
// use a .project-lock file in the compile directory to prevent
|
||||
// simultaneous compiles
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return LockManager.runWithLock(lockFile, releaseLock => CompileManager.doCompile(request, releaseLock)
|
||||
, callback);
|
||||
});
|
||||
},
|
||||
|
||||
doCompile(request, callback) {
|
||||
if (callback == null) { callback = function(error, outputFiles) {}; }
|
||||
const compileDir = getCompileDir(request.project_id, request.user_id);
|
||||
let timer = new Metrics.Timer("write-to-disk");
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id}, "syncing resources to disk");
|
||||
return ResourceWriter.syncResourcesToDisk(request, compileDir, function(error, resourceList) {
|
||||
// NOTE: resourceList is insecure, it should only be used to exclude files from the output list
|
||||
if ((error != null) && error instanceof Errors.FilesOutOfSyncError) {
|
||||
logger.warn({project_id: request.project_id, user_id: request.user_id}, "files out of sync, please retry");
|
||||
return callback(error);
|
||||
} else if (error != null) {
|
||||
logger.err({err:error, project_id: request.project_id, user_id: request.user_id}, "error writing resources to disk");
|
||||
return callback(error);
|
||||
}
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start}, "written files to disk");
|
||||
timer.done();
|
||||
|
||||
const injectDraftModeIfRequired = function(callback) {
|
||||
if (request.draft) {
|
||||
return DraftModeManager.injectDraftMode(Path.join(compileDir, request.rootResourcePath), callback);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
};
|
||||
|
||||
const createTikzFileIfRequired = callback =>
|
||||
TikzManager.checkMainFile(compileDir, request.rootResourcePath, resourceList, function(error, needsMainFile) {
|
||||
if (error != null) { return callback(error); }
|
||||
if (needsMainFile) {
|
||||
return TikzManager.injectOutputFile(compileDir, request.rootResourcePath, callback);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
})
|
||||
;
|
||||
|
||||
// set up environment variables for chktex
|
||||
const env = {};
|
||||
// only run chktex on LaTeX files (not knitr .Rtex files or any others)
|
||||
const isLaTeXFile = request.rootResourcePath != null ? request.rootResourcePath.match(/\.tex$/i) : undefined;
|
||||
if ((request.check != null) && isLaTeXFile) {
|
||||
env.CHKTEX_OPTIONS = '-nall -e9 -e10 -w15 -w16';
|
||||
env.CHKTEX_ULIMIT_OPTIONS = '-t 5 -v 64000';
|
||||
if (request.check === 'error') {
|
||||
env.CHKTEX_EXIT_ON_ERROR = 1;
|
||||
}
|
||||
if (request.check === 'validate') {
|
||||
env.CHKTEX_VALIDATE = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// apply a series of file modifications/creations for draft mode and tikz
|
||||
return async.series([injectDraftModeIfRequired, createTikzFileIfRequired], function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
timer = new Metrics.Timer("run-compile");
|
||||
// find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
|
||||
let tag = __guard__(__guard__(request.imageName != null ? request.imageName.match(/:(.*)/) : undefined, x1 => x1[1]), x => x.replace(/\./g,'-')) || "default";
|
||||
if (!request.project_id.match(/^[0-9a-f]{24}$/)) { tag = "other"; } // exclude smoke test
|
||||
Metrics.inc("compiles");
|
||||
Metrics.inc(`compiles-with-image.${tag}`);
|
||||
const compileName = getCompileName(request.project_id, request.user_id);
|
||||
return LatexRunner.runLatex(compileName, {
|
||||
directory: compileDir,
|
||||
mainFile: request.rootResourcePath,
|
||||
compiler: request.compiler,
|
||||
timeout: request.timeout,
|
||||
image: request.imageName,
|
||||
flags: request.flags,
|
||||
environment: env
|
||||
}, function(error, output, stats, timings) {
|
||||
// request was for validation only
|
||||
let metric_key, metric_value;
|
||||
if (request.check === "validate") {
|
||||
const result = (error != null ? error.code : undefined) ? "fail" : "pass";
|
||||
error = new Error("validation");
|
||||
error.validate = result;
|
||||
}
|
||||
// request was for compile, and failed on validation
|
||||
if ((request.check === "error") && ((error != null ? error.message : undefined) === 'exited')) {
|
||||
error = new Error("compilation");
|
||||
error.validate = "fail";
|
||||
}
|
||||
// compile was killed by user, was a validation, or a compile which failed validation
|
||||
if ((error != null ? error.terminated : undefined) || (error != null ? error.validate : undefined) || (error != null ? error.timedout : undefined)) {
|
||||
OutputFileFinder.findOutputFiles(resourceList, compileDir, function(err, outputFiles) {
|
||||
if (err != null) { return callback(err); }
|
||||
error.outputFiles = outputFiles; // return output files so user can check logs
|
||||
return callback(error);
|
||||
});
|
||||
return;
|
||||
}
|
||||
// compile completed normally
|
||||
if (error != null) { return callback(error); }
|
||||
Metrics.inc("compiles-succeeded");
|
||||
const object = stats || {};
|
||||
for (metric_key in object) {
|
||||
metric_value = object[metric_key];
|
||||
Metrics.count(metric_key, metric_value);
|
||||
}
|
||||
const object1 = timings || {};
|
||||
for (metric_key in object1) {
|
||||
metric_value = object1[metric_key];
|
||||
Metrics.timing(metric_key, metric_value);
|
||||
}
|
||||
const loadavg = typeof os.loadavg === 'function' ? os.loadavg() : undefined;
|
||||
if (loadavg != null) { Metrics.gauge("load-avg", loadavg[0]); }
|
||||
const ts = timer.done();
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats, timings, loadavg}, "done compile");
|
||||
if ((stats != null ? stats["latex-runs"] : undefined) > 0) {
|
||||
Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]);
|
||||
}
|
||||
if (((stats != null ? stats["latex-runs"] : undefined) > 0) && ((timings != null ? timings["cpu-time"] : undefined) > 0)) {
|
||||
Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]);
|
||||
}
|
||||
|
||||
return OutputFileFinder.findOutputFiles(resourceList, compileDir, function(error, outputFiles) {
|
||||
if (error != null) { return callback(error); }
|
||||
return OutputCacheManager.saveOutputFiles(outputFiles, compileDir, (error, newOutputFiles) => callback(null, newOutputFiles));
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
stopCompile(project_id, user_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
return LatexRunner.killLatex(compileName, callback);
|
||||
},
|
||||
|
||||
clearProject(project_id, user_id, _callback) {
|
||||
if (_callback == null) { _callback = function(error) {}; }
|
||||
const callback = function(error) {
|
||||
_callback(error);
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
|
||||
return CompileManager._checkDirectory(compileDir, function(err, exists) {
|
||||
if (err != null) { return callback(err); }
|
||||
if (!exists) { return callback(); } // skip removal if no directory present
|
||||
|
||||
const proc = child_process.spawn("rm", ["-r", compileDir]);
|
||||
|
||||
proc.on("error", callback);
|
||||
|
||||
let stderr = "";
|
||||
proc.stderr.on("data", chunk => stderr += chunk.toString());
|
||||
|
||||
return proc.on("close", function(code) {
|
||||
if (code === 0) {
|
||||
return callback(null);
|
||||
} else {
|
||||
return callback(new Error(`rm -r ${compileDir} failed: ${stderr}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_findAllDirs(callback) {
|
||||
if (callback == null) { callback = function(error, allDirs) {}; }
|
||||
const root = Settings.path.compilesDir;
|
||||
return fs.readdir(root, function(err, files) {
|
||||
if (err != null) { return callback(err); }
|
||||
const allDirs = (Array.from(files).map((file) => Path.join(root, file)));
|
||||
return callback(null, allDirs);
|
||||
});
|
||||
},
|
||||
|
||||
clearExpiredProjects(max_cache_age_ms, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const now = Date.now();
|
||||
// action for each directory
|
||||
const expireIfNeeded = (checkDir, cb) =>
|
||||
fs.stat(checkDir, function(err, stats) {
|
||||
if (err != null) { return cb(); } // ignore errors checking directory
|
||||
const age = now - stats.mtime;
|
||||
const hasExpired = (age > max_cache_age_ms);
|
||||
if (hasExpired) { return fse.remove(checkDir, cb); } else { return cb(); }
|
||||
})
|
||||
;
|
||||
// iterate over all project directories
|
||||
return CompileManager._findAllDirs(function(error, allDirs) {
|
||||
if (error != null) { return callback(); }
|
||||
return async.eachSeries(allDirs, expireIfNeeded, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_checkDirectory(compileDir, callback) {
|
||||
if (callback == null) { callback = function(error, exists) {}; }
|
||||
return fs.lstat(compileDir, function(err, stats) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
return callback(null, false); // directory does not exist
|
||||
} else if (err != null) {
|
||||
logger.err({dir: compileDir, err}, "error on stat of project directory for removal");
|
||||
return callback(err);
|
||||
} else if (!(stats != null ? stats.isDirectory() : undefined)) {
|
||||
logger.err({dir: compileDir, stats}, "bad project directory for removal");
|
||||
return callback(new Error("project directory is not directory"));
|
||||
} else {
|
||||
return callback(null, true);
|
||||
}
|
||||
});
|
||||
}, // directory exists
|
||||
|
||||
syncFromCode(project_id, user_id, file_name, line, column, callback) {
|
||||
// If LaTeX was run in a virtual environment, the file path that synctex expects
|
||||
// might not match the file path on the host. The .synctex.gz file however, will be accessed
|
||||
// wherever it is on the host.
|
||||
if (callback == null) { callback = function(error, pdfPositions) {}; }
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
const base_dir = Settings.path.synctexBaseDir(compileName);
|
||||
const file_path = base_dir + "/" + file_name;
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
const synctex_path = `${base_dir}/output.pdf`;
|
||||
const command = ["code", synctex_path, file_path, line, column];
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code");
|
||||
return callback(error);
|
||||
}
|
||||
return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({project_id, user_id, file_name, line, column, command, stdout}, "synctex code output");
|
||||
return callback(null, CompileManager._parseSynctexFromCodeOutput(stdout));
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
syncFromPdf(project_id, user_id, page, h, v, callback) {
|
||||
if (callback == null) { callback = function(error, filePositions) {}; }
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
const base_dir = Settings.path.synctexBaseDir(compileName);
|
||||
const synctex_path = `${base_dir}/output.pdf`;
|
||||
const command = ["pdf", synctex_path, page, h, v];
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync to code");
|
||||
return callback(error);
|
||||
}
|
||||
return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({project_id, user_id, page, h, v, stdout}, "synctex pdf output");
|
||||
return callback(null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir));
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_checkFileExists(path, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const synctexDir = Path.dirname(path);
|
||||
const synctexFile = Path.join(synctexDir, "output.synctex.gz");
|
||||
return fs.stat(synctexDir, function(error, stats) {
|
||||
if ((error != null ? error.code : undefined) === 'ENOENT') {
|
||||
return callback(new Errors.NotFoundError("called synctex with no output directory"));
|
||||
}
|
||||
if (error != null) { return callback(error); }
|
||||
return fs.stat(synctexFile, function(error, stats) {
|
||||
if ((error != null ? error.code : undefined) === 'ENOENT') {
|
||||
return callback(new Errors.NotFoundError("called synctex with no output file"));
|
||||
}
|
||||
if (error != null) { return callback(error); }
|
||||
if (!(stats != null ? stats.isFile() : undefined)) { return callback(new Error("not a file")); }
|
||||
return callback();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_runSynctex(project_id, user_id, command, callback) {
|
||||
if (callback == null) { callback = function(error, stdout) {}; }
|
||||
const seconds = 1000;
|
||||
|
||||
command.unshift("/opt/synctex");
|
||||
|
||||
const directory = getCompileDir(project_id, user_id);
|
||||
const timeout = 60 * 1000; // increased to allow for large projects
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
return CommandRunner.run(compileName, command, directory, Settings.clsi != null ? Settings.clsi.docker.image : undefined, timeout, {}, function(error, output) {
|
||||
if (error != null) {
|
||||
logger.err({err:error, command, project_id, user_id}, "error running synctex");
|
||||
return callback(error);
|
||||
}
|
||||
return callback(null, output.stdout);
|
||||
});
|
||||
},
|
||||
|
||||
_parseSynctexFromCodeOutput(output) {
|
||||
const results = [];
|
||||
for (const line of Array.from(output.split("\n"))) {
|
||||
const [node, page, h, v, width, height] = Array.from(line.split("\t"));
|
||||
if (node === "NODE") {
|
||||
results.push({
|
||||
page: parseInt(page, 10),
|
||||
h: parseFloat(h),
|
||||
v: parseFloat(v),
|
||||
height: parseFloat(height),
|
||||
width: parseFloat(width)
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
},
|
||||
|
||||
_parseSynctexFromPdfOutput(output, base_dir) {
|
||||
const results = [];
|
||||
for (let line of Array.from(output.split("\n"))) {
|
||||
let column, file_path, node;
|
||||
[node, file_path, line, column] = Array.from(line.split("\t"));
|
||||
if (node === "NODE") {
|
||||
const file = file_path.slice(base_dir.length + 1);
|
||||
results.push({
|
||||
file,
|
||||
line: parseInt(line, 10),
|
||||
column: parseInt(column, 10)
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
},
|
||||
|
||||
|
||||
wordcount(project_id, user_id, file_name, image, callback) {
|
||||
if (callback == null) { callback = function(error, pdfPositions) {}; }
|
||||
logger.log({project_id, user_id, file_name, image}, "running wordcount");
|
||||
const file_path = `$COMPILE_DIR/${file_name}`;
|
||||
const command = [ "texcount", '-nocol', '-inc', file_path, `-out=${file_path}.wc`];
|
||||
const compileDir = getCompileDir(project_id, user_id);
|
||||
const timeout = 60 * 1000;
|
||||
const compileName = getCompileName(project_id, user_id);
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code");
|
||||
return callback(error);
|
||||
}
|
||||
return CommandRunner.run(compileName, command, compileDir, image, timeout, {}, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return fs.readFile(compileDir + "/" + file_name + ".wc", "utf-8", function(err, stdout) {
|
||||
if (err != null) {
|
||||
// call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
|
||||
logger.err({node_err:err, command, compileDir, project_id, user_id}, "error reading word count output");
|
||||
return callback(err);
|
||||
}
|
||||
const results = CompileManager._parseWordcountFromOutput(stdout);
|
||||
logger.log({project_id, user_id, wordcount: results}, "word count results");
|
||||
return callback(null, results);
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_parseWordcountFromOutput(output) {
|
||||
const results = {
|
||||
encode: "",
|
||||
textWords: 0,
|
||||
headWords: 0,
|
||||
outside: 0,
|
||||
headers: 0,
|
||||
elements: 0,
|
||||
mathInline: 0,
|
||||
mathDisplay: 0,
|
||||
errors: 0,
|
||||
messages: ""
|
||||
};
|
||||
for (const line of Array.from(output.split("\n"))) {
|
||||
const [data, info] = Array.from(line.split(":"));
|
||||
if (data.indexOf("Encoding") > -1) {
|
||||
results.encode = info.trim();
|
||||
}
|
||||
if (data.indexOf("in text") > -1) {
|
||||
results.textWords = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("in head") > -1) {
|
||||
results.headWords = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("outside") > -1) {
|
||||
results.outside = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("of head") > -1) {
|
||||
results.headers = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("Number of floats/tables/figures") > -1) {
|
||||
results.elements = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("Number of math inlines") > -1) {
|
||||
results.mathInline = parseInt(info, 10);
|
||||
}
|
||||
if (data.indexOf("Number of math displayed") > -1) {
|
||||
results.mathDisplay = parseInt(info, 10);
|
||||
}
|
||||
if (data === "(errors") { // errors reported as (errors:123)
|
||||
results.errors = parseInt(info, 10);
|
||||
}
|
||||
if (line.indexOf("!!! ") > -1) { // errors logged as !!! message !!!
|
||||
results.messages += line + "\n";
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
33
app/js/ContentTypeMapper.js
Normal file
33
app/js/ContentTypeMapper.js
Normal file
@@ -0,0 +1,33 @@
|
||||
/* eslint-disable
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
let ContentTypeMapper;
|
||||
const Path = require('path');
|
||||
|
||||
// here we coerce html, css and js to text/plain,
|
||||
// otherwise choose correct mime type based on file extension,
|
||||
// falling back to octet-stream
|
||||
module.exports = (ContentTypeMapper = {
|
||||
map(path) {
|
||||
switch (Path.extname(path)) {
|
||||
case '.txt': case '.html': case '.js': case '.css': case '.svg':
|
||||
return 'text/plain';
|
||||
case '.csv':
|
||||
return 'text/csv';
|
||||
case '.pdf':
|
||||
return 'application/pdf';
|
||||
case '.png':
|
||||
return 'image/png';
|
||||
case '.jpg': case '.jpeg':
|
||||
return 'image/jpeg';
|
||||
case '.tiff':
|
||||
return 'image/tiff';
|
||||
case '.gif':
|
||||
return 'image/gif';
|
||||
default:
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
}
|
||||
});
|
||||
18
app/js/DbQueue.js
Normal file
18
app/js/DbQueue.js
Normal file
@@ -0,0 +1,18 @@
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const async = require("async");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const queue = async.queue((task, cb)=> task(cb)
|
||||
, Settings.parallelSqlQueryLimit);
|
||||
|
||||
queue.drain = ()=> logger.debug('all items have been processed');
|
||||
|
||||
module.exports =
|
||||
{queue};
|
||||
|
||||
89
app/js/DockerLockManager.js
Normal file
89
app/js/DockerLockManager.js
Normal file
@@ -0,0 +1,89 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LockManager;
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
const LockState = {}; // locks for docker container operations, by container name
|
||||
|
||||
module.exports = (LockManager = {
|
||||
|
||||
MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock
|
||||
MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock
|
||||
LOCK_TEST_INTERVAL: 1000, // retry time
|
||||
|
||||
tryLock(key, callback) {
|
||||
let lockValue;
|
||||
if (callback == null) { callback = function(err, gotLock) {}; }
|
||||
const existingLock = LockState[key];
|
||||
if (existingLock != null) { // the lock is already taken, check how old it is
|
||||
const lockAge = Date.now() - existingLock.created;
|
||||
if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) {
|
||||
return callback(null, false); // we didn't get the lock, bail out
|
||||
} else {
|
||||
logger.error({key, lock: existingLock, age:lockAge}, "taking old lock by force");
|
||||
}
|
||||
}
|
||||
// take the lock
|
||||
LockState[key] = (lockValue = {created: Date.now()});
|
||||
return callback(null, true, lockValue);
|
||||
},
|
||||
|
||||
getLock(key, callback) {
|
||||
let attempt;
|
||||
if (callback == null) { callback = function(error, lockValue) {}; }
|
||||
const startTime = Date.now();
|
||||
return (attempt = () =>
|
||||
LockManager.tryLock(key, function(error, gotLock, lockValue) {
|
||||
if (error != null) { return callback(error); }
|
||||
if (gotLock) {
|
||||
return callback(null, lockValue);
|
||||
} else if ((Date.now() - startTime) > LockManager.MAX_LOCK_WAIT_TIME) {
|
||||
const e = new Error("Lock timeout");
|
||||
e.key = key;
|
||||
return callback(e);
|
||||
} else {
|
||||
return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL);
|
||||
}
|
||||
})
|
||||
)();
|
||||
},
|
||||
|
||||
releaseLock(key, lockValue, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const existingLock = LockState[key];
|
||||
if (existingLock === lockValue) { // lockValue is an object, so we can test by reference
|
||||
delete LockState[key]; // our lock, so we can free it
|
||||
return callback();
|
||||
} else if (existingLock != null) { // lock exists but doesn't match ours
|
||||
logger.error({key, lock: existingLock}, "tried to release lock taken by force");
|
||||
return callback();
|
||||
} else {
|
||||
logger.error({key, lock: existingLock}, "tried to release lock that has gone");
|
||||
return callback();
|
||||
}
|
||||
},
|
||||
|
||||
runWithLock(key, runner, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return LockManager.getLock(key, function(error, lockValue) {
|
||||
if (error != null) { return callback(error); }
|
||||
return runner((error1, ...args) =>
|
||||
LockManager.releaseLock(key, lockValue, function(error2) {
|
||||
error = error1 || error2;
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, ...Array.from(args));
|
||||
})
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
483
app/js/DockerRunner.js
Normal file
483
app/js/DockerRunner.js
Normal file
@@ -0,0 +1,483 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let DockerRunner, oneHour;
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Docker = require("dockerode");
|
||||
const dockerode = new Docker();
|
||||
const crypto = require("crypto");
|
||||
const async = require("async");
|
||||
const LockManager = require("./DockerLockManager");
|
||||
const fs = require("fs");
|
||||
const Path = require('path');
|
||||
const _ = require("underscore");
|
||||
|
||||
logger.info("using docker runner");
|
||||
|
||||
const usingSiblingContainers = () => __guard__(Settings != null ? Settings.path : undefined, x => x.sandboxedCompilesHostDir) != null;
|
||||
|
||||
module.exports = (DockerRunner = {
|
||||
ERR_NOT_DIRECTORY: new Error("not a directory"),
|
||||
ERR_TERMINATED: new Error("terminated"),
|
||||
ERR_EXITED: new Error("exited"),
|
||||
ERR_TIMED_OUT: new Error("container timed out"),
|
||||
|
||||
run(project_id, command, directory, image, timeout, environment, callback) {
|
||||
|
||||
let name;
|
||||
if (callback == null) { callback = function(error, output) {}; }
|
||||
if (usingSiblingContainers()) {
|
||||
const _newPath = Settings.path.sandboxedCompilesHostDir;
|
||||
logger.log({path: _newPath}, "altering bind path for sibling containers");
|
||||
// Server Pro, example:
|
||||
// '/var/lib/sharelatex/data/compiles/<project-id>'
|
||||
// ... becomes ...
|
||||
// '/opt/sharelatex_data/data/compiles/<project-id>'
|
||||
directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory));
|
||||
}
|
||||
|
||||
const volumes = {};
|
||||
volumes[directory] = "/compile";
|
||||
|
||||
command = (Array.from(command).map((arg) => __guardMethod__(arg.toString(), 'replace', o => o.replace('$COMPILE_DIR', "/compile"))));
|
||||
if ((image == null)) {
|
||||
({ image } = Settings.clsi.docker);
|
||||
}
|
||||
|
||||
if (Settings.texliveImageNameOveride != null) {
|
||||
const img = image.split("/");
|
||||
image = `${Settings.texliveImageNameOveride}/${img[2]}`;
|
||||
}
|
||||
|
||||
const options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment);
|
||||
const fingerprint = DockerRunner._fingerprintContainer(options);
|
||||
options.name = (name = `project-${project_id}-${fingerprint}`);
|
||||
|
||||
// logOptions = _.clone(options)
|
||||
// logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
|
||||
logger.log({project_id}, "running docker container");
|
||||
DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(error, output) {
|
||||
if (__guard__(error != null ? error.message : undefined, x => x.match("HTTP code is 500"))) {
|
||||
logger.log({err: error, project_id}, "error running container so destroying and retrying");
|
||||
return DockerRunner.destroyContainer(name, null, true, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return DockerRunner._runAndWaitForContainer(options, volumes, timeout, callback);
|
||||
});
|
||||
} else {
|
||||
return callback(error, output);
|
||||
}
|
||||
});
|
||||
|
||||
return name;
|
||||
}, // pass back the container name to allow it to be killed
|
||||
|
||||
kill(container_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({container_id}, "sending kill signal to container");
|
||||
const container = dockerode.getContainer(container_id);
|
||||
return container.kill(function(error) {
|
||||
if ((error != null) && __guardMethod__(error != null ? error.message : undefined, 'match', o => o.match(/Cannot kill container .* is not running/))) {
|
||||
logger.warn({err: error, container_id}, "container not running, continuing");
|
||||
error = null;
|
||||
}
|
||||
if (error != null) {
|
||||
logger.error({err: error, container_id}, "error killing container");
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_runAndWaitForContainer(options, volumes, timeout, _callback) {
|
||||
if (_callback == null) { _callback = function(error, output) {}; }
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []));
|
||||
// Only call the callback once
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
const { name } = options;
|
||||
|
||||
let streamEnded = false;
|
||||
let containerReturned = false;
|
||||
let output = {};
|
||||
|
||||
const callbackIfFinished = function() {
|
||||
if (streamEnded && containerReturned) {
|
||||
return callback(null, output);
|
||||
}
|
||||
};
|
||||
|
||||
const attachStreamHandler = function(error, _output) {
|
||||
if (error != null) { return callback(error); }
|
||||
output = _output;
|
||||
streamEnded = true;
|
||||
return callbackIfFinished();
|
||||
};
|
||||
|
||||
return DockerRunner.startContainer(options, volumes, attachStreamHandler, function(error, containerId) {
|
||||
if (error != null) { return callback(error); }
|
||||
|
||||
return DockerRunner.waitForContainer(name, timeout, function(error, exitCode) {
|
||||
let err;
|
||||
if (error != null) { return callback(error); }
|
||||
if (exitCode === 137) { // exit status from kill -9
|
||||
err = DockerRunner.ERR_TERMINATED;
|
||||
err.terminated = true;
|
||||
return callback(err);
|
||||
}
|
||||
if (exitCode === 1) { // exit status from chktex
|
||||
err = DockerRunner.ERR_EXITED;
|
||||
err.code = exitCode;
|
||||
return callback(err);
|
||||
}
|
||||
containerReturned = true;
|
||||
__guard__(options != null ? options.HostConfig : undefined, x => x.SecurityOpt = null); // small log line
|
||||
logger.log({err, exitCode, options}, "docker container has exited");
|
||||
return callbackIfFinished();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_getContainerOptions(command, image, volumes, timeout, environment) {
|
||||
let m, year;
|
||||
let key, value, hostVol, dockerVol;
|
||||
const timeoutInSeconds = timeout / 1000;
|
||||
|
||||
const dockerVolumes = {};
|
||||
for (hostVol in volumes) {
|
||||
dockerVol = volumes[hostVol];
|
||||
dockerVolumes[dockerVol] = {};
|
||||
|
||||
if (volumes[hostVol].slice(-3).indexOf(":r") === -1) {
|
||||
volumes[hostVol] = `${dockerVol}:rw`;
|
||||
}
|
||||
}
|
||||
|
||||
// merge settings and environment parameter
|
||||
const env = {};
|
||||
for (const src of [Settings.clsi.docker.env, environment || {}]) {
|
||||
for (key in src) { value = src[key]; env[key] = value; }
|
||||
}
|
||||
// set the path based on the image year
|
||||
if ((m = image.match(/:([0-9]+)\.[0-9]+/))) {
|
||||
year = m[1];
|
||||
} else {
|
||||
year = "2014";
|
||||
}
|
||||
env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`;
|
||||
const options = {
|
||||
"Cmd" : command,
|
||||
"Image" : image,
|
||||
"Volumes" : dockerVolumes,
|
||||
"WorkingDir" : "/compile",
|
||||
"NetworkDisabled" : true,
|
||||
"Memory" : 1024 * 1024 * 1024 * 1024, // 1 Gb
|
||||
"User" : Settings.clsi.docker.user,
|
||||
"Env" : (((() => {
|
||||
const result = [];
|
||||
for (key in env) {
|
||||
value = env[key];
|
||||
result.push(`${key}=${value}`);
|
||||
}
|
||||
return result;
|
||||
})())), // convert the environment hash to an array
|
||||
"HostConfig" : {
|
||||
"Binds": (((() => {
|
||||
const result1 = [];
|
||||
for (hostVol in volumes) {
|
||||
dockerVol = volumes[hostVol];
|
||||
result1.push(`${hostVol}:${dockerVol}`);
|
||||
}
|
||||
return result1;
|
||||
})())),
|
||||
"LogConfig": {"Type": "none", "Config": {}},
|
||||
"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}],
|
||||
"CapDrop": "ALL",
|
||||
"SecurityOpt": ["no-new-privileges"]
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
if ((Settings.path != null ? Settings.path.synctexBinHostPath : undefined) != null) {
|
||||
options.HostConfig.Binds.push(`${Settings.path.synctexBinHostPath}:/opt/synctex:ro`);
|
||||
}
|
||||
|
||||
if (Settings.clsi.docker.seccomp_profile != null) {
|
||||
options.HostConfig.SecurityOpt.push(`seccomp=${Settings.clsi.docker.seccomp_profile}`);
|
||||
}
|
||||
|
||||
return options;
|
||||
},
|
||||
|
||||
_fingerprintContainer(containerOptions) {
|
||||
// Yay, Hashing!
|
||||
const json = JSON.stringify(containerOptions);
|
||||
return crypto.createHash("md5").update(json).digest("hex");
|
||||
},
|
||||
|
||||
startContainer(options, volumes, attachStreamHandler, callback) {
|
||||
return LockManager.runWithLock(options.name, releaseLock =>
|
||||
// Check that volumes exist before starting the container.
|
||||
// When a container is started with volume pointing to a
|
||||
// non-existent directory then docker creates the directory but
|
||||
// with root ownership.
|
||||
DockerRunner._checkVolumes(options, volumes, function(err) {
|
||||
if (err != null) { return releaseLock(err); }
|
||||
return DockerRunner._startContainer(options, volumes, attachStreamHandler, releaseLock);
|
||||
})
|
||||
|
||||
, callback);
|
||||
},
|
||||
|
||||
// Check that volumes exist and are directories
|
||||
_checkVolumes(options, volumes, callback) {
|
||||
if (callback == null) { callback = function(error, containerName) {}; }
|
||||
if (usingSiblingContainers()) {
|
||||
// Server Pro, with sibling-containers active, skip checks
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
const checkVolume = (path, cb) =>
|
||||
fs.stat(path, function(err, stats) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!(stats != null ? stats.isDirectory() : undefined)) { return cb(DockerRunner.ERR_NOT_DIRECTORY); }
|
||||
return cb();
|
||||
})
|
||||
;
|
||||
const jobs = [];
|
||||
for (const vol in volumes) {
|
||||
(vol => jobs.push(cb => checkVolume(vol, cb)))(vol);
|
||||
}
|
||||
return async.series(jobs, callback);
|
||||
},
|
||||
|
||||
_startContainer(options, volumes, attachStreamHandler, callback) {
|
||||
if (callback == null) { callback = function(error, output) {}; }
|
||||
callback = _.once(callback);
|
||||
const { name } = options;
|
||||
|
||||
logger.log({container_name: name}, "starting container");
|
||||
const container = dockerode.getContainer(name);
|
||||
|
||||
const createAndStartContainer = () =>
|
||||
dockerode.createContainer(options, function(error, container) {
|
||||
if (error != null) { return callback(error); }
|
||||
return startExistingContainer();
|
||||
})
|
||||
;
|
||||
|
||||
var startExistingContainer = () =>
|
||||
DockerRunner.attachToContainer(options.name, attachStreamHandler, function(error){
|
||||
if (error != null) { return callback(error); }
|
||||
return container.start(function(error) {
|
||||
if ((error != null) && ((error != null ? error.statusCode : undefined) !== 304)) { // already running
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
})
|
||||
;
|
||||
|
||||
return container.inspect(function(error, stats){
|
||||
if ((error != null ? error.statusCode : undefined) === 404) {
|
||||
return createAndStartContainer();
|
||||
} else if (error != null) {
|
||||
logger.err({container_name: name, error}, "unable to inspect container to start");
|
||||
return callback(error);
|
||||
} else {
|
||||
return startExistingContainer();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
|
||||
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
|
||||
const container = dockerode.getContainer(containerId);
|
||||
return container.attach({stdout: 1, stderr: 1, stream: 1}, function(error, stream) {
|
||||
if (error != null) {
|
||||
logger.error({err: error, container_id: containerId}, "error attaching to container");
|
||||
return attachStartCallback(error);
|
||||
} else {
|
||||
attachStartCallback();
|
||||
}
|
||||
|
||||
|
||||
logger.log({container_id: containerId}, "attached to container");
|
||||
|
||||
const MAX_OUTPUT = 1024 * 1024; // limit output to 1MB
|
||||
const createStringOutputStream = function(name) {
|
||||
return {
|
||||
data: "",
|
||||
overflowed: false,
|
||||
write(data) {
|
||||
if (this.overflowed) { return; }
|
||||
if (this.data.length < MAX_OUTPUT) {
|
||||
return this.data += data;
|
||||
} else {
|
||||
logger.error({container_id: containerId, length: this.data.length, maxLen: MAX_OUTPUT}, `${name} exceeds max size`);
|
||||
this.data += `(...truncated at ${MAX_OUTPUT} chars...)`;
|
||||
return this.overflowed = true;
|
||||
}
|
||||
}
|
||||
// kill container if too much output
|
||||
// docker.containers.kill(containerId, () ->)
|
||||
};
|
||||
};
|
||||
|
||||
const stdout = createStringOutputStream("stdout");
|
||||
const stderr = createStringOutputStream("stderr");
|
||||
|
||||
container.modem.demuxStream(stream, stdout, stderr);
|
||||
|
||||
stream.on("error", err => logger.error({err, container_id: containerId}, "error reading from container stream"));
|
||||
|
||||
return stream.on("end", () => attachStreamHandler(null, {stdout: stdout.data, stderr: stderr.data}));
|
||||
});
|
||||
},
|
||||
|
||||
waitForContainer(containerId, timeout, _callback) {
|
||||
if (_callback == null) { _callback = function(error, exitCode) {}; }
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []));
|
||||
// Only call the callback once
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
const container = dockerode.getContainer(containerId);
|
||||
|
||||
let timedOut = false;
|
||||
const timeoutId = setTimeout(function() {
|
||||
timedOut = true;
|
||||
logger.log({container_id: containerId}, "timeout reached, killing container");
|
||||
return container.kill(function() {});
|
||||
}
|
||||
, timeout);
|
||||
|
||||
logger.log({container_id: containerId}, "waiting for docker container");
|
||||
return container.wait(function(error, res) {
|
||||
if (error != null) {
|
||||
clearTimeout(timeoutId);
|
||||
logger.error({err: error, container_id: containerId}, "error waiting for container");
|
||||
return callback(error);
|
||||
}
|
||||
if (timedOut) {
|
||||
logger.log({containerId}, "docker container timed out");
|
||||
error = DockerRunner.ERR_TIMED_OUT;
|
||||
error.timedout = true;
|
||||
return callback(error);
|
||||
} else {
|
||||
clearTimeout(timeoutId);
|
||||
logger.log({container_id: containerId, exitCode: res.StatusCode}, "docker container returned");
|
||||
return callback(null, res.StatusCode);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
destroyContainer(containerName, containerId, shouldForce, callback) {
|
||||
// We want the containerName for the lock and, ideally, the
|
||||
// containerId to delete. There is a bug in the docker.io module
|
||||
// where if you delete by name and there is an error, it throws an
|
||||
// async exception, but if you delete by id it just does a normal
|
||||
// error callback. We fall back to deleting by name if no id is
|
||||
// supplied.
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return LockManager.runWithLock(containerName, releaseLock => DockerRunner._destroyContainer(containerId || containerName, shouldForce, releaseLock)
|
||||
, callback);
|
||||
},
|
||||
|
||||
_destroyContainer(containerId, shouldForce, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({container_id: containerId}, "destroying docker container");
|
||||
const container = dockerode.getContainer(containerId);
|
||||
return container.remove({force: shouldForce === true}, function(error) {
|
||||
if ((error != null) && ((error != null ? error.statusCode : undefined) === 404)) {
|
||||
logger.warn({err: error, container_id: containerId}, "container not found, continuing");
|
||||
error = null;
|
||||
}
|
||||
if (error != null) {
|
||||
logger.error({err: error, container_id: containerId}, "error destroying container");
|
||||
} else {
|
||||
logger.log({container_id: containerId}, "destroyed container");
|
||||
}
|
||||
return callback(error);
|
||||
});
|
||||
},
|
||||
|
||||
// handle expiry of docker containers
|
||||
|
||||
MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000),
|
||||
|
||||
examineOldContainer(container, callback) {
|
||||
if (callback == null) { callback = function(error, name, id, ttl){}; }
|
||||
const name = container.Name || (container.Names != null ? container.Names[0] : undefined);
|
||||
const created = container.Created * 1000; // creation time is returned in seconds
|
||||
const now = Date.now();
|
||||
const age = now - created;
|
||||
const maxAge = DockerRunner.MAX_CONTAINER_AGE;
|
||||
const ttl = maxAge - age;
|
||||
logger.log({containerName: name, created, now, age, maxAge, ttl}, "checking whether to destroy container");
|
||||
return callback(null, name, container.Id, ttl);
|
||||
},
|
||||
|
||||
destroyOldContainers(callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return dockerode.listContainers({all: true}, function(error, containers) {
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = [];
|
||||
for (const container of Array.from(containers || [])) {
|
||||
(container =>
|
||||
DockerRunner.examineOldContainer(container, function(err, name, id, ttl) {
|
||||
if ((name.slice(0, 9) === '/project-') && (ttl <= 0)) {
|
||||
return jobs.push(cb => DockerRunner.destroyContainer(name, id, false, () => cb()));
|
||||
}
|
||||
})
|
||||
)(container);
|
||||
}
|
||||
// Ignore errors because some containers get stuck but
|
||||
// will be destroyed next time
|
||||
return async.series(jobs, callback);
|
||||
});
|
||||
},
|
||||
|
||||
startContainerMonitor() {
|
||||
logger.log({maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry");
|
||||
// randomise the start time
|
||||
const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000);
|
||||
return setTimeout(() =>
|
||||
setInterval(() => DockerRunner.destroyOldContainers()
|
||||
, (oneHour = 60 * 60 * 1000))
|
||||
|
||||
, randomDelay);
|
||||
}
|
||||
});
|
||||
|
||||
DockerRunner.startContainerMonitor();
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
function __guardMethod__(obj, methodName, transform) {
|
||||
if (typeof obj !== 'undefined' && obj !== null && typeof obj[methodName] === 'function') {
|
||||
return transform(obj, methodName);
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
44
app/js/DraftModeManager.js
Normal file
44
app/js/DraftModeManager.js
Normal file
@@ -0,0 +1,44 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-useless-escape,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let DraftModeManager;
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
module.exports = (DraftModeManager = {
|
||||
injectDraftMode(filename, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.readFile(filename, "utf8", function(error, content) {
|
||||
if (error != null) { return callback(error); }
|
||||
// avoid adding draft mode more than once
|
||||
if ((content != null ? content.indexOf("\\documentclass\[draft") : undefined) >= 0) {
|
||||
return callback();
|
||||
}
|
||||
const modified_content = DraftModeManager._injectDraftOption(content);
|
||||
logger.log({
|
||||
content: content.slice(0,1024), // \documentclass is normally v near the top
|
||||
modified_content: modified_content.slice(0,1024),
|
||||
filename
|
||||
}, "injected draft class");
|
||||
return fs.writeFile(filename, modified_content, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_injectDraftOption(content) {
|
||||
return content
|
||||
// With existing options (must be first, otherwise both are applied)
|
||||
.replace(/\\documentclass\[/g, "\\documentclass[draft,")
|
||||
// Without existing options
|
||||
.replace(/\\documentclass\{/g, "\\documentclass[draft]{");
|
||||
}
|
||||
});
|
||||
36
app/js/Errors.js
Normal file
36
app/js/Errors.js
Normal file
@@ -0,0 +1,36 @@
|
||||
/* eslint-disable
|
||||
no-proto,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
let Errors;
|
||||
var NotFoundError = function(message) {
|
||||
const error = new Error(message);
|
||||
error.name = "NotFoundError";
|
||||
error.__proto__ = NotFoundError.prototype;
|
||||
return error;
|
||||
};
|
||||
NotFoundError.prototype.__proto__ = Error.prototype;
|
||||
|
||||
var FilesOutOfSyncError = function(message) {
|
||||
const error = new Error(message);
|
||||
error.name = "FilesOutOfSyncError";
|
||||
error.__proto__ = FilesOutOfSyncError.prototype;
|
||||
return error;
|
||||
};
|
||||
FilesOutOfSyncError.prototype.__proto__ = Error.prototype;
|
||||
|
||||
var AlreadyCompilingError = function(message) {
|
||||
const error = new Error(message);
|
||||
error.name = "AlreadyCompilingError";
|
||||
error.__proto__ = AlreadyCompilingError.prototype;
|
||||
return error;
|
||||
};
|
||||
AlreadyCompilingError.prototype.__proto__ = Error.prototype;
|
||||
|
||||
module.exports = (Errors = {
|
||||
NotFoundError,
|
||||
FilesOutOfSyncError,
|
||||
AlreadyCompilingError
|
||||
});
|
||||
131
app/js/LatexRunner.js
Normal file
131
app/js/LatexRunner.js
Normal file
@@ -0,0 +1,131 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LatexRunner;
|
||||
const Path = require("path");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const CommandRunner = require("./CommandRunner");
|
||||
|
||||
const ProcessTable = {}; // table of currently running jobs (pids or docker container names)
|
||||
|
||||
module.exports = (LatexRunner = {
|
||||
runLatex(project_id, options, callback) {
|
||||
let command;
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
let {directory, mainFile, compiler, timeout, image, environment, flags} = options;
|
||||
if (!compiler) { compiler = "pdflatex"; }
|
||||
if (!timeout) { timeout = 60000; } // milliseconds
|
||||
|
||||
logger.log({directory, compiler, timeout, mainFile, environment, flags}, "starting compile");
|
||||
|
||||
// We want to run latexmk on the tex file which we will automatically
|
||||
// generate from the Rtex/Rmd/md file.
|
||||
mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex");
|
||||
|
||||
if (compiler === "pdflatex") {
|
||||
command = LatexRunner._pdflatexCommand(mainFile, flags);
|
||||
} else if (compiler === "latex") {
|
||||
command = LatexRunner._latexCommand(mainFile, flags);
|
||||
} else if (compiler === "xelatex") {
|
||||
command = LatexRunner._xelatexCommand(mainFile, flags);
|
||||
} else if (compiler === "lualatex") {
|
||||
command = LatexRunner._lualatexCommand(mainFile, flags);
|
||||
} else {
|
||||
return callback(new Error(`unknown compiler: ${compiler}`));
|
||||
}
|
||||
|
||||
if (Settings.clsi != null ? Settings.clsi.strace : undefined) {
|
||||
command = ["strace", "-o", "strace", "-ff"].concat(command);
|
||||
}
|
||||
|
||||
const id = `${project_id}`; // record running project under this id
|
||||
|
||||
return ProcessTable[id] = CommandRunner.run(project_id, command, directory, image, timeout, environment, function(error, output) {
|
||||
delete ProcessTable[id];
|
||||
if (error != null) { return callback(error); }
|
||||
const runs = __guard__(__guard__(output != null ? output.stderr : undefined, x1 => x1.match(/^Run number \d+ of .*latex/mg)), x => x.length) || 0;
|
||||
const failed = (__guard__(output != null ? output.stdout : undefined, x2 => x2.match(/^Latexmk: Errors/m)) != null) ? 1 : 0;
|
||||
// counters from latexmk output
|
||||
const stats = {};
|
||||
stats["latexmk-errors"] = failed;
|
||||
stats["latex-runs"] = runs;
|
||||
stats["latex-runs-with-errors"] = failed ? runs : 0;
|
||||
stats[`latex-runs-${runs}`] = 1;
|
||||
stats[`latex-runs-with-errors-${runs}`] = failed ? 1 : 0;
|
||||
// timing information from /usr/bin/time
|
||||
const timings = {};
|
||||
const stderr = output != null ? output.stderr : undefined;
|
||||
timings["cpu-percent"] = __guard__(stderr != null ? stderr.match(/Percent of CPU this job got: (\d+)/m) : undefined, x3 => x3[1]) || 0;
|
||||
timings["cpu-time"] = __guard__(stderr != null ? stderr.match(/User time.*: (\d+.\d+)/m) : undefined, x4 => x4[1]) || 0;
|
||||
timings["sys-time"] = __guard__(stderr != null ? stderr.match(/System time.*: (\d+.\d+)/m) : undefined, x5 => x5[1]) || 0;
|
||||
return callback(error, output, stats, timings);
|
||||
});
|
||||
},
|
||||
|
||||
killLatex(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const id = `${project_id}`;
|
||||
logger.log({id}, "killing running compile");
|
||||
if ((ProcessTable[id] == null)) {
|
||||
logger.warn({id}, "no such project to kill");
|
||||
return callback(null);
|
||||
} else {
|
||||
return CommandRunner.kill(ProcessTable[id], callback);
|
||||
}
|
||||
},
|
||||
|
||||
_latexmkBaseCommand(flags) {
|
||||
let args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"];
|
||||
if (flags) {
|
||||
args = args.concat(flags);
|
||||
}
|
||||
return (__guard__(Settings != null ? Settings.clsi : undefined, x => x.latexmkCommandPrefix) || []).concat(args);
|
||||
},
|
||||
|
||||
_pdflatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-pdf",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]);
|
||||
},
|
||||
|
||||
_latexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-pdfdvi",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]);
|
||||
},
|
||||
|
||||
_xelatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-xelatex",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]);
|
||||
},
|
||||
|
||||
_lualatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
"-lualatex",
|
||||
Path.join("$COMPILE_DIR", mainFile)
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
74
app/js/LocalCommandRunner.js
Normal file
74
app/js/LocalCommandRunner.js
Normal file
@@ -0,0 +1,74 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CommandRunner;
|
||||
const { spawn } = require("child_process");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
logger.info("using standard command runner");
|
||||
|
||||
module.exports = (CommandRunner = {
|
||||
run(project_id, command, directory, image, timeout, environment, callback) {
|
||||
let key, value;
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
command = (Array.from(command).map((arg) => arg.toString().replace('$COMPILE_DIR', directory)));
|
||||
logger.log({project_id, command, directory}, "running command");
|
||||
logger.warn("timeouts and sandboxing are not enabled with CommandRunner");
|
||||
|
||||
// merge environment settings
|
||||
const env = {};
|
||||
for (key in process.env) { value = process.env[key]; env[key] = value; }
|
||||
for (key in environment) { value = environment[key]; env[key] = value; }
|
||||
|
||||
// run command as detached process so it has its own process group (which can be killed if needed)
|
||||
const proc = spawn(command[0], command.slice(1), {cwd: directory, env});
|
||||
|
||||
let stdout = "";
|
||||
proc.stdout.on("data", data=> stdout += data);
|
||||
|
||||
proc.on("error", function(err){
|
||||
logger.err({err, project_id, command, directory}, "error running command");
|
||||
return callback(err);
|
||||
});
|
||||
|
||||
proc.on("close", function(code, signal) {
|
||||
let err;
|
||||
logger.info({code, signal, project_id}, "command exited");
|
||||
if (signal === 'SIGTERM') { // signal from kill method below
|
||||
err = new Error("terminated");
|
||||
err.terminated = true;
|
||||
return callback(err);
|
||||
} else if (code === 1) { // exit status from chktex
|
||||
err = new Error("exited");
|
||||
err.code = code;
|
||||
return callback(err);
|
||||
} else {
|
||||
return callback(null, {"stdout": stdout});
|
||||
}
|
||||
});
|
||||
|
||||
return proc.pid;
|
||||
}, // return process id to allow job to be killed if necessary
|
||||
|
||||
kill(pid, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
try {
|
||||
process.kill(-pid); // kill all processes in group
|
||||
} catch (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
56
app/js/LockManager.js
Normal file
56
app/js/LockManager.js
Normal file
@@ -0,0 +1,56 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LockManager;
|
||||
const Settings = require('settings-sharelatex');
|
||||
const logger = require("logger-sharelatex");
|
||||
const Lockfile = require('lockfile'); // from https://github.com/npm/lockfile
|
||||
const Errors = require("./Errors");
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
module.exports = (LockManager = {
|
||||
LOCK_TEST_INTERVAL: 1000, // 50ms between each test of the lock
|
||||
MAX_LOCK_WAIT_TIME: 15000, // 10s maximum time to spend trying to get the lock
|
||||
LOCK_STALE: 5*60*1000, // 5 mins time until lock auto expires
|
||||
|
||||
runWithLock(path, runner, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const lockOpts = {
|
||||
wait: this.MAX_LOCK_WAIT_TIME,
|
||||
pollPeriod: this.LOCK_TEST_INTERVAL,
|
||||
stale: this.LOCK_STALE
|
||||
};
|
||||
return Lockfile.lock(path, lockOpts, function(error) {
|
||||
if ((error != null ? error.code : undefined) === 'EEXIST') {
|
||||
return callback(new Errors.AlreadyCompilingError("compile in progress"));
|
||||
} else if (error != null) {
|
||||
return fs.lstat(path, (statLockErr, statLock)=>
|
||||
fs.lstat(Path.dirname(path), (statDirErr, statDir)=>
|
||||
fs.readdir(Path.dirname(path), function(readdirErr, readdirDir){
|
||||
logger.err({error, path, statLock, statLockErr, statDir, statDirErr, readdirErr, readdirDir}, "unable to get lock");
|
||||
return callback(error);
|
||||
})
|
||||
)
|
||||
);
|
||||
} else {
|
||||
return runner((error1, ...args) =>
|
||||
Lockfile.unlock(path, function(error2) {
|
||||
error = error1 || error2;
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, ...Array.from(args));
|
||||
})
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
4
app/js/Metrics.js
Normal file
4
app/js/Metrics.js
Normal file
@@ -0,0 +1,4 @@
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
module.exports = require("metrics-sharelatex");
|
||||
|
||||
275
app/js/OutputCacheManager.js
Normal file
275
app/js/OutputCacheManager.js
Normal file
@@ -0,0 +1,275 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS104: Avoid inline assignments
|
||||
* DS204: Change includes calls to have a more natural evaluation order
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputCacheManager;
|
||||
const async = require("async");
|
||||
const fs = require("fs");
|
||||
const fse = require("fs-extra");
|
||||
const Path = require("path");
|
||||
const logger = require("logger-sharelatex");
|
||||
const _ = require("underscore");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const crypto = require("crypto");
|
||||
|
||||
const OutputFileOptimiser = require("./OutputFileOptimiser");
|
||||
|
||||
module.exports = (OutputCacheManager = {
|
||||
CACHE_SUBDIR: '.cache/clsi',
|
||||
ARCHIVE_SUBDIR: '.archive/clsi',
|
||||
// build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
|
||||
// for backwards compatibility, make the randombytes part optional
|
||||
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
|
||||
CACHE_LIMIT: 2, // maximum number of cache directories
|
||||
CACHE_AGE: 60*60*1000, // up to one hour old
|
||||
|
||||
path(buildId, file) {
|
||||
// used by static server, given build id return '.cache/clsi/buildId'
|
||||
if (buildId.match(OutputCacheManager.BUILD_REGEX)) {
|
||||
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file);
|
||||
} else {
|
||||
// for invalid build id, return top level
|
||||
return file;
|
||||
}
|
||||
},
|
||||
|
||||
generateBuildId(callback) {
|
||||
// generate a secure build id from Date.now() and 8 random bytes in hex
|
||||
if (callback == null) { callback = function(error, buildId) {}; }
|
||||
return crypto.randomBytes(8, function(err, buf) {
|
||||
if (err != null) { return callback(err); }
|
||||
const random = buf.toString('hex');
|
||||
const date = Date.now().toString(16);
|
||||
return callback(err, `${date}-${random}`);
|
||||
});
|
||||
},
|
||||
|
||||
saveOutputFiles(outputFiles, compileDir, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return OutputCacheManager.generateBuildId(function(err, buildId) {
|
||||
if (err != null) { return callback(err); }
|
||||
return OutputCacheManager.saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback);
|
||||
});
|
||||
},
|
||||
|
||||
saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback) {
|
||||
// make a compileDir/CACHE_SUBDIR/build_id directory and
|
||||
// copy all the output files into it
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR);
|
||||
// Put the files into a new cache subdirectory
|
||||
const cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId);
|
||||
// Is it a per-user compile? check if compile directory is PROJECTID-USERID
|
||||
const perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/);
|
||||
|
||||
// Archive logs in background
|
||||
if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) || (Settings.clsi != null ? Settings.clsi.strace : undefined)) {
|
||||
OutputCacheManager.archiveLogs(outputFiles, compileDir, buildId, function(err) {
|
||||
if (err != null) {
|
||||
return logger.warn({err}, "erroring archiving log files");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// make the new cache directory
|
||||
return fse.ensureDir(cacheDir, function(err) {
|
||||
if (err != null) {
|
||||
logger.error({err, directory: cacheDir}, "error creating cache directory");
|
||||
return callback(err, outputFiles);
|
||||
} else {
|
||||
// copy all the output files into the new cache directory
|
||||
const results = [];
|
||||
return async.mapSeries(outputFiles, function(file, cb) {
|
||||
// don't send dot files as output, express doesn't serve them
|
||||
if (OutputCacheManager._fileIsHidden(file.path)) {
|
||||
logger.debug({compileDir, path: file.path}, "ignoring dotfile in output");
|
||||
return cb();
|
||||
}
|
||||
// copy other files into cache directory if valid
|
||||
const newFile = _.clone(file);
|
||||
const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]);
|
||||
return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!isSafe) {
|
||||
return cb();
|
||||
}
|
||||
return OutputCacheManager._checkIfShouldCopy(src, function(err, shouldCopy) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!shouldCopy) {
|
||||
return cb();
|
||||
}
|
||||
return OutputCacheManager._copyFile(src, dst, function(err) {
|
||||
if (err != null) { return cb(err); }
|
||||
newFile.build = buildId; // attach a build id if we cached the file
|
||||
results.push(newFile);
|
||||
return cb();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
, function(err) {
|
||||
if (err != null) {
|
||||
// pass back the original files if we encountered *any* error
|
||||
callback(err, outputFiles);
|
||||
// clean up the directory we just created
|
||||
return fse.remove(cacheDir, function(err) {
|
||||
if (err != null) {
|
||||
return logger.error({err, dir: cacheDir}, "error removing cache dir after failure");
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// pass back the list of new files in the cache
|
||||
callback(err, results);
|
||||
// let file expiry run in the background, expire all previous files if per-user
|
||||
return OutputCacheManager.expireOutputFiles(cacheRoot, {keep: buildId, limit: perUser ? 1 : null});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
archiveLogs(outputFiles, compileDir, buildId, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId);
|
||||
logger.log({dir: archiveDir}, "archiving log files for project");
|
||||
return fse.ensureDir(archiveDir, function(err) {
|
||||
if (err != null) { return callback(err); }
|
||||
return async.mapSeries(outputFiles, function(file, cb) {
|
||||
const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]);
|
||||
return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!isSafe) { return cb(); }
|
||||
return OutputCacheManager._checkIfShouldArchive(src, function(err, shouldArchive) {
|
||||
if (err != null) { return cb(err); }
|
||||
if (!shouldArchive) { return cb(); }
|
||||
return OutputCacheManager._copyFile(src, dst, cb);
|
||||
});
|
||||
});
|
||||
}
|
||||
, callback);
|
||||
});
|
||||
},
|
||||
|
||||
expireOutputFiles(cacheRoot, options, callback) {
|
||||
// look in compileDir for build dirs and delete if > N or age of mod time > T
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.readdir(cacheRoot, function(err, results) {
|
||||
if (err != null) {
|
||||
if (err.code === 'ENOENT') { return callback(null); } // cache directory is empty
|
||||
logger.error({err, project_id: cacheRoot}, "error clearing cache");
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
const dirs = results.sort().reverse();
|
||||
const currentTime = Date.now();
|
||||
|
||||
const isExpired = function(dir, index) {
|
||||
if ((options != null ? options.keep : undefined) === dir) { return false; }
|
||||
// remove any directories over the requested (non-null) limit
|
||||
if (((options != null ? options.limit : undefined) != null) && (index > options.limit)) { return true; }
|
||||
// remove any directories over the hard limit
|
||||
if (index > OutputCacheManager.CACHE_LIMIT) { return true; }
|
||||
// we can get the build time from the first part of the directory name DDDD-RRRR
|
||||
// DDDD is date and RRRR is random bytes
|
||||
const dirTime = parseInt(__guard__(dir.split('-'), x => x[0]), 16);
|
||||
const age = currentTime - dirTime;
|
||||
return age > OutputCacheManager.CACHE_AGE;
|
||||
};
|
||||
|
||||
const toRemove = _.filter(dirs, isExpired);
|
||||
|
||||
const removeDir = (dir, cb) =>
|
||||
fse.remove(Path.join(cacheRoot, dir), function(err, result) {
|
||||
logger.log({cache: cacheRoot, dir}, "removed expired cache dir");
|
||||
if (err != null) {
|
||||
logger.error({err, dir}, "cache remove error");
|
||||
}
|
||||
return cb(err, result);
|
||||
})
|
||||
;
|
||||
|
||||
return async.eachSeries(toRemove, (dir, cb) => removeDir(dir, cb)
|
||||
, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_fileIsHidden(path) {
|
||||
return ((path != null ? path.match(/^\.|\/\./) : undefined) != null);
|
||||
},
|
||||
|
||||
_checkFileIsSafe(src, callback) {
|
||||
// check if we have a valid file to copy into the cache
|
||||
if (callback == null) { callback = function(error, isSafe) {}; }
|
||||
return fs.stat(src, function(err, stats) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
logger.warn({err, file: src}, "file has disappeared before copying to build cache");
|
||||
return callback(err, false);
|
||||
} else if (err != null) {
|
||||
// some other problem reading the file
|
||||
logger.error({err, file: src}, "stat error for file in cache");
|
||||
return callback(err, false);
|
||||
} else if (!stats.isFile()) {
|
||||
// other filetype - reject it
|
||||
logger.warn({src, stat: stats}, "nonfile output - refusing to copy to cache");
|
||||
return callback(null, false);
|
||||
} else {
|
||||
// it's a plain file, ok to copy
|
||||
return callback(null, true);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_copyFile(src, dst, callback) {
|
||||
// copy output file into the cache
|
||||
return fse.copy(src, dst, function(err) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
logger.warn({err, file: src}, "file has disappeared when copying to build cache");
|
||||
return callback(err, false);
|
||||
} else if (err != null) {
|
||||
logger.error({err, src, dst}, "copy error for file in cache");
|
||||
return callback(err);
|
||||
} else {
|
||||
if ((Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined)) {
|
||||
// don't run any optimisations on the pdf when they are done
|
||||
// in the docker container
|
||||
return callback();
|
||||
} else {
|
||||
// call the optimiser for the file too
|
||||
return OutputFileOptimiser.optimiseFile(src, dst, callback);
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_checkIfShouldCopy(src, callback) {
|
||||
if (callback == null) { callback = function(err, shouldCopy) {}; }
|
||||
return callback(null, !Path.basename(src).match(/^strace/));
|
||||
},
|
||||
|
||||
_checkIfShouldArchive(src, callback) {
|
||||
let needle;
|
||||
if (callback == null) { callback = function(err, shouldCopy) {}; }
|
||||
if (Path.basename(src).match(/^strace/)) {
|
||||
return callback(null, true);
|
||||
}
|
||||
if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) && (needle = Path.basename(src), ["output.log", "output.blg"].includes(needle))) {
|
||||
return callback(null, true);
|
||||
}
|
||||
return callback(null, false);
|
||||
}
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
86
app/js/OutputFileFinder.js
Normal file
86
app/js/OutputFileFinder.js
Normal file
@@ -0,0 +1,86 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
no-useless-escape,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputFileFinder;
|
||||
const async = require("async");
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
const { spawn } = require("child_process");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
module.exports = (OutputFileFinder = {
|
||||
findOutputFiles(resources, directory, callback) {
|
||||
if (callback == null) { callback = function(error, outputFiles, allFiles) {}; }
|
||||
const incomingResources = {};
|
||||
for (const resource of Array.from(resources)) {
|
||||
incomingResources[resource.path] = true;
|
||||
}
|
||||
|
||||
return OutputFileFinder._getAllFiles(directory, function(error, allFiles) {
|
||||
if (allFiles == null) { allFiles = []; }
|
||||
if (error != null) {
|
||||
logger.err({err:error}, "error finding all output files");
|
||||
return callback(error);
|
||||
}
|
||||
const outputFiles = [];
|
||||
for (const file of Array.from(allFiles)) {
|
||||
if (!incomingResources[file]) {
|
||||
outputFiles.push({
|
||||
path: file,
|
||||
type: __guard__(file.match(/\.([^\.]+)$/), x => x[1])
|
||||
});
|
||||
}
|
||||
}
|
||||
return callback(null, outputFiles, allFiles);
|
||||
});
|
||||
},
|
||||
|
||||
_getAllFiles(directory, _callback) {
|
||||
if (_callback == null) { _callback = function(error, fileList) {}; }
|
||||
const callback = function(error, fileList) {
|
||||
_callback(error, fileList);
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
// don't include clsi-specific files/directories in the output list
|
||||
const EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"];
|
||||
const args = [directory, "(", ...Array.from(EXCLUDE_DIRS), ")", "-prune", "-o", "-type", "f", "-print"];
|
||||
logger.log({args}, "running find command");
|
||||
|
||||
const proc = spawn("find", args);
|
||||
let stdout = "";
|
||||
proc.stdout.on("data", chunk => stdout += chunk.toString());
|
||||
proc.on("error", callback);
|
||||
return proc.on("close", function(code) {
|
||||
if (code !== 0) {
|
||||
logger.warn({directory, code}, "find returned error, directory likely doesn't exist");
|
||||
return callback(null, []);
|
||||
}
|
||||
let fileList = stdout.trim().split("\n");
|
||||
fileList = fileList.map(function(file) {
|
||||
// Strip leading directory
|
||||
let path;
|
||||
return path = Path.relative(directory, file);
|
||||
});
|
||||
return callback(null, fileList);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
86
app/js/OutputFileOptimiser.js
Normal file
86
app/js/OutputFileOptimiser.js
Normal file
@@ -0,0 +1,86 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-undef,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputFileOptimiser;
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
const { spawn } = require("child_process");
|
||||
const logger = require("logger-sharelatex");
|
||||
const Metrics = require("./Metrics");
|
||||
const _ = require("underscore");
|
||||
|
||||
module.exports = (OutputFileOptimiser = {
|
||||
|
||||
optimiseFile(src, dst, callback) {
|
||||
// check output file (src) and see if we can optimise it, storing
|
||||
// the result in the build directory (dst)
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
if (src.match(/\/output\.pdf$/)) {
|
||||
return OutputFileOptimiser.checkIfPDFIsOptimised(src, function(err, isOptimised) {
|
||||
if ((err != null) || isOptimised) { return callback(null); }
|
||||
return OutputFileOptimiser.optimisePDF(src, dst, callback);
|
||||
});
|
||||
} else {
|
||||
return callback((null));
|
||||
}
|
||||
},
|
||||
|
||||
checkIfPDFIsOptimised(file, callback) {
|
||||
const SIZE = 16*1024; // check the header of the pdf
|
||||
const result = new Buffer(SIZE);
|
||||
result.fill(0); // prevent leakage of uninitialised buffer
|
||||
return fs.open(file, "r", function(err, fd) {
|
||||
if (err != null) { return callback(err); }
|
||||
return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) =>
|
||||
fs.close(fd, function(errClose) {
|
||||
if (errRead != null) { return callback(errRead); }
|
||||
if (typeof errReadClose !== 'undefined' && errReadClose !== null) { return callback(errClose); }
|
||||
const isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0;
|
||||
return callback(null, isOptimised);
|
||||
})
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
optimisePDF(src, dst, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const tmpOutput = dst + '.opt';
|
||||
const args = ["--linearize", src, tmpOutput];
|
||||
logger.log({args}, "running qpdf command");
|
||||
|
||||
const timer = new Metrics.Timer("qpdf");
|
||||
const proc = spawn("qpdf", args);
|
||||
let stdout = "";
|
||||
proc.stdout.on("data", chunk => stdout += chunk.toString());
|
||||
callback = _.once(callback); // avoid double call back for error and close event
|
||||
proc.on("error", function(err) {
|
||||
logger.warn({err, args}, "qpdf failed");
|
||||
return callback(null);
|
||||
}); // ignore the error
|
||||
return proc.on("close", function(code) {
|
||||
timer.done();
|
||||
if (code !== 0) {
|
||||
logger.warn({code, args}, "qpdf returned error");
|
||||
return callback(null); // ignore the error
|
||||
}
|
||||
return fs.rename(tmpOutput, dst, function(err) {
|
||||
if (err != null) {
|
||||
logger.warn({tmpOutput, dst}, "failed to rename output of qpdf command");
|
||||
}
|
||||
return callback(null);
|
||||
});
|
||||
});
|
||||
} // ignore the error
|
||||
});
|
||||
123
app/js/ProjectPersistenceManager.js
Normal file
123
app/js/ProjectPersistenceManager.js
Normal file
@@ -0,0 +1,123 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ProjectPersistenceManager;
|
||||
const UrlCache = require("./UrlCache");
|
||||
const CompileManager = require("./CompileManager");
|
||||
const db = require("./db");
|
||||
const dbQueue = require("./DbQueue");
|
||||
const async = require("async");
|
||||
const logger = require("logger-sharelatex");
|
||||
const oneDay = 24 * 60 * 60 * 1000;
|
||||
const Settings = require("settings-sharelatex");
|
||||
|
||||
module.exports = (ProjectPersistenceManager = {
|
||||
|
||||
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || (oneDay * 2.5),
|
||||
|
||||
markProjectAsJustAccessed(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const job = cb=>
|
||||
db.Project.findOrCreate({where: {project_id}})
|
||||
.spread(
|
||||
(project, created) =>
|
||||
project.updateAttributes({lastAccessed: new Date()})
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
)
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
|
||||
clearExpiredProjects(callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return ProjectPersistenceManager._findExpiredProjectIds(function(error, project_ids) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({project_ids}, "clearing expired projects");
|
||||
const jobs = (Array.from(project_ids || [])).map((project_id) =>
|
||||
(project_id =>
|
||||
callback =>
|
||||
ProjectPersistenceManager.clearProjectFromCache(project_id, function(err) {
|
||||
if (err != null) {
|
||||
logger.error({err, project_id}, "error clearing project");
|
||||
}
|
||||
return callback();
|
||||
})
|
||||
|
||||
)(project_id));
|
||||
return async.series(jobs, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return CompileManager.clearExpiredProjects(ProjectPersistenceManager.EXPIRY_TIMEOUT, error => callback());
|
||||
});
|
||||
});
|
||||
}, // ignore any errors from deleting directories
|
||||
|
||||
clearProject(project_id, user_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({project_id, user_id}, "clearing project for user");
|
||||
return CompileManager.clearProject(project_id, user_id, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ProjectPersistenceManager.clearProjectFromCache(project_id, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback();
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
clearProjectFromCache(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({project_id}, "clearing project from cache");
|
||||
return UrlCache.clearProject(project_id, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id}, "error clearing project from cache");
|
||||
return callback(error);
|
||||
}
|
||||
return ProjectPersistenceManager._clearProjectFromDatabase(project_id, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({error, project_id}, "error clearing project from database");
|
||||
}
|
||||
return callback(error);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_clearProjectFromDatabase(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
logger.log({project_id}, "clearing project from database");
|
||||
const job = cb=>
|
||||
db.Project.destroy({where: {project_id}})
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
|
||||
_findExpiredProjectIds(callback) {
|
||||
if (callback == null) { callback = function(error, project_ids) {}; }
|
||||
const job = function(cb){
|
||||
const keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT);
|
||||
const q = {};
|
||||
q[db.op.lt] = keepProjectsFrom;
|
||||
return db.Project.findAll({where:{lastAccessed:q}})
|
||||
.then(projects => cb(null, projects.map(project => project.project_id))).error(cb);
|
||||
};
|
||||
|
||||
return dbQueue.queue.push(job, callback);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
logger.log({EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout");
|
||||
193
app/js/RequestParser.js
Normal file
193
app/js/RequestParser.js
Normal file
@@ -0,0 +1,193 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-control-regex,
|
||||
no-throw-literal,
|
||||
no-unused-vars,
|
||||
no-useless-escape,
|
||||
standard/no-callback-literal,
|
||||
valid-typeof,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let RequestParser;
|
||||
const settings = require("settings-sharelatex");
|
||||
|
||||
module.exports = (RequestParser = {
|
||||
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"],
|
||||
MAX_TIMEOUT: 600,
|
||||
|
||||
parse(body, callback) {
|
||||
let resource;
|
||||
if (callback == null) { callback = function(error, data) {}; }
|
||||
const response = {};
|
||||
|
||||
if ((body.compile == null)) {
|
||||
return callback("top level object should have a compile attribute");
|
||||
}
|
||||
|
||||
const { compile } = body;
|
||||
if (!compile.options) { compile.options = {}; }
|
||||
|
||||
try {
|
||||
response.compiler = this._parseAttribute("compiler",
|
||||
compile.options.compiler, {
|
||||
validValues: this.VALID_COMPILERS,
|
||||
default: "pdflatex",
|
||||
type: "string"
|
||||
}
|
||||
);
|
||||
response.timeout = this._parseAttribute("timeout",
|
||||
compile.options.timeout, {
|
||||
default: RequestParser.MAX_TIMEOUT,
|
||||
type: "number"
|
||||
}
|
||||
);
|
||||
response.imageName = this._parseAttribute("imageName",
|
||||
compile.options.imageName,
|
||||
{type: "string"});
|
||||
response.draft = this._parseAttribute("draft",
|
||||
compile.options.draft, {
|
||||
default: false,
|
||||
type: "boolean"
|
||||
}
|
||||
);
|
||||
response.check = this._parseAttribute("check",
|
||||
compile.options.check,
|
||||
{type: "string"});
|
||||
response.flags = this._parseAttribute("flags",
|
||||
compile.options.flags, {
|
||||
default: [],
|
||||
type: "object"
|
||||
}
|
||||
);
|
||||
|
||||
// The syncType specifies whether the request contains all
|
||||
// resources (full) or only those resources to be updated
|
||||
// in-place (incremental).
|
||||
response.syncType = this._parseAttribute("syncType",
|
||||
compile.options.syncType, {
|
||||
validValues: ["full", "incremental"],
|
||||
type: "string"
|
||||
}
|
||||
);
|
||||
|
||||
// The syncState is an identifier passed in with the request
|
||||
// which has the property that it changes when any resource is
|
||||
// added, deleted, moved or renamed.
|
||||
//
|
||||
// on syncType full the syncState identifier is passed in and
|
||||
// stored
|
||||
//
|
||||
// on syncType incremental the syncState identifier must match
|
||||
// the stored value
|
||||
response.syncState = this._parseAttribute("syncState",
|
||||
compile.options.syncState,
|
||||
{type: "string"});
|
||||
|
||||
if (response.timeout > RequestParser.MAX_TIMEOUT) {
|
||||
response.timeout = RequestParser.MAX_TIMEOUT;
|
||||
}
|
||||
response.timeout = response.timeout * 1000; // milliseconds
|
||||
|
||||
response.resources = ((() => {
|
||||
const result = [];
|
||||
for (resource of Array.from((compile.resources || []))) { result.push(this._parseResource(resource));
|
||||
}
|
||||
return result;
|
||||
})());
|
||||
|
||||
const rootResourcePath = this._parseAttribute("rootResourcePath",
|
||||
compile.rootResourcePath, {
|
||||
default: "main.tex",
|
||||
type: "string"
|
||||
}
|
||||
);
|
||||
const originalRootResourcePath = rootResourcePath;
|
||||
const sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath);
|
||||
response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath);
|
||||
|
||||
for (resource of Array.from(response.resources)) {
|
||||
if (resource.path === originalRootResourcePath) {
|
||||
resource.path = sanitizedRootResourcePath;
|
||||
}
|
||||
}
|
||||
} catch (error1) {
|
||||
const error = error1;
|
||||
return callback(error);
|
||||
}
|
||||
|
||||
return callback(null, response);
|
||||
},
|
||||
|
||||
_parseResource(resource) {
|
||||
let modified;
|
||||
if ((resource.path == null) || (typeof resource.path !== "string")) {
|
||||
throw "all resources should have a path attribute";
|
||||
}
|
||||
|
||||
if (resource.modified != null) {
|
||||
modified = new Date(resource.modified);
|
||||
if (isNaN(modified.getTime())) {
|
||||
throw `resource modified date could not be understood: ${resource.modified}`;
|
||||
}
|
||||
}
|
||||
|
||||
if ((resource.url == null) && (resource.content == null)) {
|
||||
throw "all resources should have either a url or content attribute";
|
||||
}
|
||||
if ((resource.content != null) && (typeof resource.content !== "string")) {
|
||||
throw "content attribute should be a string";
|
||||
}
|
||||
if ((resource.url != null) && (typeof resource.url !== "string")) {
|
||||
throw "url attribute should be a string";
|
||||
}
|
||||
|
||||
return {
|
||||
path: resource.path,
|
||||
modified,
|
||||
url: resource.url,
|
||||
content: resource.content
|
||||
};
|
||||
},
|
||||
|
||||
_parseAttribute(name, attribute, options) {
|
||||
if (attribute != null) {
|
||||
if (options.validValues != null) {
|
||||
if (options.validValues.indexOf(attribute) === -1) {
|
||||
throw `${name} attribute should be one of: ${options.validValues.join(", ")}`;
|
||||
}
|
||||
}
|
||||
if (options.type != null) {
|
||||
if (typeof attribute !== options.type) {
|
||||
throw `${name} attribute should be a ${options.type}`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (options.default != null) { return options.default; }
|
||||
}
|
||||
return attribute;
|
||||
},
|
||||
|
||||
_sanitizePath(path) {
|
||||
// See http://php.net/manual/en/function.escapeshellcmd.php
|
||||
return path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "");
|
||||
},
|
||||
|
||||
_checkPath(path) {
|
||||
// check that the request does not use a relative path
|
||||
for (const dir of Array.from(path.split('/'))) {
|
||||
if (dir === '..') {
|
||||
throw "relative path in root resource";
|
||||
}
|
||||
}
|
||||
return path;
|
||||
}
|
||||
});
|
||||
114
app/js/ResourceStateManager.js
Normal file
114
app/js/ResourceStateManager.js
Normal file
@@ -0,0 +1,114 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS201: Simplify complex destructure assignments
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ResourceStateManager;
|
||||
const Path = require("path");
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
const settings = require("settings-sharelatex");
|
||||
const Errors = require("./Errors");
|
||||
const SafeReader = require("./SafeReader");
|
||||
|
||||
module.exports = (ResourceStateManager = {
|
||||
|
||||
// The sync state is an identifier which must match for an
|
||||
// incremental update to be allowed.
|
||||
//
|
||||
// The initial value is passed in and stored on a full
|
||||
// compile, along with the list of resources..
|
||||
//
|
||||
// Subsequent incremental compiles must come with the same value - if
|
||||
// not they will be rejected with a 409 Conflict response. The
|
||||
// previous list of resources is returned.
|
||||
//
|
||||
// An incremental compile can only update existing files with new
|
||||
// content. The sync state identifier must change if any docs or
|
||||
// files are moved, added, deleted or renamed.
|
||||
|
||||
SYNC_STATE_FILE: ".project-sync-state",
|
||||
SYNC_STATE_MAX_SIZE: 128*1024,
|
||||
|
||||
saveProjectState(state, resources, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE);
|
||||
if ((state == null)) { // remove the file if no state passed in
|
||||
logger.log({state, basePath}, "clearing sync state");
|
||||
return fs.unlink(stateFile, function(err) {
|
||||
if ((err != null) && (err.code !== 'ENOENT')) {
|
||||
return callback(err);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
logger.log({state, basePath}, "writing sync state");
|
||||
const resourceList = (Array.from(resources).map((resource) => resource.path));
|
||||
return fs.writeFile(stateFile, [...Array.from(resourceList), `stateHash:${state}`].join("\n"), callback);
|
||||
}
|
||||
},
|
||||
|
||||
checkProjectStateMatches(state, basePath, callback) {
|
||||
if (callback == null) { callback = function(error, resources) {}; }
|
||||
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE);
|
||||
const size = this.SYNC_STATE_MAX_SIZE;
|
||||
return SafeReader.readFile(stateFile, size, 'utf8', function(err, result, bytesRead) {
|
||||
if (err != null) { return callback(err); }
|
||||
if (bytesRead === size) {
|
||||
logger.error({file:stateFile, size, bytesRead}, "project state file truncated");
|
||||
}
|
||||
const array = __guard__(result != null ? result.toString() : undefined, x => x.split("\n")) || [];
|
||||
const adjustedLength = Math.max(array.length, 1);
|
||||
const resourceList = array.slice(0, adjustedLength - 1);
|
||||
const oldState = array[adjustedLength - 1];
|
||||
const newState = `stateHash:${state}`;
|
||||
logger.log({state, oldState, basePath, stateMatches: (newState === oldState)}, "checking sync state");
|
||||
if (newState !== oldState) {
|
||||
return callback(new Errors.FilesOutOfSyncError("invalid state for incremental update"));
|
||||
} else {
|
||||
const resources = (Array.from(resourceList).map((path) => ({path})));
|
||||
return callback(null, resources);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
checkResourceFiles(resources, allFiles, basePath, callback) {
|
||||
// check the paths are all relative to current directory
|
||||
let file;
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
for (file of Array.from(resources || [])) {
|
||||
for (const dir of Array.from(__guard__(file != null ? file.path : undefined, x => x.split('/')))) {
|
||||
if (dir === '..') {
|
||||
return callback(new Error("relative path in resource file list"));
|
||||
}
|
||||
}
|
||||
}
|
||||
// check if any of the input files are not present in list of files
|
||||
const seenFile = {};
|
||||
for (file of Array.from(allFiles)) {
|
||||
seenFile[file] = true;
|
||||
}
|
||||
const missingFiles = (Array.from(resources).filter((resource) => !seenFile[resource.path]).map((resource) => resource.path));
|
||||
if ((missingFiles != null ? missingFiles.length : undefined) > 0) {
|
||||
logger.err({missingFiles, basePath, allFiles, resources}, "missing input files for project");
|
||||
return callback(new Errors.FilesOutOfSyncError("resource files missing in incremental update"));
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
215
app/js/ResourceWriter.js
Normal file
215
app/js/ResourceWriter.js
Normal file
@@ -0,0 +1,215 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
no-useless-escape,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ResourceWriter;
|
||||
const UrlCache = require("./UrlCache");
|
||||
const Path = require("path");
|
||||
const fs = require("fs");
|
||||
const async = require("async");
|
||||
const mkdirp = require("mkdirp");
|
||||
const OutputFileFinder = require("./OutputFileFinder");
|
||||
const ResourceStateManager = require("./ResourceStateManager");
|
||||
const Metrics = require("./Metrics");
|
||||
const logger = require("logger-sharelatex");
|
||||
const settings = require("settings-sharelatex");
|
||||
|
||||
const parallelFileDownloads = settings.parallelFileDownloads || 1;
|
||||
|
||||
module.exports = (ResourceWriter = {
|
||||
|
||||
syncResourcesToDisk(request, basePath, callback) {
|
||||
if (callback == null) { callback = function(error, resourceList) {}; }
|
||||
if (request.syncType === "incremental") {
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id}, "incremental sync");
|
||||
return ResourceStateManager.checkProjectStateMatches(request.syncState, basePath, function(error, resourceList) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceWriter._removeExtraneousFiles(resourceList, basePath, function(error, outputFiles, allFiles) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceStateManager.checkResourceFiles(resourceList, allFiles, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceWriter.saveIncrementalResourcesToDisk(request.project_id, request.resources, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, resourceList);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.log({project_id: request.project_id, user_id: request.user_id}, "full sync");
|
||||
return this.saveAllResourcesToDisk(request.project_id, request.resources, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return ResourceStateManager.saveProjectState(request.syncState, request.resources, basePath, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, request.resources);
|
||||
});
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return this._createDirectory(basePath, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = Array.from(resources).map((resource) =>
|
||||
(resource => {
|
||||
return callback => this._writeResourceToDisk(project_id, resource, basePath, callback);
|
||||
})(resource));
|
||||
return async.parallelLimit(jobs, parallelFileDownloads, callback);
|
||||
});
|
||||
},
|
||||
|
||||
saveAllResourcesToDisk(project_id, resources, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return this._createDirectory(basePath, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
return this._removeExtraneousFiles(resources, basePath, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = Array.from(resources).map((resource) =>
|
||||
(resource => {
|
||||
return callback => this._writeResourceToDisk(project_id, resource, basePath, callback);
|
||||
})(resource));
|
||||
return async.parallelLimit(jobs, parallelFileDownloads, callback);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_createDirectory(basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.mkdir(basePath, function(err) {
|
||||
if (err != null) {
|
||||
if (err.code === 'EEXIST') {
|
||||
return callback();
|
||||
} else {
|
||||
logger.log({err, dir:basePath}, "error creating directory");
|
||||
return callback(err);
|
||||
}
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_removeExtraneousFiles(resources, basePath, _callback) {
|
||||
if (_callback == null) { _callback = function(error, outputFiles, allFiles) {}; }
|
||||
const timer = new Metrics.Timer("unlink-output-files");
|
||||
const callback = function(error, ...result) {
|
||||
timer.done();
|
||||
return _callback(error, ...Array.from(result));
|
||||
};
|
||||
|
||||
return OutputFileFinder.findOutputFiles(resources, basePath, function(error, outputFiles, allFiles) {
|
||||
if (error != null) { return callback(error); }
|
||||
|
||||
const jobs = [];
|
||||
for (const file of Array.from(outputFiles || [])) {
|
||||
(function(file) {
|
||||
const { path } = file;
|
||||
let should_delete = true;
|
||||
if (path.match(/^output\./) || path.match(/\.aux$/) || path.match(/^cache\//)) { // knitr cache
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/^output-.*/)) { // Tikz cached figures (default case)
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/\.(pdf|dpth|md5)$/)) { // Tikz cached figures (by extension)
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/\.(pygtex|pygstyle)$/) || path.match(/(^|\/)_minted-[^\/]+\//)) { // minted files/directory
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) { // markdown files/directory
|
||||
should_delete = false;
|
||||
}
|
||||
if (path.match(/-eps-converted-to\.pdf$/)) { // Epstopdf generated files
|
||||
should_delete = false;
|
||||
}
|
||||
if ((path === "output.pdf") || (path === "output.dvi") || (path === "output.log") || (path === "output.xdv")) {
|
||||
should_delete = true;
|
||||
}
|
||||
if (path === "output.tex") { // created by TikzManager if present in output files
|
||||
should_delete = true;
|
||||
}
|
||||
if (should_delete) {
|
||||
return jobs.push(callback => ResourceWriter._deleteFileIfNotDirectory(Path.join(basePath, path), callback));
|
||||
}
|
||||
})(file);
|
||||
}
|
||||
|
||||
return async.series(jobs, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, outputFiles, allFiles);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_deleteFileIfNotDirectory(path, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.stat(path, function(error, stat) {
|
||||
if ((error != null) && (error.code === 'ENOENT')) {
|
||||
return callback();
|
||||
} else if (error != null) {
|
||||
logger.err({err: error, path}, "error stating file in deleteFileIfNotDirectory");
|
||||
return callback(error);
|
||||
} else if (stat.isFile()) {
|
||||
return fs.unlink(path, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({err: error, path}, "error removing file in deleteFileIfNotDirectory");
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_writeResourceToDisk(project_id, resource, basePath, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return ResourceWriter.checkPath(basePath, resource.path, function(error, path) {
|
||||
if (error != null) { return callback(error); }
|
||||
return mkdirp(Path.dirname(path), function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
// TODO: Don't overwrite file if it hasn't been modified
|
||||
if (resource.url != null) {
|
||||
return UrlCache.downloadUrlToFile(project_id, resource.url, path, resource.modified, function(err){
|
||||
if (err != null) {
|
||||
logger.err({err, project_id, path, resource_url:resource.url, modified:resource.modified}, "error downloading file for resources");
|
||||
}
|
||||
return callback();
|
||||
}); // try and continue compiling even if http resource can not be downloaded at this time
|
||||
} else {
|
||||
const process = require("process");
|
||||
fs.writeFile(path, resource.content, callback);
|
||||
try {
|
||||
let result;
|
||||
return result = fs.lstatSync(path);
|
||||
} catch (e) {}
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
checkPath(basePath, resourcePath, callback) {
|
||||
const path = Path.normalize(Path.join(basePath, resourcePath));
|
||||
if (path.slice(0, basePath.length + 1) !== (basePath + "/")) {
|
||||
return callback(new Error("resource path is outside root directory"));
|
||||
} else {
|
||||
return callback(null, path);
|
||||
}
|
||||
}
|
||||
});
|
||||
47
app/js/SafeReader.js
Normal file
47
app/js/SafeReader.js
Normal file
@@ -0,0 +1,47 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let SafeReader;
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
module.exports = (SafeReader = {
|
||||
|
||||
// safely read up to size bytes from a file and return result as a
|
||||
// string
|
||||
|
||||
readFile(file, size, encoding, callback) {
|
||||
if (callback == null) { callback = function(error, result) {}; }
|
||||
return fs.open(file, 'r', function(err, fd) {
|
||||
if ((err != null) && (err.code === 'ENOENT')) { return callback(); }
|
||||
if (err != null) { return callback(err); }
|
||||
|
||||
// safely return always closing the file
|
||||
const callbackWithClose = (err, ...result) =>
|
||||
fs.close(fd, function(err1) {
|
||||
if (err != null) { return callback(err); }
|
||||
if (err1 != null) { return callback(err1); }
|
||||
return callback(null, ...Array.from(result));
|
||||
})
|
||||
;
|
||||
|
||||
const buff = new Buffer(size, 0); // fill with zeros
|
||||
return fs.read(fd, buff, 0, buff.length, 0, function(err, bytesRead, buffer) {
|
||||
if (err != null) { return callbackWithClose(err); }
|
||||
const result = buffer.toString(encoding, 0, bytesRead);
|
||||
return callbackWithClose(null, result, bytesRead);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
72
app/js/StaticServerForbidSymlinks.js
Normal file
72
app/js/StaticServerForbidSymlinks.js
Normal file
@@ -0,0 +1,72 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
no-cond-assign,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ForbidSymlinks;
|
||||
const Path = require("path");
|
||||
const fs = require("fs");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const logger = require("logger-sharelatex");
|
||||
const url = require("url");
|
||||
|
||||
module.exports = (ForbidSymlinks = function(staticFn, root, options) {
|
||||
const expressStatic = staticFn(root, options);
|
||||
const basePath = Path.resolve(root);
|
||||
return function(req, res, next) {
|
||||
let file, project_id, result;
|
||||
const path = __guard__(url.parse(req.url), x => x.pathname);
|
||||
// check that the path is of the form /project_id_or_name/path/to/file.log
|
||||
if (result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/)) {
|
||||
project_id = result[1];
|
||||
file = result[2];
|
||||
} else {
|
||||
logger.warn({path}, "unrecognized file request");
|
||||
return res.sendStatus(404);
|
||||
}
|
||||
// check that the file does not use a relative path
|
||||
for (const dir of Array.from(file.split('/'))) {
|
||||
if (dir === '..') {
|
||||
logger.warn({path}, "attempt to use a relative path");
|
||||
return res.sendStatus(404);
|
||||
}
|
||||
}
|
||||
// check that the requested path is normalized
|
||||
const requestedFsPath = `${basePath}/${project_id}/${file}`;
|
||||
if (requestedFsPath !== Path.normalize(requestedFsPath)) {
|
||||
logger.error({path: requestedFsPath}, "requestedFsPath is not normalized");
|
||||
return res.sendStatus(404);
|
||||
}
|
||||
// check that the requested path is not a symlink
|
||||
return fs.realpath(requestedFsPath, function(err, realFsPath){
|
||||
if (err != null) {
|
||||
if (err.code === 'ENOENT') {
|
||||
return res.sendStatus(404);
|
||||
} else {
|
||||
logger.error({err, requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "error checking file access");
|
||||
return res.sendStatus(500);
|
||||
}
|
||||
} else if (requestedFsPath !== realFsPath) {
|
||||
logger.warn({requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "trying to access a different file (symlink), aborting");
|
||||
return res.sendStatus(404);
|
||||
} else {
|
||||
return expressStatic(req, res, next);
|
||||
}
|
||||
});
|
||||
};
|
||||
});
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
|
||||
}
|
||||
62
app/js/TikzManager.js
Normal file
62
app/js/TikzManager.js
Normal file
@@ -0,0 +1,62 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let TikzManager;
|
||||
const fs = require("fs");
|
||||
const Path = require("path");
|
||||
const ResourceWriter = require("./ResourceWriter");
|
||||
const SafeReader = require("./SafeReader");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
// for \tikzexternalize or pstool to work the main file needs to match the
|
||||
// jobname. Since we set the -jobname to output, we have to create a
|
||||
// copy of the main file as 'output.tex'.
|
||||
|
||||
module.exports = (TikzManager = {
|
||||
|
||||
checkMainFile(compileDir, mainFile, resources, callback) {
|
||||
// if there's already an output.tex file, we don't want to touch it
|
||||
if (callback == null) { callback = function(error, needsMainFile) {}; }
|
||||
for (const resource of Array.from(resources)) {
|
||||
if (resource.path === "output.tex") {
|
||||
logger.log({compileDir, mainFile}, "output.tex already in resources");
|
||||
return callback(null, false);
|
||||
}
|
||||
}
|
||||
// if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
|
||||
return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) {
|
||||
if (error != null) { return callback(error); }
|
||||
return SafeReader.readFile(path, 65536, "utf8", function(error, content) {
|
||||
if (error != null) { return callback(error); }
|
||||
const usesTikzExternalize = (content != null ? content.indexOf("\\tikzexternalize") : undefined) >= 0;
|
||||
const usesPsTool = (content != null ? content.indexOf("{pstool}") : undefined) >= 0;
|
||||
logger.log({compileDir, mainFile, usesTikzExternalize, usesPsTool}, "checked for packages needing main file as output.tex");
|
||||
const needsMainFile = (usesTikzExternalize || usesPsTool);
|
||||
return callback(null, needsMainFile);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
injectOutputFile(compileDir, mainFile, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) {
|
||||
if (error != null) { return callback(error); }
|
||||
return fs.readFile(path, "utf8", function(error, content) {
|
||||
if (error != null) { return callback(error); }
|
||||
logger.log({compileDir, mainFile}, "copied file to output.tex as project uses packages which require it");
|
||||
// use wx flag to ensure that output file does not already exist
|
||||
return fs.writeFile(Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
196
app/js/UrlCache.js
Normal file
196
app/js/UrlCache.js
Normal file
@@ -0,0 +1,196 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlCache;
|
||||
const db = require("./db");
|
||||
const dbQueue = require("./DbQueue");
|
||||
const UrlFetcher = require("./UrlFetcher");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const crypto = require("crypto");
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
const async = require("async");
|
||||
|
||||
module.exports = (UrlCache = {
|
||||
downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return UrlCache._ensureUrlIsInCache(project_id, url, lastModified, (error, pathToCachedUrl) => {
|
||||
if (error != null) { return callback(error); }
|
||||
return UrlCache._copyFile(pathToCachedUrl, destPath, function(error) {
|
||||
if (error != null) {
|
||||
return UrlCache._clearUrlDetails(project_id, url, () => callback(error));
|
||||
} else {
|
||||
return callback(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
clearProject(project_id, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return UrlCache._findAllUrlsInProject(project_id, function(error, urls) {
|
||||
logger.log({project_id, url_count: urls.length}, "clearing project URLs");
|
||||
if (error != null) { return callback(error); }
|
||||
const jobs = (Array.from(urls || [])).map((url) =>
|
||||
(url =>
|
||||
callback =>
|
||||
UrlCache._clearUrlFromCache(project_id, url, function(error) {
|
||||
if (error != null) {
|
||||
logger.error({err: error, project_id, url}, "error clearing project URL");
|
||||
}
|
||||
return callback();
|
||||
})
|
||||
|
||||
)(url));
|
||||
return async.series(jobs, callback);
|
||||
});
|
||||
},
|
||||
|
||||
_ensureUrlIsInCache(project_id, url, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error, pathOnDisk) {}; }
|
||||
if (lastModified != null) {
|
||||
// MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
|
||||
// So round down to seconds
|
||||
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000);
|
||||
}
|
||||
return UrlCache._doesUrlNeedDownloading(project_id, url, lastModified, (error, needsDownloading) => {
|
||||
if (error != null) { return callback(error); }
|
||||
if (needsDownloading) {
|
||||
logger.log({url, lastModified}, "downloading URL");
|
||||
return UrlFetcher.pipeUrlToFile(url, UrlCache._cacheFilePathForUrl(project_id, url), error => {
|
||||
if (error != null) { return callback(error); }
|
||||
return UrlCache._updateOrCreateUrlDetails(project_id, url, lastModified, error => {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url));
|
||||
});
|
||||
});
|
||||
} else {
|
||||
logger.log({url, lastModified}, "URL is up to date in cache");
|
||||
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_doesUrlNeedDownloading(project_id, url, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error, needsDownloading) {}; }
|
||||
if ((lastModified == null)) {
|
||||
return callback(null, true);
|
||||
}
|
||||
return UrlCache._findUrlDetails(project_id, url, function(error, urlDetails) {
|
||||
if (error != null) { return callback(error); }
|
||||
if ((urlDetails == null) || (urlDetails.lastModified == null) || (urlDetails.lastModified.getTime() < lastModified.getTime())) {
|
||||
return callback(null, true);
|
||||
} else {
|
||||
return callback(null, false);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_cacheFileNameForUrl(project_id, url) {
|
||||
return project_id + ":" + crypto.createHash("md5").update(url).digest("hex");
|
||||
},
|
||||
|
||||
_cacheFilePathForUrl(project_id, url) {
|
||||
return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(project_id, url)}`;
|
||||
},
|
||||
|
||||
_copyFile(from, to, _callback) {
|
||||
if (_callback == null) { _callback = function(error) {}; }
|
||||
const callbackOnce = function(error) {
|
||||
if (error != null) {
|
||||
logger.error({err: error, from, to}, "error copying file from cache");
|
||||
}
|
||||
_callback(error);
|
||||
return _callback = function() {};
|
||||
};
|
||||
const writeStream = fs.createWriteStream(to);
|
||||
const readStream = fs.createReadStream(from);
|
||||
writeStream.on("error", callbackOnce);
|
||||
readStream.on("error", callbackOnce);
|
||||
writeStream.on("close", callbackOnce);
|
||||
return writeStream.on("open", () => readStream.pipe(writeStream));
|
||||
},
|
||||
|
||||
_clearUrlFromCache(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return UrlCache._clearUrlDetails(project_id, url, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return UrlCache._deleteUrlCacheFromDisk(project_id, url, function(error) {
|
||||
if (error != null) { return callback(error); }
|
||||
return callback(null);
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
_deleteUrlCacheFromDisk(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function(error) {
|
||||
if ((error != null) && (error.code !== 'ENOENT')) { // no error if the file isn't present
|
||||
return callback(error);
|
||||
} else {
|
||||
return callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
_findUrlDetails(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error, urlDetails) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.find({where: { url, project_id }})
|
||||
.then(urlDetails => cb(null, urlDetails))
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
_updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.findOrCreate({where: {url, project_id}})
|
||||
.spread(
|
||||
(urlDetails, created) =>
|
||||
urlDetails.updateAttributes({lastModified})
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
)
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
_clearUrlDetails(project_id, url, callback) {
|
||||
if (callback == null) { callback = function(error) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.destroy({where: {url, project_id}})
|
||||
.then(() => cb(null))
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
},
|
||||
|
||||
|
||||
_findAllUrlsInProject(project_id, callback) {
|
||||
if (callback == null) { callback = function(error, urls) {}; }
|
||||
const job = cb=>
|
||||
db.UrlCache.findAll({where: { project_id }})
|
||||
.then(
|
||||
urlEntries => cb(null, urlEntries.map(entry => entry.url)))
|
||||
.error(cb)
|
||||
;
|
||||
return dbQueue.queue.push(job, callback);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
96
app/js/UrlFetcher.js
Normal file
96
app/js/UrlFetcher.js
Normal file
@@ -0,0 +1,96 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlFetcher;
|
||||
const request = require("request").defaults({jar: false});
|
||||
const fs = require("fs");
|
||||
const logger = require("logger-sharelatex");
|
||||
const settings = require("settings-sharelatex");
|
||||
const URL = require('url');
|
||||
|
||||
const oneMinute = 60 * 1000;
|
||||
|
||||
module.exports = (UrlFetcher = {
|
||||
pipeUrlToFile(url, filePath, _callback) {
|
||||
if (_callback == null) { _callback = function(error) {}; }
|
||||
const callbackOnce = function(error) {
|
||||
if (timeoutHandler != null) { clearTimeout(timeoutHandler); }
|
||||
_callback(error);
|
||||
return _callback = function() {};
|
||||
};
|
||||
|
||||
if (settings.filestoreDomainOveride != null) {
|
||||
const p = URL.parse(url).path;
|
||||
url = `${settings.filestoreDomainOveride}${p}`;
|
||||
}
|
||||
var timeoutHandler = setTimeout(function() {
|
||||
timeoutHandler = null;
|
||||
logger.error({url, filePath}, "Timed out downloading file to cache");
|
||||
return callbackOnce(new Error(`Timed out downloading file to cache ${url}`));
|
||||
}
|
||||
// FIXME: maybe need to close fileStream here
|
||||
, 3 * oneMinute);
|
||||
|
||||
logger.log({url, filePath}, "started downloading url to cache");
|
||||
const urlStream = request.get({url, timeout: oneMinute});
|
||||
urlStream.pause(); // stop data flowing until we are ready
|
||||
|
||||
// attach handlers before setting up pipes
|
||||
urlStream.on("error", function(error) {
|
||||
logger.error({err: error, url, filePath}, "error downloading url");
|
||||
return callbackOnce(error || new Error(`Something went wrong downloading the URL ${url}`));
|
||||
});
|
||||
|
||||
urlStream.on("end", () => logger.log({url, filePath}, "finished downloading file into cache"));
|
||||
|
||||
return urlStream.on("response", function(res) {
|
||||
if ((res.statusCode >= 200) && (res.statusCode < 300)) {
|
||||
const fileStream = fs.createWriteStream(filePath);
|
||||
|
||||
// attach handlers before setting up pipes
|
||||
fileStream.on('error', function(error) {
|
||||
logger.error({err: error, url, filePath}, "error writing file into cache");
|
||||
return fs.unlink(filePath, function(err) {
|
||||
if (err != null) {
|
||||
logger.err({err, filePath}, "error deleting file from cache");
|
||||
}
|
||||
return callbackOnce(error);
|
||||
});
|
||||
});
|
||||
|
||||
fileStream.on('finish', function() {
|
||||
logger.log({url, filePath}, "finished writing file into cache");
|
||||
return callbackOnce();
|
||||
});
|
||||
|
||||
fileStream.on('pipe', () => logger.log({url, filePath}, "piping into filestream"));
|
||||
|
||||
urlStream.pipe(fileStream);
|
||||
return urlStream.resume(); // now we are ready to handle the data
|
||||
} else {
|
||||
logger.error({statusCode: res.statusCode, url, filePath}, "unexpected status code downloading url to cache");
|
||||
// https://nodejs.org/api/http.html#http_class_http_clientrequest
|
||||
// If you add a 'response' event handler, then you must consume
|
||||
// the data from the response object, either by calling
|
||||
// response.read() whenever there is a 'readable' event, or by
|
||||
// adding a 'data' handler, or by calling the .resume()
|
||||
// method. Until the data is consumed, the 'end' event will not
|
||||
// fire. Also, until the data is read it will consume memory
|
||||
// that can eventually lead to a 'process out of memory' error.
|
||||
urlStream.resume(); // discard the data
|
||||
return callbackOnce(new Error(`URL returned non-success status code: ${res.statusCode} ${url}`));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
64
app/js/db.js
Normal file
64
app/js/db.js
Normal file
@@ -0,0 +1,64 @@
|
||||
/* eslint-disable
|
||||
no-console,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const Sequelize = require("sequelize");
|
||||
const Settings = require("settings-sharelatex");
|
||||
const _ = require("underscore");
|
||||
const logger = require("logger-sharelatex");
|
||||
|
||||
const options = _.extend({logging:false}, Settings.mysql.clsi);
|
||||
|
||||
logger.log({dbPath:Settings.mysql.clsi.storage}, "connecting to db");
|
||||
|
||||
const sequelize = new Sequelize(
|
||||
Settings.mysql.clsi.database,
|
||||
Settings.mysql.clsi.username,
|
||||
Settings.mysql.clsi.password,
|
||||
options
|
||||
);
|
||||
|
||||
if (Settings.mysql.clsi.dialect === "sqlite") {
|
||||
logger.log("running PRAGMA journal_mode=WAL;");
|
||||
sequelize.query("PRAGMA journal_mode=WAL;");
|
||||
sequelize.query("PRAGMA synchronous=OFF;");
|
||||
sequelize.query("PRAGMA read_uncommitted = true;");
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
UrlCache: sequelize.define("UrlCache", {
|
||||
url: Sequelize.STRING,
|
||||
project_id: Sequelize.STRING,
|
||||
lastModified: Sequelize.DATE
|
||||
}, {
|
||||
indexes: [
|
||||
{fields: ['url', 'project_id']},
|
||||
{fields: ['project_id']}
|
||||
]
|
||||
}),
|
||||
|
||||
Project: sequelize.define("Project", {
|
||||
project_id: {type: Sequelize.STRING, primaryKey: true},
|
||||
lastAccessed: Sequelize.DATE
|
||||
}, {
|
||||
indexes: [
|
||||
{fields: ['lastAccessed']}
|
||||
]
|
||||
}),
|
||||
|
||||
op: Sequelize.Op,
|
||||
|
||||
sync() {
|
||||
logger.log({dbPath:Settings.mysql.clsi.storage}, "syncing db schema");
|
||||
return sequelize.sync()
|
||||
.then(() => logger.log("db sync complete")).catch(err=> console.log(err, "error syncing"));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user