Craftadoc/clsi
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

decaffeinate: Convert CommandRunner.coffee and 25 other files to JS

Browse Source
This commit is contained in:
decaffeinate
2020-02-19 12:14:14 +01:00
committed by mserranom
parent 37794788ce
commit 4655768fd2
26 changed files with 2801 additions and 1964 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
app/coffee/CommandRunner.js
View File

@@ -1,11 +1,18 @@
Settings = require "settings-sharelatex" /*
logger = require "logger-sharelatex" * decaffeinate suggestions:
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let commandRunnerPath;
const Settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
if Settings.clsi?.dockerRunner == true if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) {
commandRunnerPath = "./DockerRunner" commandRunnerPath = "./DockerRunner";
else } else {
commandRunnerPath = "./LocalCommandRunner" commandRunnerPath = "./LocalCommandRunner";
logger.info commandRunnerPath:commandRunnerPath, "selecting command runner for clsi" }
CommandRunner = require(commandRunnerPath) logger.info({commandRunnerPath}, "selecting command runner for clsi");
const CommandRunner = require(commandRunnerPath);
module.exports = CommandRunner module.exports = CommandRunner;

244
app/coffee/CompileController.js
View File

@@ -1,119 +1,163 @@
RequestParser = require "./RequestParser" /*
CompileManager = require "./CompileManager" * decaffeinate suggestions:
Settings = require "settings-sharelatex" * DS101: Remove unnecessary use of Array.from
Metrics = require "./Metrics" * DS102: Remove unnecessary code created because of implicit returns
ProjectPersistenceManager = require "./ProjectPersistenceManager" * DS207: Consider shorter variations of null checks
logger = require "logger-sharelatex" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
Errors = require "./Errors" */
let CompileController;
const RequestParser = require("./RequestParser");
const CompileManager = require("./CompileManager");
const Settings = require("settings-sharelatex");
const Metrics = require("./Metrics");
const ProjectPersistenceManager = require("./ProjectPersistenceManager");
const logger = require("logger-sharelatex");
const Errors = require("./Errors");
module.exports = CompileController = module.exports = (CompileController = {
compile: (req, res, next = (error) ->) -> compile(req, res, next) {
timer = new Metrics.Timer("compile-request") if (next == null) { next = function(error) {}; }
RequestParser.parse req.body, (error, request) -> const timer = new Metrics.Timer("compile-request");
return next(error) if error? return RequestParser.parse(req.body, function(error, request) {
request.project_id = req.params.project_id if (error != null) { return next(error); }
request.user_id = req.params.user_id if req.params.user_id? request.project_id = req.params.project_id;
ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) -> if (req.params.user_id != null) { request.user_id = req.params.user_id; }
return next(error) if error? return ProjectPersistenceManager.markProjectAsJustAccessed(request.project_id, function(error) {
CompileManager.doCompileWithLock request, (error, outputFiles = []) -> if (error != null) { return next(error); }
if error instanceof Errors.AlreadyCompilingError return CompileManager.doCompileWithLock(request, function(error, outputFiles) {
code = 423 # Http 423 Locked let code, status;
status = "compile-in-progress" if (outputFiles == null) { outputFiles = []; }
else if error instanceof Errors.FilesOutOfSyncError if (error instanceof Errors.AlreadyCompilingError) {
code = 409 # Http 409 Conflict code = 423; // Http 423 Locked
status = "retry" status = "compile-in-progress";
else if error?.terminated } else if (error instanceof Errors.FilesOutOfSyncError) {
status = "terminated" code = 409; // Http 409 Conflict
else if error?.validate status = "retry";
status = "validation-#{error.validate}" } else if (error != null ? error.terminated : undefined) {
else if error?.timedout status = "terminated";
status = "timedout" } else if (error != null ? error.validate : undefined) {
logger.log err: error, project_id: request.project_id, "timeout running compile" status = `validation-${error.validate}`;
else if error? } else if (error != null ? error.timedout : undefined) {
status = "error" status = "timedout";
code = 500 logger.log({err: error, project_id: request.project_id}, "timeout running compile");
logger.warn err: error, project_id: request.project_id, "error running compile" } else if (error != null) {
else status = "error";
status = "failure" code = 500;
for file in outputFiles logger.warn({err: error, project_id: request.project_id}, "error running compile");
if file.path?.match(/output\.pdf$/) } else {
status = "success" let file;
status = "failure";
for (file of Array.from(outputFiles)) {
if (file.path != null ? file.path.match(/output\.pdf$/) : undefined) {
status = "success";
}
}
if status == "failure" if (status === "failure") {
logger.warn project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated" logger.warn({project_id: request.project_id, outputFiles}, "project failed to compile successfully, no output.pdf generated");
}
# log an error if any core files are found // log an error if any core files are found
for file in outputFiles for (file of Array.from(outputFiles)) {
if file.path is "core" if (file.path === "core") {
logger.error project_id:request.project_id, req:req, outputFiles:outputFiles, "core file found in output" logger.error({project_id:request.project_id, req, outputFiles}, "core file found in output");
}
}
}
if error? if (error != null) {
outputFiles = error.outputFiles || [] outputFiles = error.outputFiles || [];
}
timer.done() timer.done();
res.status(code or 200).send { return res.status(code || 200).send({
compile: compile: {
status: status status,
error: error?.message or error error: (error != null ? error.message : undefined) || error,
outputFiles: outputFiles.map (file) -> outputFiles: outputFiles.map(file =>
({
url: url:
"#{Settings.apis.clsi.url}/project/#{request.project_id}" + `${Settings.apis.clsi.url}/project/${request.project_id}` +
(if request.user_id? then "/user/#{request.user_id}" else "") + ((request.user_id != null) ? `/user/${request.user_id}` : "") +
(if file.build? then "/build/#{file.build}" else "") + ((file.build != null) ? `/build/${file.build}` : "") +
"/output/#{file.path}" `/output/${file.path}`,
path: file.path path: file.path,
type: file.type type: file.type,
build: file.build build: file.build
})
)
} }
});
});
});
});
},
stopCompile: (req, res, next) -> stopCompile(req, res, next) {
{project_id, user_id} = req.params const {project_id, user_id} = req.params;
CompileManager.stopCompile project_id, user_id, (error) -> return CompileManager.stopCompile(project_id, user_id, function(error) {
return next(error) if error? if (error != null) { return next(error); }
res.sendStatus(204) return res.sendStatus(204);
});
},
clearCache: (req, res, next = (error) ->) -> clearCache(req, res, next) {
ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) -> if (next == null) { next = function(error) {}; }
return next(error) if error? return ProjectPersistenceManager.clearProject(req.params.project_id, req.params.user_id, function(error) {
res.sendStatus(204) # No content if (error != null) { return next(error); }
return res.sendStatus(204);
});
}, // No content
syncFromCode: (req, res, next = (error) ->) -> syncFromCode(req, res, next) {
file = req.query.file if (next == null) { next = function(error) {}; }
line = parseInt(req.query.line, 10) const { file } = req.query;
column = parseInt(req.query.column, 10) const line = parseInt(req.query.line, 10);
project_id = req.params.project_id const column = parseInt(req.query.column, 10);
user_id = req.params.user_id const { project_id } = req.params;
CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) -> const { user_id } = req.params;
return next(error) if error? return CompileManager.syncFromCode(project_id, user_id, file, line, column, function(error, pdfPositions) {
res.json { if (error != null) { return next(error); }
return res.json({
pdf: pdfPositions pdf: pdfPositions
} });
});
},
syncFromPdf: (req, res, next = (error) ->) -> syncFromPdf(req, res, next) {
page = parseInt(req.query.page, 10) if (next == null) { next = function(error) {}; }
h = parseFloat(req.query.h) const page = parseInt(req.query.page, 10);
v = parseFloat(req.query.v) const h = parseFloat(req.query.h);
project_id = req.params.project_id const v = parseFloat(req.query.v);
user_id = req.params.user_id const { project_id } = req.params;
CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) -> const { user_id } = req.params;
return next(error) if error? return CompileManager.syncFromPdf(project_id, user_id, page, h, v, function(error, codePositions) {
res.json { if (error != null) { return next(error); }
return res.json({
code: codePositions code: codePositions
} });
});
},
wordcount: (req, res, next = (error) ->) -> wordcount(req, res, next) {
file = req.query.file || "main.tex" if (next == null) { next = function(error) {}; }
project_id = req.params.project_id const file = req.query.file || "main.tex";
user_id = req.params.user_id const { project_id } = req.params;
image = req.query.image const { user_id } = req.params;
logger.log {image, file, project_id}, "word count request" const { image } = req.query;
logger.log({image, file, project_id}, "word count request");
CompileManager.wordcount project_id, user_id, file, image, (error, result) -> return CompileManager.wordcount(project_id, user_id, file, image, function(error, result) {
return next(error) if error? if (error != null) { return next(error); }
res.json { return res.json({
texcount: result texcount: result
});
});
},
status(req, res, next ){
if (next == null) { next = function(error){}; }
return res.send("OK");
} }
});
status: (req, res, next = (error)-> )->
res.send("OK")

721
app/coffee/CompileManager.js
View File

@@ -1,345 +1,454 @@
ResourceWriter = require "./ResourceWriter" /*
LatexRunner = require "./LatexRunner" * decaffeinate suggestions:
OutputFileFinder = require "./OutputFileFinder" * DS101: Remove unnecessary use of Array.from
OutputCacheManager = require "./OutputCacheManager" * DS102: Remove unnecessary code created because of implicit returns
Settings = require("settings-sharelatex") * DS103: Rewrite code to no longer use __guard__
Path = require "path" * DS207: Consider shorter variations of null checks
logger = require "logger-sharelatex" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
Metrics = require "./Metrics" */
child_process = require "child_process" let CompileManager;
DraftModeManager = require "./DraftModeManager" const ResourceWriter = require("./ResourceWriter");
TikzManager = require "./TikzManager" const LatexRunner = require("./LatexRunner");
LockManager = require "./LockManager" const OutputFileFinder = require("./OutputFileFinder");
fs = require("fs") const OutputCacheManager = require("./OutputCacheManager");
fse = require "fs-extra" const Settings = require("settings-sharelatex");
os = require("os") const Path = require("path");
async = require "async" const logger = require("logger-sharelatex");
Errors = require './Errors' const Metrics = require("./Metrics");
CommandRunner = require "./CommandRunner" const child_process = require("child_process");
const DraftModeManager = require("./DraftModeManager");
const TikzManager = require("./TikzManager");
const LockManager = require("./LockManager");
const fs = require("fs");
const fse = require("fs-extra");
const os = require("os");
const async = require("async");
const Errors = require('./Errors');
const CommandRunner = require("./CommandRunner");
getCompileName = (project_id, user_id) -> const getCompileName = function(project_id, user_id) {
if user_id? then "#{project_id}-#{user_id}" else project_id if (user_id != null) { return `${project_id}-${user_id}`; } else { return project_id; }
};
getCompileDir = (project_id, user_id) -> const getCompileDir = (project_id, user_id) => Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id));
Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
module.exports = CompileManager = module.exports = (CompileManager = {
doCompileWithLock: (request, callback = (error, outputFiles) ->) -> doCompileWithLock(request, callback) {
compileDir = getCompileDir(request.project_id, request.user_id) if (callback == null) { callback = function(error, outputFiles) {}; }
lockFile = Path.join(compileDir, ".project-lock") const compileDir = getCompileDir(request.project_id, request.user_id);
# use a .project-lock file in the compile directory to prevent const lockFile = Path.join(compileDir, ".project-lock");
# simultaneous compiles // use a .project-lock file in the compile directory to prevent
fse.ensureDir compileDir, (error) -> // simultaneous compiles
return callback(error) if error? return fse.ensureDir(compileDir, function(error) {
LockManager.runWithLock lockFile, (releaseLock) -> if (error != null) { return callback(error); }
CompileManager.doCompile(request, releaseLock) return LockManager.runWithLock(lockFile, releaseLock => CompileManager.doCompile(request, releaseLock)
, callback , callback);
});
},
doCompile: (request, callback = (error, outputFiles) ->) -> doCompile(request, callback) {
compileDir = getCompileDir(request.project_id, request.user_id) if (callback == null) { callback = function(error, outputFiles) {}; }
timer = new Metrics.Timer("write-to-disk") const compileDir = getCompileDir(request.project_id, request.user_id);
logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk" let timer = new Metrics.Timer("write-to-disk");
ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) -> logger.log({project_id: request.project_id, user_id: request.user_id}, "syncing resources to disk");
# NOTE: resourceList is insecure, it should only be used to exclude files from the output list return ResourceWriter.syncResourcesToDisk(request, compileDir, function(error, resourceList) {
if error? and error instanceof Errors.FilesOutOfSyncError // NOTE: resourceList is insecure, it should only be used to exclude files from the output list
logger.warn project_id: request.project_id, user_id: request.user_id, "files out of sync, please retry" if ((error != null) && error instanceof Errors.FilesOutOfSyncError) {
return callback(error) logger.warn({project_id: request.project_id, user_id: request.user_id}, "files out of sync, please retry");
else if error? return callback(error);
logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk" } else if (error != null) {
return callback(error) logger.err({err:error, project_id: request.project_id, user_id: request.user_id}, "error writing resources to disk");
logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk" return callback(error);
timer.done() }
logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start}, "written files to disk");
timer.done();
injectDraftModeIfRequired = (callback) -> const injectDraftModeIfRequired = function(callback) {
if request.draft if (request.draft) {
DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback return DraftModeManager.injectDraftMode(Path.join(compileDir, request.rootResourcePath), callback);
else } else {
callback() return callback();
}
};
createTikzFileIfRequired = (callback) -> const createTikzFileIfRequired = callback =>
TikzManager.checkMainFile compileDir, request.rootResourcePath, resourceList, (error, needsMainFile) -> TikzManager.checkMainFile(compileDir, request.rootResourcePath, resourceList, function(error, needsMainFile) {
return callback(error) if error? if (error != null) { return callback(error); }
if needsMainFile if (needsMainFile) {
TikzManager.injectOutputFile compileDir, request.rootResourcePath, callback return TikzManager.injectOutputFile(compileDir, request.rootResourcePath, callback);
else } else {
callback() return callback();
}
})
;
# set up environment variables for chktex // set up environment variables for chktex
env = {} const env = {};
# only run chktex on LaTeX files (not knitr .Rtex files or any others) // only run chktex on LaTeX files (not knitr .Rtex files or any others)
isLaTeXFile = request.rootResourcePath?.match(/\.tex$/i) const isLaTeXFile = request.rootResourcePath != null ? request.rootResourcePath.match(/\.tex$/i) : undefined;
if request.check? and isLaTeXFile if ((request.check != null) && isLaTeXFile) {
env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16' env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16';
env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000' env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000';
if request.check is 'error' if (request.check === 'error') {
env['CHKTEX_EXIT_ON_ERROR'] = 1 env['CHKTEX_EXIT_ON_ERROR'] = 1;
if request.check is 'validate' }
env['CHKTEX_VALIDATE'] = 1 if (request.check === 'validate') {
env['CHKTEX_VALIDATE'] = 1;
}
}
# apply a series of file modifications/creations for draft mode and tikz // apply a series of file modifications/creations for draft mode and tikz
async.series [injectDraftModeIfRequired, createTikzFileIfRequired], (error) -> return async.series([injectDraftModeIfRequired, createTikzFileIfRequired], function(error) {
return callback(error) if error? if (error != null) { return callback(error); }
timer = new Metrics.Timer("run-compile") timer = new Metrics.Timer("run-compile");
# find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite) // find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
tag = request.imageName?.match(/:(.*)/)?[1]?.replace(/\./g,'-') or "default" let tag = __guard__(__guard__(request.imageName != null ? request.imageName.match(/:(.*)/) : undefined, x1 => x1[1]), x => x.replace(/\./g,'-')) || "default";
tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test if (!request.project_id.match(/^[0-9a-f]{24}$/)) { tag = "other"; } // exclude smoke test
Metrics.inc("compiles") Metrics.inc("compiles");
Metrics.inc("compiles-with-image.#{tag}") Metrics.inc(`compiles-with-image.${tag}`);
compileName = getCompileName(request.project_id, request.user_id) const compileName = getCompileName(request.project_id, request.user_id);
LatexRunner.runLatex compileName, { return LatexRunner.runLatex(compileName, {
directory: compileDir directory: compileDir,
mainFile: request.rootResourcePath mainFile: request.rootResourcePath,
compiler: request.compiler compiler: request.compiler,
timeout: request.timeout timeout: request.timeout,
image: request.imageName image: request.imageName,
flags: request.flags flags: request.flags,
environment: env environment: env
}, (error, output, stats, timings) -> }, function(error, output, stats, timings) {
# request was for validation only // request was for validation only
if request.check is "validate" let metric_key, metric_value;
result = if error?.code then "fail" else "pass" if (request.check === "validate") {
error = new Error("validation") const result = (error != null ? error.code : undefined) ? "fail" : "pass";
error.validate = result error = new Error("validation");
# request was for compile, and failed on validation error.validate = result;
if request.check is "error" and error?.message is 'exited' }
error = new Error("compilation") // request was for compile, and failed on validation
error.validate = "fail" if ((request.check === "error") && ((error != null ? error.message : undefined) === 'exited')) {
# compile was killed by user, was a validation, or a compile which failed validation error = new Error("compilation");
if error?.terminated or error?.validate or error?.timedout error.validate = "fail";
OutputFileFinder.findOutputFiles resourceList, compileDir, (err, outputFiles) -> }
return callback(err) if err? // compile was killed by user, was a validation, or a compile which failed validation
error.outputFiles = outputFiles # return output files so user can check logs if ((error != null ? error.terminated : undefined) || (error != null ? error.validate : undefined) || (error != null ? error.timedout : undefined)) {
callback(error) OutputFileFinder.findOutputFiles(resourceList, compileDir, function(err, outputFiles) {
return if (err != null) { return callback(err); }
# compile completed normally error.outputFiles = outputFiles; // return output files so user can check logs
return callback(error) if error? return callback(error);
Metrics.inc("compiles-succeeded") });
for metric_key, metric_value of stats or {} return;
Metrics.count(metric_key, metric_value) }
for metric_key, metric_value of timings or {} // compile completed normally
Metrics.timing(metric_key, metric_value) if (error != null) { return callback(error); }
loadavg = os.loadavg?() Metrics.inc("compiles-succeeded");
Metrics.gauge("load-avg", loadavg[0]) if loadavg? const object = stats || {};
ts = timer.done() for (metric_key in object) {
logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" metric_value = object[metric_key];
if stats?["latex-runs"] > 0 Metrics.count(metric_key, metric_value);
Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]) }
if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0 const object1 = timings || {};
Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]) for (metric_key in object1) {
metric_value = object1[metric_key];
Metrics.timing(metric_key, metric_value);
}
const loadavg = typeof os.loadavg === 'function' ? os.loadavg() : undefined;
if (loadavg != null) { Metrics.gauge("load-avg", loadavg[0]); }
const ts = timer.done();
logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats, timings, loadavg}, "done compile");
if ((stats != null ? stats["latex-runs"] : undefined) > 0) {
Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]);
}
if (((stats != null ? stats["latex-runs"] : undefined) > 0) && ((timings != null ? timings["cpu-time"] : undefined) > 0)) {
Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]);
}
OutputFileFinder.findOutputFiles resourceList, compileDir, (error, outputFiles) -> return OutputFileFinder.findOutputFiles(resourceList, compileDir, function(error, outputFiles) {
return callback(error) if error? if (error != null) { return callback(error); }
OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> return OutputCacheManager.saveOutputFiles(outputFiles, compileDir, (error, newOutputFiles) => callback(null, newOutputFiles));
callback null, newOutputFiles });
});
});
});
},
stopCompile: (project_id, user_id, callback = (error) ->) -> stopCompile(project_id, user_id, callback) {
compileName = getCompileName(project_id, user_id) if (callback == null) { callback = function(error) {}; }
LatexRunner.killLatex compileName, callback const compileName = getCompileName(project_id, user_id);
return LatexRunner.killLatex(compileName, callback);
},
clearProject: (project_id, user_id, _callback = (error) ->) -> clearProject(project_id, user_id, _callback) {
callback = (error) -> if (_callback == null) { _callback = function(error) {}; }
_callback(error) const callback = function(error) {
_callback = () -> _callback(error);
return _callback = function() {};
};
compileDir = getCompileDir(project_id, user_id) const compileDir = getCompileDir(project_id, user_id);
CompileManager._checkDirectory compileDir, (err, exists) -> return CompileManager._checkDirectory(compileDir, function(err, exists) {
return callback(err) if err? if (err != null) { return callback(err); }
return callback() if not exists # skip removal if no directory present if (!exists) { return callback(); } // skip removal if no directory present
proc = child_process.spawn "rm", ["-r", compileDir] const proc = child_process.spawn("rm", ["-r", compileDir]);
proc.on "error", callback proc.on("error", callback);
stderr = "" let stderr = "";
proc.stderr.on "data", (chunk) -> stderr += chunk.toString() proc.stderr.on("data", chunk => stderr += chunk.toString());
proc.on "close", (code) -> return proc.on("close", function(code) {
if code == 0 if (code === 0) {
return callback(null) return callback(null);
else } else {
return callback(new Error("rm -r #{compileDir} failed: #{stderr}")) return callback(new Error(`rm -r ${compileDir} failed: ${stderr}`));
}
});
});
},
_findAllDirs: (callback = (error, allDirs) ->) -> _findAllDirs(callback) {
root = Settings.path.compilesDir if (callback == null) { callback = function(error, allDirs) {}; }
fs.readdir root, (err, files) -> const root = Settings.path.compilesDir;
return callback(err) if err? return fs.readdir(root, function(err, files) {
allDirs = (Path.join(root, file) for file in files) if (err != null) { return callback(err); }
callback(null, allDirs) const allDirs = (Array.from(files).map((file) => Path.join(root, file)));
return callback(null, allDirs);
});
},
clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) -> clearExpiredProjects(max_cache_age_ms, callback) {
now = Date.now() if (callback == null) { callback = function(error) {}; }
# action for each directory const now = Date.now();
expireIfNeeded = (checkDir, cb) -> // action for each directory
fs.stat checkDir, (err, stats) -> const expireIfNeeded = (checkDir, cb) =>
return cb() if err? # ignore errors checking directory fs.stat(checkDir, function(err, stats) {
age = now - stats.mtime if (err != null) { return cb(); } // ignore errors checking directory
hasExpired = (age > max_cache_age_ms) const age = now - stats.mtime;
if hasExpired then fse.remove(checkDir, cb) else cb() const hasExpired = (age > max_cache_age_ms);
# iterate over all project directories if (hasExpired) { return fse.remove(checkDir, cb); } else { return cb(); }
CompileManager._findAllDirs (error, allDirs) -> })
return callback() if error? ;
async.eachSeries allDirs, expireIfNeeded, callback // iterate over all project directories
return CompileManager._findAllDirs(function(error, allDirs) {
if (error != null) { return callback(); }
return async.eachSeries(allDirs, expireIfNeeded, callback);
});
},
_checkDirectory: (compileDir, callback = (error, exists) ->) -> _checkDirectory(compileDir, callback) {
fs.lstat compileDir, (err, stats) -> if (callback == null) { callback = function(error, exists) {}; }
if err?.code is 'ENOENT' return fs.lstat(compileDir, function(err, stats) {
return callback(null, false) # directory does not exist if ((err != null ? err.code : undefined) === 'ENOENT') {
else if err? return callback(null, false); // directory does not exist
logger.err {dir: compileDir, err:err}, "error on stat of project directory for removal" } else if (err != null) {
return callback(err) logger.err({dir: compileDir, err}, "error on stat of project directory for removal");
else if not stats?.isDirectory() return callback(err);
logger.err {dir: compileDir, stats:stats}, "bad project directory for removal" } else if (!(stats != null ? stats.isDirectory() : undefined)) {
return callback new Error("project directory is not directory") logger.err({dir: compileDir, stats}, "bad project directory for removal");
else return callback(new Error("project directory is not directory"));
callback(null, true) # directory exists } else {
return callback(null, true);
}
});
}, // directory exists
syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) -> syncFromCode(project_id, user_id, file_name, line, column, callback) {
# If LaTeX was run in a virtual environment, the file path that synctex expects // If LaTeX was run in a virtual environment, the file path that synctex expects
# might not match the file path on the host. The .synctex.gz file however, will be accessed // might not match the file path on the host. The .synctex.gz file however, will be accessed
# wherever it is on the host. // wherever it is on the host.
compileName = getCompileName(project_id, user_id) if (callback == null) { callback = function(error, pdfPositions) {}; }
base_dir = Settings.path.synctexBaseDir(compileName) const compileName = getCompileName(project_id, user_id);
file_path = base_dir + "/" + file_name const base_dir = Settings.path.synctexBaseDir(compileName);
compileDir = getCompileDir(project_id, user_id) const file_path = base_dir + "/" + file_name;
synctex_path = "#{base_dir}/output.pdf" const compileDir = getCompileDir(project_id, user_id);
command = ["code", synctex_path, file_path, line, column] const synctex_path = `${base_dir}/output.pdf`;
fse.ensureDir compileDir, (error) -> const command = ["code", synctex_path, file_path, line, column];
if error? return fse.ensureDir(compileDir, function(error) {
logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code" if (error != null) {
return callback(error) logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code");
CompileManager._runSynctex project_id, user_id, command, (error, stdout) -> return callback(error);
return callback(error) if error? }
logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output" return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) {
callback null, CompileManager._parseSynctexFromCodeOutput(stdout) if (error != null) { return callback(error); }
logger.log({project_id, user_id, file_name, line, column, command, stdout}, "synctex code output");
return callback(null, CompileManager._parseSynctexFromCodeOutput(stdout));
});
});
},
syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) -> syncFromPdf(project_id, user_id, page, h, v, callback) {
compileName = getCompileName(project_id, user_id) if (callback == null) { callback = function(error, filePositions) {}; }
compileDir = getCompileDir(project_id, user_id) const compileName = getCompileName(project_id, user_id);
base_dir = Settings.path.synctexBaseDir(compileName) const compileDir = getCompileDir(project_id, user_id);
synctex_path = "#{base_dir}/output.pdf" const base_dir = Settings.path.synctexBaseDir(compileName);
command = ["pdf", synctex_path, page, h, v] const synctex_path = `${base_dir}/output.pdf`;
fse.ensureDir compileDir, (error) -> const command = ["pdf", synctex_path, page, h, v];
if error? return fse.ensureDir(compileDir, function(error) {
logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync to code" if (error != null) {
return callback(error) logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync to code");
CompileManager._runSynctex project_id, user_id, command, (error, stdout) -> return callback(error);
return callback(error) if error? }
logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) {
callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir) if (error != null) { return callback(error); }
logger.log({project_id, user_id, page, h, v, stdout}, "synctex pdf output");
return callback(null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir));
});
});
},
_checkFileExists: (path, callback = (error) ->) -> _checkFileExists(path, callback) {
synctexDir = Path.dirname(path) if (callback == null) { callback = function(error) {}; }
synctexFile = Path.join(synctexDir, "output.synctex.gz") const synctexDir = Path.dirname(path);
fs.stat synctexDir, (error, stats) -> const synctexFile = Path.join(synctexDir, "output.synctex.gz");
if error?.code is 'ENOENT' return fs.stat(synctexDir, function(error, stats) {
return callback(new Errors.NotFoundError("called synctex with no output directory")) if ((error != null ? error.code : undefined) === 'ENOENT') {
return callback(error) if error? return callback(new Errors.NotFoundError("called synctex with no output directory"));
fs.stat synctexFile, (error, stats) -> }
if error?.code is 'ENOENT' if (error != null) { return callback(error); }
return callback(new Errors.NotFoundError("called synctex with no output file")) return fs.stat(synctexFile, function(error, stats) {
return callback(error) if error? if ((error != null ? error.code : undefined) === 'ENOENT') {
return callback(new Error("not a file")) if not stats?.isFile() return callback(new Errors.NotFoundError("called synctex with no output file"));
callback() }
if (error != null) { return callback(error); }
if (!(stats != null ? stats.isFile() : undefined)) { return callback(new Error("not a file")); }
return callback();
});
});
},
_runSynctex: (project_id, user_id, command, callback = (error, stdout) ->) -> _runSynctex(project_id, user_id, command, callback) {
seconds = 1000 if (callback == null) { callback = function(error, stdout) {}; }
const seconds = 1000;
command.unshift("/opt/synctex") command.unshift("/opt/synctex");
directory = getCompileDir(project_id, user_id) const directory = getCompileDir(project_id, user_id);
timeout = 60 * 1000 # increased to allow for large projects const timeout = 60 * 1000; // increased to allow for large projects
compileName = getCompileName(project_id, user_id) const compileName = getCompileName(project_id, user_id);
CommandRunner.run compileName, command, directory, Settings.clsi?.docker.image, timeout, {}, (error, output) -> return CommandRunner.run(compileName, command, directory, Settings.clsi != null ? Settings.clsi.docker.image : undefined, timeout, {}, function(error, output) {
if error? if (error != null) {
logger.err err:error, command:command, project_id:project_id, user_id:user_id, "error running synctex" logger.err({err:error, command, project_id, user_id}, "error running synctex");
return callback(error) return callback(error);
callback(null, output.stdout) }
return callback(null, output.stdout);
});
},
_parseSynctexFromCodeOutput: (output) -> _parseSynctexFromCodeOutput(output) {
results = [] const results = [];
for line in output.split("\n") for (let line of Array.from(output.split("\n"))) {
[node, page, h, v, width, height] = line.split("\t") const [node, page, h, v, width, height] = Array.from(line.split("\t"));
if node == "NODE" if (node === "NODE") {
results.push { results.push({
page: parseInt(page, 10) page: parseInt(page, 10),
h: parseFloat(h) h: parseFloat(h),
v: parseFloat(v) v: parseFloat(v),
height: parseFloat(height) height: parseFloat(height),
width: parseFloat(width) width: parseFloat(width)
});
} }
return results }
return results;
},
_parseSynctexFromPdfOutput: (output, base_dir) -> _parseSynctexFromPdfOutput(output, base_dir) {
results = [] const results = [];
for line in output.split("\n") for (let line of Array.from(output.split("\n"))) {
[node, file_path, line, column] = line.split("\t") let column, file_path, node;
if node == "NODE" [node, file_path, line, column] = Array.from(line.split("\t"));
file = file_path.slice(base_dir.length + 1) if (node === "NODE") {
results.push { const file = file_path.slice(base_dir.length + 1);
file: file results.push({
line: parseInt(line, 10) file,
line: parseInt(line, 10),
column: parseInt(column, 10) column: parseInt(column, 10)
});
} }
return results }
return results;
},
wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) -> wordcount(project_id, user_id, file_name, image, callback) {
logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount" if (callback == null) { callback = function(error, pdfPositions) {}; }
file_path = "$COMPILE_DIR/" + file_name logger.log({project_id, user_id, file_name, image}, "running wordcount");
command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"] const file_path = `$COMPILE_DIR/${file_name}`;
compileDir = getCompileDir(project_id, user_id) const command = [ "texcount", '-nocol', '-inc', file_path, `-out=${file_path}.wc`];
timeout = 60 * 1000 const compileDir = getCompileDir(project_id, user_id);
compileName = getCompileName(project_id, user_id) const timeout = 60 * 1000;
fse.ensureDir compileDir, (error) -> const compileName = getCompileName(project_id, user_id);
if error? return fse.ensureDir(compileDir, function(error) {
logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code" if (error != null) {
return callback(error) logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code");
CommandRunner.run compileName, command, compileDir, image, timeout, {}, (error) -> return callback(error);
return callback(error) if error? }
fs.readFile compileDir + "/" + file_name + ".wc", "utf-8", (err, stdout) -> return CommandRunner.run(compileName, command, compileDir, image, timeout, {}, function(error) {
if err? if (error != null) { return callback(error); }
#call it node_err so sentry doesn't use random path error as unique id so it can't be ignored return fs.readFile(compileDir + "/" + file_name + ".wc", "utf-8", function(err, stdout) {
logger.err node_err:err, command:command, compileDir:compileDir, project_id:project_id, user_id:user_id, "error reading word count output" if (err != null) {
return callback(err) //call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
results = CompileManager._parseWordcountFromOutput(stdout) logger.err({node_err:err, command, compileDir, project_id, user_id}, "error reading word count output");
logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results" return callback(err);
callback null, results }
const results = CompileManager._parseWordcountFromOutput(stdout);
logger.log({project_id, user_id, wordcount: results}, "word count results");
return callback(null, results);
});
});
});
},
_parseWordcountFromOutput: (output) -> _parseWordcountFromOutput(output) {
results = { const results = {
encode: "" encode: "",
textWords: 0 textWords: 0,
headWords: 0 headWords: 0,
outside: 0 outside: 0,
headers: 0 headers: 0,
elements: 0 elements: 0,
mathInline: 0 mathInline: 0,
mathDisplay: 0 mathDisplay: 0,
errors: 0 errors: 0,
messages: "" messages: ""
};
for (let line of Array.from(output.split("\n"))) {
const [data, info] = Array.from(line.split(":"));
if (data.indexOf("Encoding") > -1) {
results['encode'] = info.trim();
} }
for line in output.split("\n") if (data.indexOf("in text") > -1) {
[data, info] = line.split(":") results['textWords'] = parseInt(info, 10);
if data.indexOf("Encoding") > -1 }
results['encode'] = info.trim() if (data.indexOf("in head") > -1) {
if data.indexOf("in text") > -1 results['headWords'] = parseInt(info, 10);
results['textWords'] = parseInt(info, 10) }
if data.indexOf("in head") > -1 if (data.indexOf("outside") > -1) {
results['headWords'] = parseInt(info, 10) results['outside'] = parseInt(info, 10);
if data.indexOf("outside") > -1 }
results['outside'] = parseInt(info, 10) if (data.indexOf("of head") > -1) {
if data.indexOf("of head") > -1 results['headers'] = parseInt(info, 10);
results['headers'] = parseInt(info, 10) }
if data.indexOf("Number of floats/tables/figures") > -1 if (data.indexOf("Number of floats/tables/figures") > -1) {
results['elements'] = parseInt(info, 10) results['elements'] = parseInt(info, 10);
if data.indexOf("Number of math inlines") > -1 }
results['mathInline'] = parseInt(info, 10) if (data.indexOf("Number of math inlines") > -1) {
if data.indexOf("Number of math displayed") > -1 results['mathInline'] = parseInt(info, 10);
results['mathDisplay'] = parseInt(info, 10) }
if data is "(errors" # errors reported as (errors:123) if (data.indexOf("Number of math displayed") > -1) {
results['errors'] = parseInt(info, 10) results['mathDisplay'] = parseInt(info, 10);
if line.indexOf("!!! ") > -1 # errors logged as !!! message !!! }
results['messages'] += line + "\n" if (data === "(errors") { // errors reported as (errors:123)
return results results['errors'] = parseInt(info, 10);
}
if (line.indexOf("!!! ") > -1) { // errors logged as !!! message !!!
results['messages'] += line + "\n";
}
}
return results;
}
});
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}

50
app/coffee/ContentTypeMapper.js
View File

@@ -1,24 +1,28 @@
Path = require 'path' let ContentTypeMapper;
const Path = require('path');
# here we coerce html, css and js to text/plain, // here we coerce html, css and js to text/plain,
# otherwise choose correct mime type based on file extension, // otherwise choose correct mime type based on file extension,
# falling back to octet-stream // falling back to octet-stream
module.exports = ContentTypeMapper = module.exports = (ContentTypeMapper = {
map: (path) -> map(path) {
switch Path.extname(path) switch (Path.extname(path)) {
when '.txt', '.html', '.js', '.css', '.svg' case '.txt': case '.html': case '.js': case '.css': case '.svg':
return 'text/plain' return 'text/plain';
when '.csv' case '.csv':
return 'text/csv' return 'text/csv';
when '.pdf' case '.pdf':
return 'application/pdf' return 'application/pdf';
when '.png' case '.png':
return 'image/png' return 'image/png';
when '.jpg', '.jpeg' case '.jpg': case '.jpeg':
return 'image/jpeg' return 'image/jpeg';
when '.tiff' case '.tiff':
return 'image/tiff' return 'image/tiff';
when '.gif' case '.gif':
return 'image/gif' return 'image/gif';
else default:
return 'application/octet-stream' return 'application/octet-stream';
}
}
});

21
app/coffee/DbQueue.js
View File

@@ -1,13 +1,16 @@
async = require "async" /*
Settings = require "settings-sharelatex" * decaffeinate suggestions:
logger = require("logger-sharelatex") * DS102: Remove unnecessary code created because of implicit returns
queue = async.queue((task, cb)-> * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
task(cb) */
, Settings.parallelSqlQueryLimit) const async = require("async");
const Settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const queue = async.queue((task, cb)=> task(cb)
, Settings.parallelSqlQueryLimit);
queue.drain = ()-> queue.drain = ()=> logger.debug('all items have been processed');
logger.debug('all items have been processed')
module.exports = module.exports =
queue: queue {queue};

126
app/coffee/DockerLockManager.js
View File

@@ -1,56 +1,84 @@
logger = require "logger-sharelatex" /*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LockManager;
const logger = require("logger-sharelatex");
LockState = {} # locks for docker container operations, by container name const LockState = {}; // locks for docker container operations, by container name
module.exports = LockManager = module.exports = (LockManager = {
MAX_LOCK_HOLD_TIME: 15000 # how long we can keep a lock MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock
MAX_LOCK_WAIT_TIME: 10000 # how long we wait for a lock MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock
LOCK_TEST_INTERVAL: 1000 # retry time LOCK_TEST_INTERVAL: 1000, // retry time
tryLock: (key, callback = (err, gotLock) ->) -> tryLock(key, callback) {
existingLock = LockState[key] let lockValue;
if existingLock? # the lock is already taken, check how old it is if (callback == null) { callback = function(err, gotLock) {}; }
lockAge = Date.now() - existingLock.created const existingLock = LockState[key];
if lockAge < LockManager.MAX_LOCK_HOLD_TIME if (existingLock != null) { // the lock is already taken, check how old it is
return callback(null, false) # we didn't get the lock, bail out const lockAge = Date.now() - existingLock.created;
else if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) {
logger.error {key: key, lock: existingLock, age:lockAge}, "taking old lock by force" return callback(null, false); // we didn't get the lock, bail out
# take the lock } else {
LockState[key] = lockValue = {created: Date.now()} logger.error({key, lock: existingLock, age:lockAge}, "taking old lock by force");
callback(null, true, lockValue) }
}
// take the lock
LockState[key] = (lockValue = {created: Date.now()});
return callback(null, true, lockValue);
},
getLock: (key, callback = (error, lockValue) ->) -> getLock(key, callback) {
startTime = Date.now() let attempt;
do attempt = () -> if (callback == null) { callback = function(error, lockValue) {}; }
LockManager.tryLock key, (error, gotLock, lockValue) -> const startTime = Date.now();
return callback(error) if error? return (attempt = () =>
if gotLock LockManager.tryLock(key, function(error, gotLock, lockValue) {
callback(null, lockValue) if (error != null) { return callback(error); }
else if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME if (gotLock) {
e = new Error("Lock timeout") return callback(null, lockValue);
e.key = key } else if ((Date.now() - startTime) > LockManager.MAX_LOCK_WAIT_TIME) {
return callback(e) const e = new Error("Lock timeout");
else e.key = key;
setTimeout attempt, LockManager.LOCK_TEST_INTERVAL return callback(e);
} else {
return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL);
}
})
)();
},
releaseLock: (key, lockValue, callback = (error) ->) -> releaseLock(key, lockValue, callback) {
existingLock = LockState[key] if (callback == null) { callback = function(error) {}; }
if existingLock is lockValue # lockValue is an object, so we can test by reference const existingLock = LockState[key];
delete LockState[key] # our lock, so we can free it if (existingLock === lockValue) { // lockValue is an object, so we can test by reference
callback() delete LockState[key]; // our lock, so we can free it
else if existingLock? # lock exists but doesn't match ours return callback();
logger.error {key:key, lock: existingLock}, "tried to release lock taken by force" } else if (existingLock != null) { // lock exists but doesn't match ours
callback() logger.error({key, lock: existingLock}, "tried to release lock taken by force");
else return callback();
logger.error {key:key, lock: existingLock}, "tried to release lock that has gone" } else {
callback() logger.error({key, lock: existingLock}, "tried to release lock that has gone");
return callback();
}
},
runWithLock: (key, runner, callback = ( (error) -> )) -> runWithLock(key, runner, callback) {
LockManager.getLock key, (error, lockValue) -> if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return LockManager.getLock(key, function(error, lockValue) {
runner (error1, args...) -> if (error != null) { return callback(error); }
LockManager.releaseLock key, lockValue, (error2) -> return runner((error1, ...args) =>
error = error1 or error2 LockManager.releaseLock(key, lockValue, function(error2) {
return callback(error) if error? error = error1 || error2;
callback(null, args...) if (error != null) { return callback(error); }
return callback(null, ...Array.from(args));
})
);
});
}
});

789
app/coffee/DockerRunner.js
View File

@@ -1,358 +1,475 @@
Settings = require "settings-sharelatex" /*
logger = require "logger-sharelatex" * decaffeinate suggestions:
Docker = require("dockerode") * DS101: Remove unnecessary use of Array.from
dockerode = new Docker() * DS102: Remove unnecessary code created because of implicit returns
crypto = require "crypto" * DS103: Rewrite code to no longer use __guard__
async = require "async" * DS205: Consider reworking code to avoid use of IIFEs
LockManager = require "./DockerLockManager" * DS207: Consider shorter variations of null checks
fs = require "fs" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
Path = require 'path' */
_ = require "underscore" let DockerRunner, oneHour;
const Settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const Docker = require("dockerode");
const dockerode = new Docker();
const crypto = require("crypto");
const async = require("async");
const LockManager = require("./DockerLockManager");
const fs = require("fs");
const Path = require('path');
const _ = require("underscore");
logger.info "using docker runner" logger.info("using docker runner");
usingSiblingContainers = () -> const usingSiblingContainers = () => __guard__(Settings != null ? Settings.path : undefined, x => x.sandboxedCompilesHostDir) != null;
Settings?.path?.sandboxedCompilesHostDir?
module.exports = DockerRunner = module.exports = (DockerRunner = {
ERR_NOT_DIRECTORY: new Error("not a directory") ERR_NOT_DIRECTORY: new Error("not a directory"),
ERR_TERMINATED: new Error("terminated") ERR_TERMINATED: new Error("terminated"),
ERR_EXITED: new Error("exited") ERR_EXITED: new Error("exited"),
ERR_TIMED_OUT: new Error("container timed out") ERR_TIMED_OUT: new Error("container timed out"),
run: (project_id, command, directory, image, timeout, environment, callback = (error, output) ->) -> run(project_id, command, directory, image, timeout, environment, callback) {
if usingSiblingContainers() let name;
_newPath = Settings.path.sandboxedCompilesHostDir if (callback == null) { callback = function(error, output) {}; }
logger.log {path: _newPath}, "altering bind path for sibling containers" if (usingSiblingContainers()) {
# Server Pro, example: const _newPath = Settings.path.sandboxedCompilesHostDir;
# '/var/lib/sharelatex/data/compiles/<project-id>' logger.log({path: _newPath}, "altering bind path for sibling containers");
# ... becomes ... // Server Pro, example:
# '/opt/sharelatex_data/data/compiles/<project-id>' // '/var/lib/sharelatex/data/compiles/<project-id>'
directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory)) // ... becomes ...
// '/opt/sharelatex_data/data/compiles/<project-id>'
volumes = {} directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory));
volumes[directory] = "/compile"
command = (arg.toString().replace?('$COMPILE_DIR', "/compile") for arg in command)
if !image?
image = Settings.clsi.docker.image
if Settings.texliveImageNameOveride?
img = image.split("/")
image = "#{Settings.texliveImageNameOveride}/#{img[2]}"
options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment)
fingerprint = DockerRunner._fingerprintContainer(options)
options.name = name = "project-#{project_id}-#{fingerprint}"
# logOptions = _.clone(options)
# logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
logger.log project_id: project_id, "running docker container"
DockerRunner._runAndWaitForContainer options, volumes, timeout, (error, output) ->
if error?.message?.match("HTTP code is 500")
logger.log err: error, project_id: project_id, "error running container so destroying and retrying"
DockerRunner.destroyContainer name, null, true, (error) ->
return callback(error) if error?
DockerRunner._runAndWaitForContainer options, volumes, timeout, callback
else
callback(error, output)
return name # pass back the container name to allow it to be killed
kill: (container_id, callback = (error) ->) ->
logger.log container_id: container_id, "sending kill signal to container"
container = dockerode.getContainer(container_id)
container.kill (error) ->
if error? and error?.message?.match?(/Cannot kill container .* is not running/)
logger.warn err: error, container_id: container_id, "container not running, continuing"
error = null
if error?
logger.error err: error, container_id: container_id, "error killing container"
return callback(error)
else
callback()
_runAndWaitForContainer: (options, volumes, timeout, _callback = (error, output) ->) ->
callback = (args...) ->
_callback(args...)
# Only call the callback once
_callback = () ->
name = options.name
streamEnded = false
containerReturned = false
output = {}
callbackIfFinished = () ->
if streamEnded and containerReturned
callback(null, output)
attachStreamHandler = (error, _output) ->
return callback(error) if error?
output = _output
streamEnded = true
callbackIfFinished()
DockerRunner.startContainer options, volumes, attachStreamHandler, (error, containerId) ->
return callback(error) if error?
DockerRunner.waitForContainer name, timeout, (error, exitCode) ->
return callback(error) if error?
if exitCode is 137 # exit status from kill -9
err = DockerRunner.ERR_TERMINATED
err.terminated = true
return callback(err)
if exitCode is 1 # exit status from chktex
err = DockerRunner.ERR_EXITED
err.code = exitCode
return callback(err)
containerReturned = true
options?.HostConfig?.SecurityOpt = null #small log line
logger.log err:err, exitCode:exitCode, options:options, "docker container has exited"
callbackIfFinished()
_getContainerOptions: (command, image, volumes, timeout, environment) ->
timeoutInSeconds = timeout / 1000
dockerVolumes = {}
for hostVol, dockerVol of volumes
dockerVolumes[dockerVol] = {}
if volumes[hostVol].slice(-3).indexOf(":r") == -1
volumes[hostVol] = "#{dockerVol}:rw"
# merge settings and environment parameter
env = {}
for src in [Settings.clsi.docker.env, environment or {}]
env[key] = value for key, value of src
# set the path based on the image year
if m = image.match /:([0-9]+)\.[0-9]+/
year = m[1]
else
year = "2014"
env['PATH'] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/#{year}/bin/x86_64-linux/"
options =
"Cmd" : command,
"Image" : image
"Volumes" : dockerVolumes
"WorkingDir" : "/compile"
"NetworkDisabled" : true
"Memory" : 1024 * 1024 * 1024 * 1024 # 1 Gb
"User" : Settings.clsi.docker.user
"Env" : ("#{key}=#{value}" for key, value of env) # convert the environment hash to an array
"HostConfig" :
"Binds": ("#{hostVol}:#{dockerVol}" for hostVol, dockerVol of volumes)
"LogConfig": {"Type": "none", "Config": {}}
"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}]
"CapDrop": "ALL"
"SecurityOpt": ["no-new-privileges"]
if Settings.path?.synctexBinHostPath?
options["HostConfig"]["Binds"].push("#{Settings.path.synctexBinHostPath}:/opt/synctex:ro")
if Settings.clsi.docker.seccomp_profile?
options.HostConfig.SecurityOpt.push "seccomp=#{Settings.clsi.docker.seccomp_profile}"
return options
_fingerprintContainer: (containerOptions) ->
# Yay, Hashing!
json = JSON.stringify(containerOptions)
return crypto.createHash("md5").update(json).digest("hex")
startContainer: (options, volumes, attachStreamHandler, callback) ->
LockManager.runWithLock options.name, (releaseLock) ->
# Check that volumes exist before starting the container.
# When a container is started with volume pointing to a
# non-existent directory then docker creates the directory but
# with root ownership.
DockerRunner._checkVolumes options, volumes, (err) ->
return releaseLock(err) if err?
DockerRunner._startContainer options, volumes, attachStreamHandler, releaseLock
, callback
# Check that volumes exist and are directories
_checkVolumes: (options, volumes, callback = (error, containerName) ->) ->
if usingSiblingContainers()
# Server Pro, with sibling-containers active, skip checks
return callback(null)
checkVolume = (path, cb) ->
fs.stat path, (err, stats) ->
return cb(err) if err?
return cb(DockerRunner.ERR_NOT_DIRECTORY) if not stats?.isDirectory()
cb()
jobs = []
for vol of volumes
do (vol) ->
jobs.push (cb) -> checkVolume(vol, cb)
async.series jobs, callback
_startContainer: (options, volumes, attachStreamHandler, callback = ((error, output) ->)) ->
callback = _.once(callback)
name = options.name
logger.log {container_name: name}, "starting container"
container = dockerode.getContainer(name)
createAndStartContainer = ->
dockerode.createContainer options, (error, container) ->
return callback(error) if error?
startExistingContainer()
startExistingContainer = ->
DockerRunner.attachToContainer options.name, attachStreamHandler, (error)->
return callback(error) if error?
container.start (error) ->
if error? and error?.statusCode != 304 #already running
return callback(error)
else
callback()
container.inspect (error, stats)->
if error?.statusCode == 404
createAndStartContainer()
else if error?
logger.err {container_name: name, error:error}, "unable to inspect container to start"
return callback(error)
else
startExistingContainer()
attachToContainer: (containerId, attachStreamHandler, attachStartCallback) ->
container = dockerode.getContainer(containerId)
container.attach {stdout: 1, stderr: 1, stream: 1}, (error, stream) ->
if error?
logger.error err: error, container_id: containerId, "error attaching to container"
return attachStartCallback(error)
else
attachStartCallback()
logger.log container_id: containerId, "attached to container"
MAX_OUTPUT = 1024 * 1024 # limit output to 1MB
createStringOutputStream = (name) ->
return {
data: ""
overflowed: false
write: (data) ->
return if @overflowed
if @data.length < MAX_OUTPUT
@data += data
else
logger.error container_id: containerId, length: @data.length, maxLen: MAX_OUTPUT, "#{name} exceeds max size"
@data += "(...truncated at #{MAX_OUTPUT} chars...)"
@overflowed = true
# kill container if too much output
# docker.containers.kill(containerId, () ->)
} }
stdout = createStringOutputStream "stdout" const volumes = {};
stderr = createStringOutputStream "stderr" volumes[directory] = "/compile";
container.modem.demuxStream(stream, stdout, stderr) command = (Array.from(command).map((arg) => __guardMethod__(arg.toString(), 'replace', o => o.replace('$COMPILE_DIR', "/compile"))));
if ((image == null)) {
({ image } = Settings.clsi.docker);
}
stream.on "error", (err) -> if (Settings.texliveImageNameOveride != null) {
logger.error err: err, container_id: containerId, "error reading from container stream" const img = image.split("/");
image = `${Settings.texliveImageNameOveride}/${img[2]}`;
}
stream.on "end", () -> const options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment);
attachStreamHandler null, {stdout: stdout.data, stderr: stderr.data} const fingerprint = DockerRunner._fingerprintContainer(options);
options.name = (name = `project-${project_id}-${fingerprint}`);
waitForContainer: (containerId, timeout, _callback = (error, exitCode) ->) -> // logOptions = _.clone(options)
callback = (args...) -> // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
_callback(args...) logger.log({project_id}, "running docker container");
# Only call the callback once DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(error, output) {
_callback = () -> if (__guard__(error != null ? error.message : undefined, x => x.match("HTTP code is 500"))) {
logger.log({err: error, project_id}, "error running container so destroying and retrying");
return DockerRunner.destroyContainer(name, null, true, function(error) {
if (error != null) { return callback(error); }
return DockerRunner._runAndWaitForContainer(options, volumes, timeout, callback);
});
} else {
return callback(error, output);
}
});
container = dockerode.getContainer(containerId) return name;
}, // pass back the container name to allow it to be killed
timedOut = false kill(container_id, callback) {
timeoutId = setTimeout () -> if (callback == null) { callback = function(error) {}; }
timedOut = true logger.log({container_id}, "sending kill signal to container");
logger.log container_id: containerId, "timeout reached, killing container" const container = dockerode.getContainer(container_id);
container.kill(() ->) return container.kill(function(error) {
, timeout if ((error != null) && __guardMethod__(error != null ? error.message : undefined, 'match', o => o.match(/Cannot kill container .* is not running/))) {
logger.warn({err: error, container_id}, "container not running, continuing");
error = null;
}
if (error != null) {
logger.error({err: error, container_id}, "error killing container");
return callback(error);
} else {
return callback();
}
});
},
logger.log container_id: containerId, "waiting for docker container" _runAndWaitForContainer(options, volumes, timeout, _callback) {
container.wait (error, res) -> if (_callback == null) { _callback = function(error, output) {}; }
if error? const callback = function(...args) {
clearTimeout timeoutId _callback(...Array.from(args || []));
logger.error err: error, container_id: containerId, "error waiting for container" // Only call the callback once
return callback(error) return _callback = function() {};
if timedOut };
logger.log containerId: containerId, "docker container timed out"
error = DockerRunner.ERR_TIMED_OUT
error.timedout = true
callback error
else
clearTimeout timeoutId
logger.log container_id: containerId, exitCode: res.StatusCode, "docker container returned"
callback null, res.StatusCode
destroyContainer: (containerName, containerId, shouldForce, callback = (error) ->) -> const { name } = options;
# We want the containerName for the lock and, ideally, the
# containerId to delete. There is a bug in the docker.io module
# where if you delete by name and there is an error, it throws an
# async exception, but if you delete by id it just does a normal
# error callback. We fall back to deleting by name if no id is
# supplied.
LockManager.runWithLock containerName, (releaseLock) ->
DockerRunner._destroyContainer containerId or containerName, shouldForce, releaseLock
, callback
_destroyContainer: (containerId, shouldForce, callback = (error) ->) -> let streamEnded = false;
logger.log container_id: containerId, "destroying docker container" let containerReturned = false;
container = dockerode.getContainer(containerId) let output = {};
container.remove {force: shouldForce == true}, (error) ->
if error? and error?.statusCode == 404
logger.warn err: error, container_id: containerId, "container not found, continuing"
error = null
if error?
logger.error err: error, container_id: containerId, "error destroying container"
else
logger.log container_id: containerId, "destroyed container"
callback(error)
# handle expiry of docker containers const callbackIfFinished = function() {
if (streamEnded && containerReturned) {
return callback(null, output);
}
};
MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge or oneHour = 60 * 60 * 1000 const attachStreamHandler = function(error, _output) {
if (error != null) { return callback(error); }
output = _output;
streamEnded = true;
return callbackIfFinished();
};
examineOldContainer: (container, callback = (error, name, id, ttl)->) -> return DockerRunner.startContainer(options, volumes, attachStreamHandler, function(error, containerId) {
name = container.Name or container.Names?[0] if (error != null) { return callback(error); }
created = container.Created * 1000 # creation time is returned in seconds
now = Date.now()
age = now - created
maxAge = DockerRunner.MAX_CONTAINER_AGE
ttl = maxAge - age
logger.log {containerName: name, created: created, now: now, age: age, maxAge: maxAge, ttl: ttl}, "checking whether to destroy container"
callback(null, name, container.Id, ttl)
destroyOldContainers: (callback = (error) ->) -> return DockerRunner.waitForContainer(name, timeout, function(error, exitCode) {
dockerode.listContainers all: true, (error, containers) -> let err;
return callback(error) if error? if (error != null) { return callback(error); }
jobs = [] if (exitCode === 137) { // exit status from kill -9
for container in containers or [] err = DockerRunner.ERR_TERMINATED;
do (container) -> err.terminated = true;
DockerRunner.examineOldContainer container, (err, name, id, ttl) -> return callback(err);
if name.slice(0, 9) == '/project-' && ttl <= 0 }
jobs.push (cb) -> if (exitCode === 1) { // exit status from chktex
DockerRunner.destroyContainer name, id, false, () -> cb() err = DockerRunner.ERR_EXITED;
# Ignore errors because some containers get stuck but err.code = exitCode;
# will be destroyed next time return callback(err);
async.series jobs, callback }
containerReturned = true;
__guard__(options != null ? options.HostConfig : undefined, x => x.SecurityOpt = null); //small log line
logger.log({err, exitCode, options}, "docker container has exited");
return callbackIfFinished();
});
});
},
startContainerMonitor: () -> _getContainerOptions(command, image, volumes, timeout, environment) {
logger.log {maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry" let m, year;
# randomise the start time let key, value, hostVol, dockerVol;
randomDelay = Math.floor(Math.random() * 5 * 60 * 1000) const timeoutInSeconds = timeout / 1000;
setTimeout () ->
setInterval () ->
DockerRunner.destroyOldContainers()
, oneHour = 60 * 60 * 1000
, randomDelay
DockerRunner.startContainerMonitor() const dockerVolumes = {};
for (hostVol in volumes) {
dockerVol = volumes[hostVol];
dockerVolumes[dockerVol] = {};
if (volumes[hostVol].slice(-3).indexOf(":r") === -1) {
volumes[hostVol] = `${dockerVol}:rw`;
}
}
// merge settings and environment parameter
const env = {};
for (let src of [Settings.clsi.docker.env, environment || {}]) {
for (key in src) { value = src[key]; env[key] = value; }
}
// set the path based on the image year
if ((m = image.match(/:([0-9]+)\.[0-9]+/))) {
year = m[1];
} else {
year = "2014";
}
env['PATH'] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`;
const options = {
"Cmd" : command,
"Image" : image,
"Volumes" : dockerVolumes,
"WorkingDir" : "/compile",
"NetworkDisabled" : true,
"Memory" : 1024 * 1024 * 1024 * 1024, // 1 Gb
"User" : Settings.clsi.docker.user,
"Env" : (((() => {
const result = [];
for (key in env) {
value = env[key];
result.push(`${key}=${value}`);
}
return result;
})())), // convert the environment hash to an array
"HostConfig" : {
"Binds": (((() => {
const result1 = [];
for (hostVol in volumes) {
dockerVol = volumes[hostVol];
result1.push(`${hostVol}:${dockerVol}`);
}
return result1;
})())),
"LogConfig": {"Type": "none", "Config": {}},
"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}],
"CapDrop": "ALL",
"SecurityOpt": ["no-new-privileges"]
}
};
if ((Settings.path != null ? Settings.path.synctexBinHostPath : undefined) != null) {
options["HostConfig"]["Binds"].push(`${Settings.path.synctexBinHostPath}:/opt/synctex:ro`);
}
if (Settings.clsi.docker.seccomp_profile != null) {
options.HostConfig.SecurityOpt.push(`seccomp=${Settings.clsi.docker.seccomp_profile}`);
}
return options;
},
_fingerprintContainer(containerOptions) {
// Yay, Hashing!
const json = JSON.stringify(containerOptions);
return crypto.createHash("md5").update(json).digest("hex");
},
startContainer(options, volumes, attachStreamHandler, callback) {
return LockManager.runWithLock(options.name, releaseLock =>
// Check that volumes exist before starting the container.
// When a container is started with volume pointing to a
// non-existent directory then docker creates the directory but
// with root ownership.
DockerRunner._checkVolumes(options, volumes, function(err) {
if (err != null) { return releaseLock(err); }
return DockerRunner._startContainer(options, volumes, attachStreamHandler, releaseLock);
})
, callback);
},
// Check that volumes exist and are directories
_checkVolumes(options, volumes, callback) {
if (callback == null) { callback = function(error, containerName) {}; }
if (usingSiblingContainers()) {
// Server Pro, with sibling-containers active, skip checks
return callback(null);
}
const checkVolume = (path, cb) =>
fs.stat(path, function(err, stats) {
if (err != null) { return cb(err); }
if (!(stats != null ? stats.isDirectory() : undefined)) { return cb(DockerRunner.ERR_NOT_DIRECTORY); }
return cb();
})
;
const jobs = [];
for (let vol in volumes) {
(vol => jobs.push(cb => checkVolume(vol, cb)))(vol);
}
return async.series(jobs, callback);
},
_startContainer(options, volumes, attachStreamHandler, callback) {
if (callback == null) { callback = function(error, output) {}; }
callback = _.once(callback);
const { name } = options;
logger.log({container_name: name}, "starting container");
const container = dockerode.getContainer(name);
const createAndStartContainer = () =>
dockerode.createContainer(options, function(error, container) {
if (error != null) { return callback(error); }
return startExistingContainer();
})
;
var startExistingContainer = () =>
DockerRunner.attachToContainer(options.name, attachStreamHandler, function(error){
if (error != null) { return callback(error); }
return container.start(function(error) {
if ((error != null) && ((error != null ? error.statusCode : undefined) !== 304)) { //already running
return callback(error);
} else {
return callback();
}
});
})
;
return container.inspect(function(error, stats){
if ((error != null ? error.statusCode : undefined) === 404) {
return createAndStartContainer();
} else if (error != null) {
logger.err({container_name: name, error}, "unable to inspect container to start");
return callback(error);
} else {
return startExistingContainer();
}
});
},
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
const container = dockerode.getContainer(containerId);
return container.attach({stdout: 1, stderr: 1, stream: 1}, function(error, stream) {
if (error != null) {
logger.error({err: error, container_id: containerId}, "error attaching to container");
return attachStartCallback(error);
} else {
attachStartCallback();
}
logger.log({container_id: containerId}, "attached to container");
const MAX_OUTPUT = 1024 * 1024; // limit output to 1MB
const createStringOutputStream = function(name) {
return {
data: "",
overflowed: false,
write(data) {
if (this.overflowed) { return; }
if (this.data.length < MAX_OUTPUT) {
return this.data += data;
} else {
logger.error({container_id: containerId, length: this.data.length, maxLen: MAX_OUTPUT}, `${name} exceeds max size`);
this.data += `(...truncated at ${MAX_OUTPUT} chars...)`;
return this.overflowed = true;
}
}
// kill container if too much output
// docker.containers.kill(containerId, () ->)
};
};
const stdout = createStringOutputStream("stdout");
const stderr = createStringOutputStream("stderr");
container.modem.demuxStream(stream, stdout, stderr);
stream.on("error", err => logger.error({err, container_id: containerId}, "error reading from container stream"));
return stream.on("end", () => attachStreamHandler(null, {stdout: stdout.data, stderr: stderr.data}));
});
},
waitForContainer(containerId, timeout, _callback) {
if (_callback == null) { _callback = function(error, exitCode) {}; }
const callback = function(...args) {
_callback(...Array.from(args || []));
// Only call the callback once
return _callback = function() {};
};
const container = dockerode.getContainer(containerId);
let timedOut = false;
const timeoutId = setTimeout(function() {
timedOut = true;
logger.log({container_id: containerId}, "timeout reached, killing container");
return container.kill(function() {});
}
, timeout);
logger.log({container_id: containerId}, "waiting for docker container");
return container.wait(function(error, res) {
if (error != null) {
clearTimeout(timeoutId);
logger.error({err: error, container_id: containerId}, "error waiting for container");
return callback(error);
}
if (timedOut) {
logger.log({containerId}, "docker container timed out");
error = DockerRunner.ERR_TIMED_OUT;
error.timedout = true;
return callback(error);
} else {
clearTimeout(timeoutId);
logger.log({container_id: containerId, exitCode: res.StatusCode}, "docker container returned");
return callback(null, res.StatusCode);
}
});
},
destroyContainer(containerName, containerId, shouldForce, callback) {
// We want the containerName for the lock and, ideally, the
// containerId to delete. There is a bug in the docker.io module
// where if you delete by name and there is an error, it throws an
// async exception, but if you delete by id it just does a normal
// error callback. We fall back to deleting by name if no id is
// supplied.
if (callback == null) { callback = function(error) {}; }
return LockManager.runWithLock(containerName, releaseLock => DockerRunner._destroyContainer(containerId || containerName, shouldForce, releaseLock)
, callback);
},
_destroyContainer(containerId, shouldForce, callback) {
if (callback == null) { callback = function(error) {}; }
logger.log({container_id: containerId}, "destroying docker container");
const container = dockerode.getContainer(containerId);
return container.remove({force: shouldForce === true}, function(error) {
if ((error != null) && ((error != null ? error.statusCode : undefined) === 404)) {
logger.warn({err: error, container_id: containerId}, "container not found, continuing");
error = null;
}
if (error != null) {
logger.error({err: error, container_id: containerId}, "error destroying container");
} else {
logger.log({container_id: containerId}, "destroyed container");
}
return callback(error);
});
},
// handle expiry of docker containers
MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000),
examineOldContainer(container, callback) {
if (callback == null) { callback = function(error, name, id, ttl){}; }
const name = container.Name || (container.Names != null ? container.Names[0] : undefined);
const created = container.Created * 1000; // creation time is returned in seconds
const now = Date.now();
const age = now - created;
const maxAge = DockerRunner.MAX_CONTAINER_AGE;
const ttl = maxAge - age;
logger.log({containerName: name, created, now, age, maxAge, ttl}, "checking whether to destroy container");
return callback(null, name, container.Id, ttl);
},
destroyOldContainers(callback) {
if (callback == null) { callback = function(error) {}; }
return dockerode.listContainers({all: true}, function(error, containers) {
if (error != null) { return callback(error); }
const jobs = [];
for (let container of Array.from(containers || [])) {
(container =>
DockerRunner.examineOldContainer(container, function(err, name, id, ttl) {
if ((name.slice(0, 9) === '/project-') && (ttl <= 0)) {
return jobs.push(cb => DockerRunner.destroyContainer(name, id, false, () => cb()));
}
})
)(container);
}
// Ignore errors because some containers get stuck but
// will be destroyed next time
return async.series(jobs, callback);
});
},
startContainerMonitor() {
logger.log({maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry");
// randomise the start time
const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000);
return setTimeout(() =>
setInterval(() => DockerRunner.destroyOldContainers()
, (oneHour = 60 * 60 * 1000))
, randomDelay);
}
});
DockerRunner.startContainerMonitor();
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}
function __guardMethod__(obj, methodName, transform) {
if (typeof obj !== 'undefined' && obj !== null && typeof obj[methodName] === 'function') {
return transform(obj, methodName);
} else {
return undefined;
}
}

51
app/coffee/DraftModeManager.js
View File

@@ -1,24 +1,37 @@
fs = require "fs" /*
logger = require "logger-sharelatex" * decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let DraftModeManager;
const fs = require("fs");
const logger = require("logger-sharelatex");
module.exports = DraftModeManager = module.exports = (DraftModeManager = {
injectDraftMode: (filename, callback = (error) ->) -> injectDraftMode(filename, callback) {
fs.readFile filename, "utf8", (error, content) -> if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return fs.readFile(filename, "utf8", function(error, content) {
# avoid adding draft mode more than once if (error != null) { return callback(error); }
if content?.indexOf("\\documentclass\[draft") >= 0 // avoid adding draft mode more than once
return callback() if ((content != null ? content.indexOf("\\documentclass\[draft") : undefined) >= 0) {
modified_content = DraftModeManager._injectDraftOption content return callback();
logger.log { }
content: content.slice(0,1024), # \documentclass is normally v near the top const modified_content = DraftModeManager._injectDraftOption(content);
logger.log({
content: content.slice(0,1024), // \documentclass is normally v near the top
modified_content: modified_content.slice(0,1024), modified_content: modified_content.slice(0,1024),
filename filename
}, "injected draft class" }, "injected draft class");
fs.writeFile filename, modified_content, callback return fs.writeFile(filename, modified_content, callback);
});
},
_injectDraftOption: (content) -> _injectDraftOption(content) {
content return content
# With existing options (must be first, otherwise both are applied) // With existing options (must be first, otherwise both are applied)
.replace(/\\documentclass\[/g, "\\documentclass[draft,") .replace(/\\documentclass\[/g, "\\documentclass[draft,")
# Without existing options // Without existing options
.replace(/\\documentclass\{/g, "\\documentclass[draft]{") .replace(/\\documentclass\{/g, "\\documentclass[draft]{");
}
});

49
app/coffee/Errors.js
View File

@@ -1,25 +1,30 @@
NotFoundError = (message) -> let Errors;
error = new Error(message) var NotFoundError = function(message) {
error.name = "NotFoundError" const error = new Error(message);
error.__proto__ = NotFoundError.prototype error.name = "NotFoundError";
return error error.__proto__ = NotFoundError.prototype;
NotFoundError.prototype.__proto__ = Error.prototype return error;
};
NotFoundError.prototype.__proto__ = Error.prototype;
FilesOutOfSyncError = (message) -> var FilesOutOfSyncError = function(message) {
error = new Error(message) const error = new Error(message);
error.name = "FilesOutOfSyncError" error.name = "FilesOutOfSyncError";
error.__proto__ = FilesOutOfSyncError.prototype error.__proto__ = FilesOutOfSyncError.prototype;
return error return error;
FilesOutOfSyncError.prototype.__proto__ = Error.prototype };
FilesOutOfSyncError.prototype.__proto__ = Error.prototype;
AlreadyCompilingError = (message) -> var AlreadyCompilingError = function(message) {
error = new Error(message) const error = new Error(message);
error.name = "AlreadyCompilingError" error.name = "AlreadyCompilingError";
error.__proto__ = AlreadyCompilingError.prototype error.__proto__ = AlreadyCompilingError.prototype;
return error return error;
AlreadyCompilingError.prototype.__proto__ = Error.prototype };
AlreadyCompilingError.prototype.__proto__ = Error.prototype;
module.exports = Errors = module.exports = (Errors = {
NotFoundError: NotFoundError NotFoundError,
FilesOutOfSyncError: FilesOutOfSyncError FilesOutOfSyncError,
AlreadyCompilingError: AlreadyCompilingError AlreadyCompilingError
});

172
app/coffee/LatexRunner.js
View File

@@ -1,95 +1,123 @@
Path = require "path" /*
Settings = require "settings-sharelatex" * decaffeinate suggestions:
logger = require "logger-sharelatex" * DS102: Remove unnecessary code created because of implicit returns
Metrics = require "./Metrics" * DS103: Rewrite code to no longer use __guard__
CommandRunner = require "./CommandRunner" * DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let LatexRunner;
const Path = require("path");
const Settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const Metrics = require("./Metrics");
const CommandRunner = require("./CommandRunner");
ProcessTable = {} # table of currently running jobs (pids or docker container names) const ProcessTable = {}; // table of currently running jobs (pids or docker container names)
module.exports = LatexRunner = module.exports = (LatexRunner = {
runLatex: (project_id, options, callback = (error) ->) -> runLatex(project_id, options, callback) {
{directory, mainFile, compiler, timeout, image, environment, flags} = options let command;
compiler ||= "pdflatex" if (callback == null) { callback = function(error) {}; }
timeout ||= 60000 # milliseconds let {directory, mainFile, compiler, timeout, image, environment, flags} = options;
if (!compiler) { compiler = "pdflatex"; }
if (!timeout) { timeout = 60000; } // milliseconds
logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, environment: environment, flags:flags, "starting compile" logger.log({directory, compiler, timeout, mainFile, environment, flags}, "starting compile");
# We want to run latexmk on the tex file which we will automatically // We want to run latexmk on the tex file which we will automatically
# generate from the Rtex/Rmd/md file. // generate from the Rtex/Rmd/md file.
mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex") mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex");
if compiler == "pdflatex" if (compiler === "pdflatex") {
command = LatexRunner._pdflatexCommand mainFile, flags command = LatexRunner._pdflatexCommand(mainFile, flags);
else if compiler == "latex" } else if (compiler === "latex") {
command = LatexRunner._latexCommand mainFile, flags command = LatexRunner._latexCommand(mainFile, flags);
else if compiler == "xelatex" } else if (compiler === "xelatex") {
command = LatexRunner._xelatexCommand mainFile, flags command = LatexRunner._xelatexCommand(mainFile, flags);
else if compiler == "lualatex" } else if (compiler === "lualatex") {
command = LatexRunner._lualatexCommand mainFile, flags command = LatexRunner._lualatexCommand(mainFile, flags);
else } else {
return callback new Error("unknown compiler: #{compiler}") return callback(new Error(`unknown compiler: ${compiler}`));
}
if Settings.clsi?.strace if (Settings.clsi != null ? Settings.clsi.strace : undefined) {
command = ["strace", "-o", "strace", "-ff"].concat(command) command = ["strace", "-o", "strace", "-ff"].concat(command);
}
id = "#{project_id}" # record running project under this id const id = `${project_id}`; // record running project under this id
ProcessTable[id] = CommandRunner.run project_id, command, directory, image, timeout, environment, (error, output) -> return ProcessTable[id] = CommandRunner.run(project_id, command, directory, image, timeout, environment, function(error, output) {
delete ProcessTable[id] delete ProcessTable[id];
return callback(error) if error? if (error != null) { return callback(error); }
runs = output?.stderr?.match(/^Run number \d+ of .*latex/mg)?.length or 0 const runs = __guard__(__guard__(output != null ? output.stderr : undefined, x1 => x1.match(/^Run number \d+ of .*latex/mg)), x => x.length) || 0;
failed = if output?.stdout?.match(/^Latexmk: Errors/m)? then 1 else 0 const failed = (__guard__(output != null ? output.stdout : undefined, x2 => x2.match(/^Latexmk: Errors/m)) != null) ? 1 : 0;
# counters from latexmk output // counters from latexmk output
stats = {} const stats = {};
stats["latexmk-errors"] = failed stats["latexmk-errors"] = failed;
stats["latex-runs"] = runs stats["latex-runs"] = runs;
stats["latex-runs-with-errors"] = if failed then runs else 0 stats["latex-runs-with-errors"] = failed ? runs : 0;
stats["latex-runs-#{runs}"] = 1 stats[`latex-runs-${runs}`] = 1;
stats["latex-runs-with-errors-#{runs}"] = if failed then 1 else 0 stats[`latex-runs-with-errors-${runs}`] = failed ? 1 : 0;
# timing information from /usr/bin/time // timing information from /usr/bin/time
timings = {} const timings = {};
stderr = output?.stderr const stderr = output != null ? output.stderr : undefined;
timings["cpu-percent"] = stderr?.match(/Percent of CPU this job got: (\d+)/m)?[1] or 0 timings["cpu-percent"] = __guard__(stderr != null ? stderr.match(/Percent of CPU this job got: (\d+)/m) : undefined, x3 => x3[1]) || 0;
timings["cpu-time"] = stderr?.match(/User time.*: (\d+.\d+)/m)?[1] or 0 timings["cpu-time"] = __guard__(stderr != null ? stderr.match(/User time.*: (\d+.\d+)/m) : undefined, x4 => x4[1]) || 0;
timings["sys-time"] = stderr?.match(/System time.*: (\d+.\d+)/m)?[1] or 0 timings["sys-time"] = __guard__(stderr != null ? stderr.match(/System time.*: (\d+.\d+)/m) : undefined, x5 => x5[1]) || 0;
callback error, output, stats, timings return callback(error, output, stats, timings);
});
},
killLatex: (project_id, callback = (error) ->) -> killLatex(project_id, callback) {
id = "#{project_id}" if (callback == null) { callback = function(error) {}; }
logger.log {id:id}, "killing running compile" const id = `${project_id}`;
if not ProcessTable[id]? logger.log({id}, "killing running compile");
logger.warn {id}, "no such project to kill" if ((ProcessTable[id] == null)) {
return callback(null) logger.warn({id}, "no such project to kill");
else return callback(null);
CommandRunner.kill ProcessTable[id], callback } else {
return CommandRunner.kill(ProcessTable[id], callback);
}
},
_latexmkBaseCommand: (flags) -> _latexmkBaseCommand(flags) {
args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"] let args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"];
if flags if (flags) {
args = args.concat(flags) args = args.concat(flags);
(Settings?.clsi?.latexmkCommandPrefix || []).concat(args) }
return (__guard__(Settings != null ? Settings.clsi : undefined, x => x.latexmkCommandPrefix) || []).concat(args);
},
_pdflatexCommand: (mainFile, flags) -> _pdflatexCommand(mainFile, flags) {
LatexRunner._latexmkBaseCommand(flags).concat [ return LatexRunner._latexmkBaseCommand(flags).concat([
"-pdf", "-pdf",
Path.join("$COMPILE_DIR", mainFile) Path.join("$COMPILE_DIR", mainFile)
] ]);
},
_latexCommand: (mainFile, flags) -> _latexCommand(mainFile, flags) {
LatexRunner._latexmkBaseCommand(flags).concat [ return LatexRunner._latexmkBaseCommand(flags).concat([
"-pdfdvi", "-pdfdvi",
Path.join("$COMPILE_DIR", mainFile) Path.join("$COMPILE_DIR", mainFile)
] ]);
},
_xelatexCommand: (mainFile, flags) -> _xelatexCommand(mainFile, flags) {
LatexRunner._latexmkBaseCommand(flags).concat [ return LatexRunner._latexmkBaseCommand(flags).concat([
"-xelatex", "-xelatex",
Path.join("$COMPILE_DIR", mainFile) Path.join("$COMPILE_DIR", mainFile)
] ]);
},
_lualatexCommand: (mainFile, flags) -> _lualatexCommand(mainFile, flags) {
LatexRunner._latexmkBaseCommand(flags).concat [ return LatexRunner._latexmkBaseCommand(flags).concat([
"-lualatex", "-lualatex",
Path.join("$COMPILE_DIR", mainFile) Path.join("$COMPILE_DIR", mainFile)
] ]);
}
});
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}

96
app/coffee/LocalCommandRunner.js
View File

@@ -1,48 +1,66 @@
spawn = require("child_process").spawn /*
logger = require "logger-sharelatex" * decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let CommandRunner;
const { spawn } = require("child_process");
const logger = require("logger-sharelatex");
logger.info "using standard command runner" logger.info("using standard command runner");
module.exports = CommandRunner = module.exports = (CommandRunner = {
run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) -> run(project_id, command, directory, image, timeout, environment, callback) {
command = (arg.toString().replace('$COMPILE_DIR', directory) for arg in command) let key, value;
logger.log project_id: project_id, command: command, directory: directory, "running command" if (callback == null) { callback = function(error) {}; }
logger.warn "timeouts and sandboxing are not enabled with CommandRunner" command = (Array.from(command).map((arg) => arg.toString().replace('$COMPILE_DIR', directory)));
logger.log({project_id, command, directory}, "running command");
logger.warn("timeouts and sandboxing are not enabled with CommandRunner");
# merge environment settings // merge environment settings
env = {} const env = {};
env[key] = value for key, value of process.env for (key in process.env) { value = process.env[key]; env[key] = value; }
env[key] = value for key, value of environment for (key in environment) { value = environment[key]; env[key] = value; }
# run command as detached process so it has its own process group (which can be killed if needed) // run command as detached process so it has its own process group (which can be killed if needed)
proc = spawn command[0], command.slice(1), cwd: directory, env: env const proc = spawn(command[0], command.slice(1), {cwd: directory, env});
stdout = "" let stdout = "";
proc.stdout.on "data", (data)-> proc.stdout.on("data", data=> stdout += data);
stdout += data
proc.on "error", (err)-> proc.on("error", function(err){
logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command" logger.err({err, project_id, command, directory}, "error running command");
callback(err) return callback(err);
});
proc.on "close", (code, signal) -> proc.on("close", function(code, signal) {
logger.info code:code, signal:signal, project_id:project_id, "command exited" let err;
if signal is 'SIGTERM' # signal from kill method below logger.info({code, signal, project_id}, "command exited");
err = new Error("terminated") if (signal === 'SIGTERM') { // signal from kill method below
err.terminated = true err = new Error("terminated");
return callback(err) err.terminated = true;
else if code is 1 # exit status from chktex return callback(err);
err = new Error("exited") } else if (code === 1) { // exit status from chktex
err.code = code err = new Error("exited");
return callback(err) err.code = code;
else return callback(err);
callback(null, {"stdout": stdout}) } else {
return callback(null, {"stdout": stdout});
}
});
return proc.pid # return process id to allow job to be killed if necessary return proc.pid;
}, // return process id to allow job to be killed if necessary
kill: (pid, callback = (error) ->) -> kill(pid, callback) {
try if (callback == null) { callback = function(error) {}; }
process.kill -pid # kill all processes in group try {
catch err process.kill(-pid); // kill all processes in group
return callback(err) } catch (err) {
callback() return callback(err);
}
return callback();
}
});

79
app/coffee/LockManager.js
View File

@@ -1,31 +1,50 @@
Settings = require('settings-sharelatex') /*
logger = require "logger-sharelatex" * decaffeinate suggestions:
Lockfile = require('lockfile') # from https://github.com/npm/lockfile * DS101: Remove unnecessary use of Array.from
Errors = require "./Errors" * DS102: Remove unnecessary code created because of implicit returns
fs = require("fs") * DS207: Consider shorter variations of null checks
Path = require("path") * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
module.exports = LockManager = */
LOCK_TEST_INTERVAL: 1000 # 50ms between each test of the lock let LockManager;
MAX_LOCK_WAIT_TIME: 15000 # 10s maximum time to spend trying to get the lock const Settings = require('settings-sharelatex');
LOCK_STALE: 5*60*1000 # 5 mins time until lock auto expires const logger = require("logger-sharelatex");
const Lockfile = require('lockfile'); // from https://github.com/npm/lockfile
const Errors = require("./Errors");
const fs = require("fs");
const Path = require("path");
module.exports = (LockManager = {
LOCK_TEST_INTERVAL: 1000, // 50ms between each test of the lock
MAX_LOCK_WAIT_TIME: 15000, // 10s maximum time to spend trying to get the lock
LOCK_STALE: 5*60*1000, // 5 mins time until lock auto expires
runWithLock: (path, runner, callback = ((error) ->)) -> runWithLock(path, runner, callback) {
lockOpts = if (callback == null) { callback = function(error) {}; }
wait: @MAX_LOCK_WAIT_TIME const lockOpts = {
pollPeriod: @LOCK_TEST_INTERVAL wait: this.MAX_LOCK_WAIT_TIME,
stale: @LOCK_STALE pollPeriod: this.LOCK_TEST_INTERVAL,
Lockfile.lock path, lockOpts, (error) -> stale: this.LOCK_STALE
if error?.code is 'EEXIST' };
return callback new Errors.AlreadyCompilingError("compile in progress") return Lockfile.lock(path, lockOpts, function(error) {
else if error? if ((error != null ? error.code : undefined) === 'EEXIST') {
fs.lstat path, (statLockErr, statLock)-> return callback(new Errors.AlreadyCompilingError("compile in progress"));
fs.lstat Path.dirname(path), (statDirErr, statDir)-> } else if (error != null) {
fs.readdir Path.dirname(path), (readdirErr, readdirDir)-> return fs.lstat(path, (statLockErr, statLock)=>
logger.err error:error, path:path, statLock:statLock, statLockErr:statLockErr, statDir:statDir, statDirErr: statDirErr, readdirErr:readdirErr, readdirDir:readdirDir, "unable to get lock" fs.lstat(Path.dirname(path), (statDirErr, statDir)=>
return callback(error) fs.readdir(Path.dirname(path), function(readdirErr, readdirDir){
else logger.err({error, path, statLock, statLockErr, statDir, statDirErr, readdirErr, readdirDir}, "unable to get lock");
runner (error1, args...) -> return callback(error);
Lockfile.unlock path, (error2) -> })
error = error1 or error2 )
return callback(error) if error? );
callback(null, args...) } else {
return runner((error1, ...args) =>
Lockfile.unlock(path, function(error2) {
error = error1 || error2;
if (error != null) { return callback(error); }
return callback(null, ...Array.from(args));
})
);
}
});
}
});

2
app/coffee/Metrics.js
View File

@@ -1,2 +1,2 @@
module.exports = require "metrics-sharelatex" module.exports = require("metrics-sharelatex");

429
app/coffee/OutputCacheManager.js
View File

@@ -1,199 +1,270 @@
async = require "async" /*
fs = require "fs" * decaffeinate suggestions:
fse = require "fs-extra" * DS101: Remove unnecessary use of Array.from
Path = require "path" * DS102: Remove unnecessary code created because of implicit returns
logger = require "logger-sharelatex" * DS103: Rewrite code to no longer use __guard__
_ = require "underscore" * DS104: Avoid inline assignments
Settings = require "settings-sharelatex" * DS204: Change includes calls to have a more natural evaluation order
crypto = require "crypto" * DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputCacheManager;
const async = require("async");
const fs = require("fs");
const fse = require("fs-extra");
const Path = require("path");
const logger = require("logger-sharelatex");
const _ = require("underscore");
const Settings = require("settings-sharelatex");
const crypto = require("crypto");
OutputFileOptimiser = require "./OutputFileOptimiser" const OutputFileOptimiser = require("./OutputFileOptimiser");
module.exports = OutputCacheManager = module.exports = (OutputCacheManager = {
CACHE_SUBDIR: '.cache/clsi' CACHE_SUBDIR: '.cache/clsi',
ARCHIVE_SUBDIR: '.archive/clsi' ARCHIVE_SUBDIR: '.archive/clsi',
# build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes // build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
# for backwards compatibility, make the randombytes part optional // for backwards compatibility, make the randombytes part optional
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/ BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
CACHE_LIMIT: 2 # maximum number of cache directories CACHE_LIMIT: 2, // maximum number of cache directories
CACHE_AGE: 60*60*1000 # up to one hour old CACHE_AGE: 60*60*1000, // up to one hour old
path: (buildId, file) -> path(buildId, file) {
# used by static server, given build id return '.cache/clsi/buildId' // used by static server, given build id return '.cache/clsi/buildId'
if buildId.match OutputCacheManager.BUILD_REGEX if (buildId.match(OutputCacheManager.BUILD_REGEX)) {
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file) return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file);
else } else {
# for invalid build id, return top level // for invalid build id, return top level
return file return file;
}
},
generateBuildId: (callback = (error, buildId) ->) -> generateBuildId(callback) {
# generate a secure build id from Date.now() and 8 random bytes in hex // generate a secure build id from Date.now() and 8 random bytes in hex
crypto.randomBytes 8, (err, buf) -> if (callback == null) { callback = function(error, buildId) {}; }
return callback(err) if err? return crypto.randomBytes(8, function(err, buf) {
random = buf.toString('hex') if (err != null) { return callback(err); }
date = Date.now().toString(16) const random = buf.toString('hex');
callback err, "#{date}-#{random}" const date = Date.now().toString(16);
return callback(err, `${date}-${random}`);
});
},
saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) -> saveOutputFiles(outputFiles, compileDir, callback) {
OutputCacheManager.generateBuildId (err, buildId) -> if (callback == null) { callback = function(error) {}; }
return callback(err) if err? return OutputCacheManager.generateBuildId(function(err, buildId) {
OutputCacheManager.saveOutputFilesInBuildDir outputFiles, compileDir, buildId, callback if (err != null) { return callback(err); }
return OutputCacheManager.saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback);
});
},
saveOutputFilesInBuildDir: (outputFiles, compileDir, buildId, callback = (error) ->) -> saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback) {
# make a compileDir/CACHE_SUBDIR/build_id directory and // make a compileDir/CACHE_SUBDIR/build_id directory and
# copy all the output files into it // copy all the output files into it
cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR) if (callback == null) { callback = function(error) {}; }
# Put the files into a new cache subdirectory const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR);
cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId) // Put the files into a new cache subdirectory
# Is it a per-user compile? check if compile directory is PROJECTID-USERID const cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId);
perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/) // Is it a per-user compile? check if compile directory is PROJECTID-USERID
const perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/);
# Archive logs in background // Archive logs in background
if Settings.clsi?.archive_logs or Settings.clsi?.strace if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) || (Settings.clsi != null ? Settings.clsi.strace : undefined)) {
OutputCacheManager.archiveLogs outputFiles, compileDir, buildId, (err) -> OutputCacheManager.archiveLogs(outputFiles, compileDir, buildId, function(err) {
if err? if (err != null) {
logger.warn err:err, "erroring archiving log files" return logger.warn({err}, "erroring archiving log files");
}
});
}
# make the new cache directory // make the new cache directory
fse.ensureDir cacheDir, (err) -> return fse.ensureDir(cacheDir, function(err) {
if err? if (err != null) {
logger.error err: err, directory: cacheDir, "error creating cache directory" logger.error({err, directory: cacheDir}, "error creating cache directory");
callback(err, outputFiles) return callback(err, outputFiles);
else } else {
# copy all the output files into the new cache directory // copy all the output files into the new cache directory
results = [] const results = [];
async.mapSeries outputFiles, (file, cb) -> return async.mapSeries(outputFiles, function(file, cb) {
# don't send dot files as output, express doesn't serve them // don't send dot files as output, express doesn't serve them
if OutputCacheManager._fileIsHidden(file.path) if (OutputCacheManager._fileIsHidden(file.path)) {
logger.debug compileDir: compileDir, path: file.path, "ignoring dotfile in output" logger.debug({compileDir, path: file.path}, "ignoring dotfile in output");
return cb() return cb();
# copy other files into cache directory if valid }
newFile = _.clone(file) // copy other files into cache directory if valid
[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)] const newFile = _.clone(file);
OutputCacheManager._checkFileIsSafe src, (err, isSafe) -> const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]);
return cb(err) if err? return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) {
if !isSafe if (err != null) { return cb(err); }
return cb() if (!isSafe) {
OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) -> return cb();
return cb(err) if err? }
if !shouldCopy return OutputCacheManager._checkIfShouldCopy(src, function(err, shouldCopy) {
return cb() if (err != null) { return cb(err); }
OutputCacheManager._copyFile src, dst, (err) -> if (!shouldCopy) {
return cb(err) if err? return cb();
newFile.build = buildId # attach a build id if we cached the file }
results.push newFile return OutputCacheManager._copyFile(src, dst, function(err) {
cb() if (err != null) { return cb(err); }
, (err) -> newFile.build = buildId; // attach a build id if we cached the file
if err? results.push(newFile);
# pass back the original files if we encountered *any* error return cb();
callback(err, outputFiles) });
# clean up the directory we just created });
fse.remove cacheDir, (err) -> });
if err? }
logger.error err: err, dir: cacheDir, "error removing cache dir after failure" , function(err) {
else if (err != null) {
# pass back the list of new files in the cache // pass back the original files if we encountered *any* error
callback(err, results) callback(err, outputFiles);
# let file expiry run in the background, expire all previous files if per-user // clean up the directory we just created
OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 1 else null} return fse.remove(cacheDir, function(err) {
if (err != null) {
return logger.error({err, dir: cacheDir}, "error removing cache dir after failure");
}
});
} else {
// pass back the list of new files in the cache
callback(err, results);
// let file expiry run in the background, expire all previous files if per-user
return OutputCacheManager.expireOutputFiles(cacheRoot, {keep: buildId, limit: perUser ? 1 : null});
}
});
}
});
},
archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) -> archiveLogs(outputFiles, compileDir, buildId, callback) {
archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId) if (callback == null) { callback = function(error) {}; }
logger.log {dir: archiveDir}, "archiving log files for project" const archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId);
fse.ensureDir archiveDir, (err) -> logger.log({dir: archiveDir}, "archiving log files for project");
return callback(err) if err? return fse.ensureDir(archiveDir, function(err) {
async.mapSeries outputFiles, (file, cb) -> if (err != null) { return callback(err); }
[src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)] return async.mapSeries(outputFiles, function(file, cb) {
OutputCacheManager._checkFileIsSafe src, (err, isSafe) -> const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]);
return cb(err) if err? return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) {
return cb() if !isSafe if (err != null) { return cb(err); }
OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) -> if (!isSafe) { return cb(); }
return cb(err) if err? return OutputCacheManager._checkIfShouldArchive(src, function(err, shouldArchive) {
return cb() if !shouldArchive if (err != null) { return cb(err); }
OutputCacheManager._copyFile src, dst, cb if (!shouldArchive) { return cb(); }
, callback return OutputCacheManager._copyFile(src, dst, cb);
});
});
}
, callback);
});
},
expireOutputFiles: (cacheRoot, options, callback = (error) ->) -> expireOutputFiles(cacheRoot, options, callback) {
# look in compileDir for build dirs and delete if > N or age of mod time > T // look in compileDir for build dirs and delete if > N or age of mod time > T
fs.readdir cacheRoot, (err, results) -> if (callback == null) { callback = function(error) {}; }
if err? return fs.readdir(cacheRoot, function(err, results) {
return callback(null) if err.code == 'ENOENT' # cache directory is empty if (err != null) {
logger.error err: err, project_id: cacheRoot, "error clearing cache" if (err.code === 'ENOENT') { return callback(null); } // cache directory is empty
return callback(err) logger.error({err, project_id: cacheRoot}, "error clearing cache");
return callback(err);
}
dirs = results.sort().reverse() const dirs = results.sort().reverse();
currentTime = Date.now() const currentTime = Date.now();
isExpired = (dir, index) -> const isExpired = function(dir, index) {
return false if options?.keep == dir if ((options != null ? options.keep : undefined) === dir) { return false; }
# remove any directories over the requested (non-null) limit // remove any directories over the requested (non-null) limit
return true if options?.limit? and index > options.limit if (((options != null ? options.limit : undefined) != null) && (index > options.limit)) { return true; }
# remove any directories over the hard limit // remove any directories over the hard limit
return true if index > OutputCacheManager.CACHE_LIMIT if (index > OutputCacheManager.CACHE_LIMIT) { return true; }
# we can get the build time from the first part of the directory name DDDD-RRRR // we can get the build time from the first part of the directory name DDDD-RRRR
# DDDD is date and RRRR is random bytes // DDDD is date and RRRR is random bytes
dirTime = parseInt(dir.split('-')?[0], 16) const dirTime = parseInt(__guard__(dir.split('-'), x => x[0]), 16);
age = currentTime - dirTime const age = currentTime - dirTime;
return age > OutputCacheManager.CACHE_AGE return age > OutputCacheManager.CACHE_AGE;
};
toRemove = _.filter(dirs, isExpired) const toRemove = _.filter(dirs, isExpired);
removeDir = (dir, cb) -> const removeDir = (dir, cb) =>
fse.remove Path.join(cacheRoot, dir), (err, result) -> fse.remove(Path.join(cacheRoot, dir), function(err, result) {
logger.log cache: cacheRoot, dir: dir, "removed expired cache dir" logger.log({cache: cacheRoot, dir}, "removed expired cache dir");
if err? if (err != null) {
logger.error err: err, dir: dir, "cache remove error" logger.error({err, dir}, "cache remove error");
cb(err, result) }
return cb(err, result);
})
;
async.eachSeries toRemove, (dir, cb) -> return async.eachSeries(toRemove, (dir, cb) => removeDir(dir, cb)
removeDir dir, cb , callback);
, callback });
},
_fileIsHidden: (path) -> _fileIsHidden(path) {
return path?.match(/^\.|\/\./)? return ((path != null ? path.match(/^\.|\/\./) : undefined) != null);
},
_checkFileIsSafe: (src, callback = (error, isSafe) ->) -> _checkFileIsSafe(src, callback) {
# check if we have a valid file to copy into the cache // check if we have a valid file to copy into the cache
fs.stat src, (err, stats) -> if (callback == null) { callback = function(error, isSafe) {}; }
if err?.code is 'ENOENT' return fs.stat(src, function(err, stats) {
logger.warn err: err, file: src, "file has disappeared before copying to build cache" if ((err != null ? err.code : undefined) === 'ENOENT') {
callback(err, false) logger.warn({err, file: src}, "file has disappeared before copying to build cache");
else if err? return callback(err, false);
# some other problem reading the file } else if (err != null) {
logger.error err: err, file: src, "stat error for file in cache" // some other problem reading the file
callback(err, false) logger.error({err, file: src}, "stat error for file in cache");
else if not stats.isFile() return callback(err, false);
# other filetype - reject it } else if (!stats.isFile()) {
logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache" // other filetype - reject it
callback(null, false) logger.warn({src, stat: stats}, "nonfile output - refusing to copy to cache");
else return callback(null, false);
# it's a plain file, ok to copy } else {
callback(null, true) // it's a plain file, ok to copy
return callback(null, true);
}
});
},
_copyFile: (src, dst, callback) -> _copyFile(src, dst, callback) {
# copy output file into the cache // copy output file into the cache
fse.copy src, dst, (err) -> return fse.copy(src, dst, function(err) {
if err?.code is 'ENOENT' if ((err != null ? err.code : undefined) === 'ENOENT') {
logger.warn err: err, file: src, "file has disappeared when copying to build cache" logger.warn({err, file: src}, "file has disappeared when copying to build cache");
callback(err, false) return callback(err, false);
else if err? } else if (err != null) {
logger.error err: err, src: src, dst: dst, "copy error for file in cache" logger.error({err, src, dst}, "copy error for file in cache");
callback(err) return callback(err);
else } else {
if Settings.clsi?.optimiseInDocker if ((Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined)) {
# don't run any optimisations on the pdf when they are done // don't run any optimisations on the pdf when they are done
# in the docker container // in the docker container
callback() return callback();
else } else {
# call the optimiser for the file too // call the optimiser for the file too
OutputFileOptimiser.optimiseFile src, dst, callback return OutputFileOptimiser.optimiseFile(src, dst, callback);
}
}
});
},
_checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) -> _checkIfShouldCopy(src, callback) {
return callback(null, !Path.basename(src).match(/^strace/)) if (callback == null) { callback = function(err, shouldCopy) {}; }
return callback(null, !Path.basename(src).match(/^strace/));
},
_checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) -> _checkIfShouldArchive(src, callback) {
if Path.basename(src).match(/^strace/) let needle;
return callback(null, true) if (callback == null) { callback = function(err, shouldCopy) {}; }
if Settings.clsi?.archive_logs and Path.basename(src) in ["output.log", "output.blg"] if (Path.basename(src).match(/^strace/)) {
return callback(null, true) return callback(null, true);
return callback(null, false) }
if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) && (needle = Path.basename(src), ["output.log", "output.blg"].includes(needle))) {
return callback(null, true);
}
return callback(null, false);
}
});
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}

116
app/coffee/OutputFileFinder.js
View File

@@ -1,50 +1,78 @@
async = require "async" /*
fs = require "fs" * decaffeinate suggestions:
Path = require "path" * DS101: Remove unnecessary use of Array.from
spawn = require("child_process").spawn * DS102: Remove unnecessary code created because of implicit returns
logger = require "logger-sharelatex" * DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputFileFinder;
const async = require("async");
const fs = require("fs");
const Path = require("path");
const { spawn } = require("child_process");
const logger = require("logger-sharelatex");
module.exports = OutputFileFinder = module.exports = (OutputFileFinder = {
findOutputFiles: (resources, directory, callback = (error, outputFiles, allFiles) ->) -> findOutputFiles(resources, directory, callback) {
incomingResources = {} if (callback == null) { callback = function(error, outputFiles, allFiles) {}; }
for resource in resources const incomingResources = {};
incomingResources[resource.path] = true for (let resource of Array.from(resources)) {
incomingResources[resource.path] = true;
OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
if error?
logger.err err:error, "error finding all output files"
return callback(error)
outputFiles = []
for file in allFiles
if !incomingResources[file]
outputFiles.push {
path: file
type: file.match(/\.([^\.]+)$/)?[1]
} }
callback null, outputFiles, allFiles
_getAllFiles: (directory, _callback = (error, fileList) ->) -> return OutputFileFinder._getAllFiles(directory, function(error, allFiles) {
callback = (error, fileList) -> if (allFiles == null) { allFiles = []; }
_callback(error, fileList) if (error != null) {
_callback = () -> logger.err({err:error}, "error finding all output files");
return callback(error);
}
const outputFiles = [];
for (let file of Array.from(allFiles)) {
if (!incomingResources[file]) {
outputFiles.push({
path: file,
type: __guard__(file.match(/\.([^\.]+)$/), x => x[1])
});
}
}
return callback(null, outputFiles, allFiles);
});
},
# don't include clsi-specific files/directories in the output list _getAllFiles(directory, _callback) {
EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"] if (_callback == null) { _callback = function(error, fileList) {}; }
args = [directory, "(", EXCLUDE_DIRS..., ")", "-prune", "-o", "-type", "f", "-print"] const callback = function(error, fileList) {
logger.log args: args, "running find command" _callback(error, fileList);
return _callback = function() {};
};
proc = spawn("find", args) // don't include clsi-specific files/directories in the output list
stdout = "" const EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"];
proc.stdout.on "data", (chunk) -> const args = [directory, "(", ...Array.from(EXCLUDE_DIRS), ")", "-prune", "-o", "-type", "f", "-print"];
stdout += chunk.toString() logger.log({args}, "running find command");
proc.on "error", callback
proc.on "close", (code) ->
if code != 0
logger.warn {directory, code}, "find returned error, directory likely doesn't exist"
return callback null, []
fileList = stdout.trim().split("\n")
fileList = fileList.map (file) ->
# Strip leading directory
path = Path.relative(directory, file)
return callback null, fileList
const proc = spawn("find", args);
let stdout = "";
proc.stdout.on("data", chunk => stdout += chunk.toString());
proc.on("error", callback);
return proc.on("close", function(code) {
if (code !== 0) {
logger.warn({directory, code}, "find returned error, directory likely doesn't exist");
return callback(null, []);
}
let fileList = stdout.trim().split("\n");
fileList = fileList.map(function(file) {
// Strip leading directory
let path;
return path = Path.relative(directory, file);
});
return callback(null, fileList);
});
}
});
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}

122
app/coffee/OutputFileOptimiser.js
View File

@@ -1,55 +1,77 @@
fs = require "fs" /*
Path = require "path" * decaffeinate suggestions:
spawn = require("child_process").spawn * DS102: Remove unnecessary code created because of implicit returns
logger = require "logger-sharelatex" * DS207: Consider shorter variations of null checks
Metrics = require "./Metrics" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
_ = require "underscore" */
let OutputFileOptimiser;
const fs = require("fs");
const Path = require("path");
const { spawn } = require("child_process");
const logger = require("logger-sharelatex");
const Metrics = require("./Metrics");
const _ = require("underscore");
module.exports = OutputFileOptimiser = module.exports = (OutputFileOptimiser = {
optimiseFile: (src, dst, callback = (error) ->) -> optimiseFile(src, dst, callback) {
# check output file (src) and see if we can optimise it, storing // check output file (src) and see if we can optimise it, storing
# the result in the build directory (dst) // the result in the build directory (dst)
if src.match(/\/output\.pdf$/) if (callback == null) { callback = function(error) {}; }
OutputFileOptimiser.checkIfPDFIsOptimised src, (err, isOptimised) -> if (src.match(/\/output\.pdf$/)) {
return callback(null) if err? or isOptimised return OutputFileOptimiser.checkIfPDFIsOptimised(src, function(err, isOptimised) {
OutputFileOptimiser.optimisePDF src, dst, callback if ((err != null) || isOptimised) { return callback(null); }
else return OutputFileOptimiser.optimisePDF(src, dst, callback);
callback (null) });
} else {
return callback((null));
}
},
checkIfPDFIsOptimised: (file, callback) -> checkIfPDFIsOptimised(file, callback) {
SIZE = 16*1024 # check the header of the pdf const SIZE = 16*1024; // check the header of the pdf
result = new Buffer(SIZE) const result = new Buffer(SIZE);
result.fill(0) # prevent leakage of uninitialised buffer result.fill(0); // prevent leakage of uninitialised buffer
fs.open file, "r", (err, fd) -> return fs.open(file, "r", function(err, fd) {
return callback(err) if err? if (err != null) { return callback(err); }
fs.read fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) -> return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) =>
fs.close fd, (errClose) -> fs.close(fd, function(errClose) {
return callback(errRead) if errRead? if (errRead != null) { return callback(errRead); }
return callback(errClose) if errReadClose? if (typeof errReadClose !== 'undefined' && errReadClose !== null) { return callback(errClose); }
isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0 const isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0;
callback(null, isOptimised) return callback(null, isOptimised);
})
);
});
},
optimisePDF: (src, dst, callback = (error) ->) -> optimisePDF(src, dst, callback) {
tmpOutput = dst + '.opt' if (callback == null) { callback = function(error) {}; }
args = ["--linearize", src, tmpOutput] const tmpOutput = dst + '.opt';
logger.log args: args, "running qpdf command" const args = ["--linearize", src, tmpOutput];
logger.log({args}, "running qpdf command");
timer = new Metrics.Timer("qpdf") const timer = new Metrics.Timer("qpdf");
proc = spawn("qpdf", args) const proc = spawn("qpdf", args);
stdout = "" let stdout = "";
proc.stdout.on "data", (chunk) -> proc.stdout.on("data", chunk => stdout += chunk.toString());
stdout += chunk.toString() callback = _.once(callback); // avoid double call back for error and close event
callback = _.once(callback) # avoid double call back for error and close event proc.on("error", function(err) {
proc.on "error", (err) -> logger.warn({err, args}, "qpdf failed");
logger.warn {err, args}, "qpdf failed" return callback(null);
callback(null) # ignore the error }); // ignore the error
proc.on "close", (code) -> return proc.on("close", function(code) {
timer.done() timer.done();
if code != 0 if (code !== 0) {
logger.warn {code, args}, "qpdf returned error" logger.warn({code, args}, "qpdf returned error");
return callback(null) # ignore the error return callback(null); // ignore the error
fs.rename tmpOutput, dst, (err) -> }
if err? return fs.rename(tmpOutput, dst, function(err) {
logger.warn {tmpOutput, dst}, "failed to rename output of qpdf command" if (err != null) {
callback(null) # ignore the error logger.warn({tmpOutput, dst}, "failed to rename output of qpdf command");
}
return callback(null);
});
});
} // ignore the error
});

171
app/coffee/ProjectPersistenceManager.js
View File

@@ -1,84 +1,117 @@
UrlCache = require "./UrlCache" /*
CompileManager = require "./CompileManager" * decaffeinate suggestions:
db = require "./db" * DS101: Remove unnecessary use of Array.from
dbQueue = require "./DbQueue" * DS102: Remove unnecessary code created because of implicit returns
async = require "async" * DS207: Consider shorter variations of null checks
logger = require "logger-sharelatex" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
oneDay = 24 * 60 * 60 * 1000 */
Settings = require "settings-sharelatex" let ProjectPersistenceManager;
const UrlCache = require("./UrlCache");
const CompileManager = require("./CompileManager");
const db = require("./db");
const dbQueue = require("./DbQueue");
const async = require("async");
const logger = require("logger-sharelatex");
const oneDay = 24 * 60 * 60 * 1000;
const Settings = require("settings-sharelatex");
module.exports = ProjectPersistenceManager = module.exports = (ProjectPersistenceManager = {
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5 EXPIRY_TIMEOUT: Settings.project_cache_length_ms || (oneDay * 2.5),
markProjectAsJustAccessed: (project_id, callback = (error) ->) -> markProjectAsJustAccessed(project_id, callback) {
job = (cb)-> if (callback == null) { callback = function(error) {}; }
db.Project.findOrCreate(where: {project_id: project_id}) const job = cb=>
db.Project.findOrCreate({where: {project_id}})
.spread( .spread(
(project, created) -> (project, created) =>
project.updateAttributes(lastAccessed: new Date()) project.updateAttributes({lastAccessed: new Date()})
.then(() -> cb()) .then(() => cb())
.error cb .error(cb)
) )
.error cb .error(cb)
dbQueue.queue.push(job, callback) ;
return dbQueue.queue.push(job, callback);
},
clearExpiredProjects: (callback = (error) ->) -> clearExpiredProjects(callback) {
ProjectPersistenceManager._findExpiredProjectIds (error, project_ids) -> if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return ProjectPersistenceManager._findExpiredProjectIds(function(error, project_ids) {
logger.log project_ids: project_ids, "clearing expired projects" if (error != null) { return callback(error); }
jobs = for project_id in (project_ids or []) logger.log({project_ids}, "clearing expired projects");
do (project_id) -> const jobs = (Array.from(project_ids || [])).map((project_id) =>
(callback) -> (project_id =>
ProjectPersistenceManager.clearProjectFromCache project_id, (err) -> callback =>
if err? ProjectPersistenceManager.clearProjectFromCache(project_id, function(err) {
logger.error err: err, project_id: project_id, "error clearing project" if (err != null) {
callback() logger.error({err, project_id}, "error clearing project");
async.series jobs, (error) -> }
return callback(error) if error? return callback();
CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) -> })
callback() # ignore any errors from deleting directories
clearProject: (project_id, user_id, callback = (error) ->) -> )(project_id));
logger.log project_id: project_id, user_id:user_id, "clearing project for user" return async.series(jobs, function(error) {
CompileManager.clearProject project_id, user_id, (error) -> if (error != null) { return callback(error); }
return callback(error) if error? return CompileManager.clearExpiredProjects(ProjectPersistenceManager.EXPIRY_TIMEOUT, error => callback());
ProjectPersistenceManager.clearProjectFromCache project_id, (error) -> });
return callback(error) if error? });
callback() }, // ignore any errors from deleting directories
clearProjectFromCache: (project_id, callback = (error) ->) -> clearProject(project_id, user_id, callback) {
logger.log project_id: project_id, "clearing project from cache" if (callback == null) { callback = function(error) {}; }
UrlCache.clearProject project_id, (error) -> logger.log({project_id, user_id}, "clearing project for user");
if error? return CompileManager.clearProject(project_id, user_id, function(error) {
logger.err error:error, project_id: project_id, "error clearing project from cache" if (error != null) { return callback(error); }
return callback(error) return ProjectPersistenceManager.clearProjectFromCache(project_id, function(error) {
ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> if (error != null) { return callback(error); }
if error? return callback();
logger.err error:error, project_id:project_id, "error clearing project from database" });
callback(error) });
},
_clearProjectFromDatabase: (project_id, callback = (error) ->) -> clearProjectFromCache(project_id, callback) {
logger.log project_id:project_id, "clearing project from database" if (callback == null) { callback = function(error) {}; }
job = (cb)-> logger.log({project_id}, "clearing project from cache");
db.Project.destroy(where: {project_id: project_id}) return UrlCache.clearProject(project_id, function(error) {
.then(() -> cb()) if (error != null) {
.error cb logger.err({error, project_id}, "error clearing project from cache");
dbQueue.queue.push(job, callback) return callback(error);
}
return ProjectPersistenceManager._clearProjectFromDatabase(project_id, function(error) {
if (error != null) {
logger.err({error, project_id}, "error clearing project from database");
}
return callback(error);
});
});
},
_clearProjectFromDatabase(project_id, callback) {
if (callback == null) { callback = function(error) {}; }
logger.log({project_id}, "clearing project from database");
const job = cb=>
db.Project.destroy({where: {project_id}})
.then(() => cb())
.error(cb)
;
return dbQueue.queue.push(job, callback);
},
_findExpiredProjectIds: (callback = (error, project_ids) ->) -> _findExpiredProjectIds(callback) {
job = (cb)-> if (callback == null) { callback = function(error, project_ids) {}; }
keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT) const job = function(cb){
q = {} const keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT);
q[db.op.lt] = keepProjectsFrom const q = {};
db.Project.findAll(where:{lastAccessed:q}) q[db.op.lt] = keepProjectsFrom;
.then((projects) -> return db.Project.findAll({where:{lastAccessed:q}})
cb null, projects.map((project) -> project.project_id) .then(projects => cb(null, projects.map(project => project.project_id))).error(cb);
).error cb };
dbQueue.queue.push(job, callback) return dbQueue.queue.push(job, callback);
}
});
logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout" logger.log({EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout");

296
app/coffee/RequestParser.js
View File

@@ -1,128 +1,182 @@
settings = require("settings-sharelatex") /*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let RequestParser;
const settings = require("settings-sharelatex");
module.exports = RequestParser = module.exports = (RequestParser = {
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"] VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"],
MAX_TIMEOUT: 600 MAX_TIMEOUT: 600,
parse: (body, callback = (error, data) ->) -> parse(body, callback) {
response = {} let resource;
if (callback == null) { callback = function(error, data) {}; }
const response = {};
if !body.compile? if ((body.compile == null)) {
return callback "top level object should have a compile attribute" return callback("top level object should have a compile attribute");
compile = body.compile
compile.options ||= {}
try
response.compiler = @_parseAttribute "compiler",
compile.options.compiler,
validValues: @VALID_COMPILERS
default: "pdflatex"
type: "string"
response.timeout = @_parseAttribute "timeout",
compile.options.timeout
default: RequestParser.MAX_TIMEOUT
type: "number"
response.imageName = @_parseAttribute "imageName",
compile.options.imageName,
type: "string"
response.draft = @_parseAttribute "draft",
compile.options.draft,
default: false,
type: "boolean"
response.check = @_parseAttribute "check",
compile.options.check,
type: "string"
response.flags = @_parseAttribute "flags",
compile.options.flags,
default: [],
type: "object"
# The syncType specifies whether the request contains all
# resources (full) or only those resources to be updated
# in-place (incremental).
response.syncType = @_parseAttribute "syncType",
compile.options.syncType,
validValues: ["full", "incremental"]
type: "string"
# The syncState is an identifier passed in with the request
# which has the property that it changes when any resource is
# added, deleted, moved or renamed.
#
# on syncType full the syncState identifier is passed in and
# stored
#
# on syncType incremental the syncState identifier must match
# the stored value
response.syncState = @_parseAttribute "syncState",
compile.options.syncState,
type: "string"
if response.timeout > RequestParser.MAX_TIMEOUT
response.timeout = RequestParser.MAX_TIMEOUT
response.timeout = response.timeout * 1000 # milliseconds
response.resources = (@_parseResource(resource) for resource in (compile.resources or []))
rootResourcePath = @_parseAttribute "rootResourcePath",
compile.rootResourcePath
default: "main.tex"
type: "string"
originalRootResourcePath = rootResourcePath
sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath)
response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath)
for resource in response.resources
if resource.path == originalRootResourcePath
resource.path = sanitizedRootResourcePath
catch error
return callback error
callback null, response
_parseResource: (resource) ->
if !resource.path? or typeof resource.path != "string"
throw "all resources should have a path attribute"
if resource.modified?
modified = new Date(resource.modified)
if isNaN(modified.getTime())
throw "resource modified date could not be understood: #{resource.modified}"
if !resource.url? and !resource.content?
throw "all resources should have either a url or content attribute"
if resource.content? and typeof resource.content != "string"
throw "content attribute should be a string"
if resource.url? and typeof resource.url != "string"
throw "url attribute should be a string"
return {
path: resource.path
modified: modified
url: resource.url
content: resource.content
} }
_parseAttribute: (name, attribute, options) -> const { compile } = body;
if attribute? if (!compile.options) { compile.options = {}; }
if options.validValues?
if options.validValues.indexOf(attribute) == -1
throw "#{name} attribute should be one of: #{options.validValues.join(", ")}"
if options.type?
if typeof attribute != options.type
throw "#{name} attribute should be a #{options.type}"
else
return options.default if options.default?
return attribute
_sanitizePath: (path) -> try {
# See http://php.net/manual/en/function.escapeshellcmd.php response.compiler = this._parseAttribute("compiler",
path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "") compile.options.compiler, {
validValues: this.VALID_COMPILERS,
default: "pdflatex",
type: "string"
}
);
response.timeout = this._parseAttribute("timeout",
compile.options.timeout, {
default: RequestParser.MAX_TIMEOUT,
type: "number"
}
);
response.imageName = this._parseAttribute("imageName",
compile.options.imageName,
{type: "string"});
response.draft = this._parseAttribute("draft",
compile.options.draft, {
default: false,
type: "boolean"
}
);
response.check = this._parseAttribute("check",
compile.options.check,
{type: "string"});
response.flags = this._parseAttribute("flags",
compile.options.flags, {
default: [],
type: "object"
}
);
_checkPath: (path) -> // The syncType specifies whether the request contains all
# check that the request does not use a relative path // resources (full) or only those resources to be updated
for dir in path.split('/') // in-place (incremental).
if dir == '..' response.syncType = this._parseAttribute("syncType",
throw "relative path in root resource" compile.options.syncType, {
return path validValues: ["full", "incremental"],
type: "string"
}
);
// The syncState is an identifier passed in with the request
// which has the property that it changes when any resource is
// added, deleted, moved or renamed.
//
// on syncType full the syncState identifier is passed in and
// stored
//
// on syncType incremental the syncState identifier must match
// the stored value
response.syncState = this._parseAttribute("syncState",
compile.options.syncState,
{type: "string"});
if (response.timeout > RequestParser.MAX_TIMEOUT) {
response.timeout = RequestParser.MAX_TIMEOUT;
}
response.timeout = response.timeout * 1000; // milliseconds
response.resources = ((() => {
const result = [];
for (resource of Array.from((compile.resources || []))) { result.push(this._parseResource(resource));
}
return result;
})());
const rootResourcePath = this._parseAttribute("rootResourcePath",
compile.rootResourcePath, {
default: "main.tex",
type: "string"
}
);
const originalRootResourcePath = rootResourcePath;
const sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath);
response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath);
for (resource of Array.from(response.resources)) {
if (resource.path === originalRootResourcePath) {
resource.path = sanitizedRootResourcePath;
}
}
} catch (error1) {
const error = error1;
return callback(error);
}
return callback(null, response);
},
_parseResource(resource) {
let modified;
if ((resource.path == null) || (typeof resource.path !== "string")) {
throw "all resources should have a path attribute";
}
if (resource.modified != null) {
modified = new Date(resource.modified);
if (isNaN(modified.getTime())) {
throw `resource modified date could not be understood: ${resource.modified}`;
}
}
if ((resource.url == null) && (resource.content == null)) {
throw "all resources should have either a url or content attribute";
}
if ((resource.content != null) && (typeof resource.content !== "string")) {
throw "content attribute should be a string";
}
if ((resource.url != null) && (typeof resource.url !== "string")) {
throw "url attribute should be a string";
}
return {
path: resource.path,
modified,
url: resource.url,
content: resource.content
};
},
_parseAttribute(name, attribute, options) {
if (attribute != null) {
if (options.validValues != null) {
if (options.validValues.indexOf(attribute) === -1) {
throw `${name} attribute should be one of: ${options.validValues.join(", ")}`;
}
}
if (options.type != null) {
if (typeof attribute !== options.type) {
throw `${name} attribute should be a ${options.type}`;
}
}
} else {
if (options.default != null) { return options.default; }
}
return attribute;
},
_sanitizePath(path) {
// See http://php.net/manual/en/function.escapeshellcmd.php
return path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "");
},
_checkPath(path) {
// check that the request does not use a relative path
for (let dir of Array.from(path.split('/'))) {
if (dir === '..') {
throw "relative path in root resource";
}
}
return path;
}
});

168
app/coffee/ResourceStateManager.js
View File

@@ -1,72 +1,108 @@
Path = require "path" /*
fs = require "fs" * decaffeinate suggestions:
logger = require "logger-sharelatex" * DS101: Remove unnecessary use of Array.from
settings = require("settings-sharelatex") * DS102: Remove unnecessary code created because of implicit returns
Errors = require "./Errors" * DS103: Rewrite code to no longer use __guard__
SafeReader = require "./SafeReader" * DS201: Simplify complex destructure assignments
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ResourceStateManager;
const Path = require("path");
const fs = require("fs");
const logger = require("logger-sharelatex");
const settings = require("settings-sharelatex");
const Errors = require("./Errors");
const SafeReader = require("./SafeReader");
module.exports = ResourceStateManager = module.exports = (ResourceStateManager = {
# The sync state is an identifier which must match for an // The sync state is an identifier which must match for an
# incremental update to be allowed. // incremental update to be allowed.
# //
# The initial value is passed in and stored on a full // The initial value is passed in and stored on a full
# compile, along with the list of resources.. // compile, along with the list of resources..
# //
# Subsequent incremental compiles must come with the same value - if // Subsequent incremental compiles must come with the same value - if
# not they will be rejected with a 409 Conflict response. The // not they will be rejected with a 409 Conflict response. The
# previous list of resources is returned. // previous list of resources is returned.
# //
# An incremental compile can only update existing files with new // An incremental compile can only update existing files with new
# content. The sync state identifier must change if any docs or // content. The sync state identifier must change if any docs or
# files are moved, added, deleted or renamed. // files are moved, added, deleted or renamed.
SYNC_STATE_FILE: ".project-sync-state" SYNC_STATE_FILE: ".project-sync-state",
SYNC_STATE_MAX_SIZE: 128*1024 SYNC_STATE_MAX_SIZE: 128*1024,
saveProjectState: (state, resources, basePath, callback = (error) ->) -> saveProjectState(state, resources, basePath, callback) {
stateFile = Path.join(basePath, @SYNC_STATE_FILE) if (callback == null) { callback = function(error) {}; }
if not state? # remove the file if no state passed in const stateFile = Path.join(basePath, this.SYNC_STATE_FILE);
logger.log state:state, basePath:basePath, "clearing sync state" if ((state == null)) { // remove the file if no state passed in
fs.unlink stateFile, (err) -> logger.log({state, basePath}, "clearing sync state");
if err? and err.code isnt 'ENOENT' return fs.unlink(stateFile, function(err) {
return callback(err) if ((err != null) && (err.code !== 'ENOENT')) {
else return callback(err);
return callback() } else {
else return callback();
logger.log state:state, basePath:basePath, "writing sync state" }
resourceList = (resource.path for resource in resources) });
fs.writeFile stateFile, [resourceList..., "stateHash:#{state}"].join("\n"), callback } else {
logger.log({state, basePath}, "writing sync state");
const resourceList = (Array.from(resources).map((resource) => resource.path));
return fs.writeFile(stateFile, [...Array.from(resourceList), `stateHash:${state}`].join("\n"), callback);
}
},
checkProjectStateMatches: (state, basePath, callback = (error, resources) ->) -> checkProjectStateMatches(state, basePath, callback) {
stateFile = Path.join(basePath, @SYNC_STATE_FILE) if (callback == null) { callback = function(error, resources) {}; }
size = @SYNC_STATE_MAX_SIZE const stateFile = Path.join(basePath, this.SYNC_STATE_FILE);
SafeReader.readFile stateFile, size, 'utf8', (err, result, bytesRead) -> const size = this.SYNC_STATE_MAX_SIZE;
return callback(err) if err? return SafeReader.readFile(stateFile, size, 'utf8', function(err, result, bytesRead) {
if bytesRead is size if (err != null) { return callback(err); }
logger.error file:stateFile, size:size, bytesRead:bytesRead, "project state file truncated" if (bytesRead === size) {
[resourceList..., oldState] = result?.toString()?.split("\n") or [] logger.error({file:stateFile, size, bytesRead}, "project state file truncated");
newState = "stateHash:#{state}" }
logger.log state:state, oldState: oldState, basePath:basePath, stateMatches: (newState is oldState), "checking sync state" const array = __guard__(result != null ? result.toString() : undefined, x => x.split("\n")) || [],
if newState isnt oldState adjustedLength = Math.max(array.length, 1),
return callback new Errors.FilesOutOfSyncError("invalid state for incremental update") resourceList = array.slice(0, adjustedLength - 1),
else oldState = array[adjustedLength - 1];
resources = ({path: path} for path in resourceList) const newState = `stateHash:${state}`;
callback(null, resources) logger.log({state, oldState, basePath, stateMatches: (newState === oldState)}, "checking sync state");
if (newState !== oldState) {
return callback(new Errors.FilesOutOfSyncError("invalid state for incremental update"));
} else {
const resources = (Array.from(resourceList).map((path) => ({path})));
return callback(null, resources);
}
});
},
checkResourceFiles: (resources, allFiles, basePath, callback = (error) ->) -> checkResourceFiles(resources, allFiles, basePath, callback) {
# check the paths are all relative to current directory // check the paths are all relative to current directory
for file in resources or [] let file;
for dir in file?.path?.split('/') if (callback == null) { callback = function(error) {}; }
if dir == '..' for (file of Array.from(resources || [])) {
return callback new Error("relative path in resource file list") for (let dir of Array.from(__guard__(file != null ? file.path : undefined, x => x.split('/')))) {
# check if any of the input files are not present in list of files if (dir === '..') {
seenFile = {} return callback(new Error("relative path in resource file list"));
for file in allFiles }
seenFile[file] = true }
missingFiles = (resource.path for resource in resources when not seenFile[resource.path]) }
if missingFiles?.length > 0 // check if any of the input files are not present in list of files
logger.err missingFiles:missingFiles, basePath:basePath, allFiles:allFiles, resources:resources, "missing input files for project" const seenFile = {};
return callback new Errors.FilesOutOfSyncError("resource files missing in incremental update") for (file of Array.from(allFiles)) {
else seenFile[file] = true;
callback() }
const missingFiles = (Array.from(resources).filter((resource) => !seenFile[resource.path]).map((resource) => resource.path));
if ((missingFiles != null ? missingFiles.length : undefined) > 0) {
logger.err({missingFiles, basePath, allFiles, resources}, "missing input files for project");
return callback(new Errors.FilesOutOfSyncError("resource files missing in incremental update"));
} else {
return callback();
}
}
});
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}

322
app/coffee/ResourceWriter.js
View File

@@ -1,142 +1,206 @@
UrlCache = require "./UrlCache" /*
Path = require "path" * decaffeinate suggestions:
fs = require "fs" * DS101: Remove unnecessary use of Array.from
async = require "async" * DS102: Remove unnecessary code created because of implicit returns
mkdirp = require "mkdirp" * DS207: Consider shorter variations of null checks
OutputFileFinder = require "./OutputFileFinder" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
ResourceStateManager = require "./ResourceStateManager" */
Metrics = require "./Metrics" let ResourceWriter;
logger = require "logger-sharelatex" const UrlCache = require("./UrlCache");
settings = require("settings-sharelatex") const Path = require("path");
const fs = require("fs");
const async = require("async");
const mkdirp = require("mkdirp");
const OutputFileFinder = require("./OutputFileFinder");
const ResourceStateManager = require("./ResourceStateManager");
const Metrics = require("./Metrics");
const logger = require("logger-sharelatex");
const settings = require("settings-sharelatex");
parallelFileDownloads = settings.parallelFileDownloads or 1 const parallelFileDownloads = settings.parallelFileDownloads || 1;
module.exports = ResourceWriter = module.exports = (ResourceWriter = {
syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) -> syncResourcesToDisk(request, basePath, callback) {
if request.syncType is "incremental" if (callback == null) { callback = function(error, resourceList) {}; }
logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync" if (request.syncType === "incremental") {
ResourceStateManager.checkProjectStateMatches request.syncState, basePath, (error, resourceList) -> logger.log({project_id: request.project_id, user_id: request.user_id}, "incremental sync");
return callback(error) if error? return ResourceStateManager.checkProjectStateMatches(request.syncState, basePath, function(error, resourceList) {
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) -> if (error != null) { return callback(error); }
return callback(error) if error? return ResourceWriter._removeExtraneousFiles(resourceList, basePath, function(error, outputFiles, allFiles) {
ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) -> if (error != null) { return callback(error); }
return callback(error) if error? return ResourceStateManager.checkResourceFiles(resourceList, allFiles, basePath, function(error) {
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) -> if (error != null) { return callback(error); }
return callback(error) if error? return ResourceWriter.saveIncrementalResourcesToDisk(request.project_id, request.resources, basePath, function(error) {
callback(null, resourceList) if (error != null) { return callback(error); }
else return callback(null, resourceList);
logger.log project_id: request.project_id, user_id: request.user_id, "full sync" });
@saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) -> });
return callback(error) if error? });
ResourceStateManager.saveProjectState request.syncState, request.resources, basePath, (error) -> });
return callback(error) if error? } else {
callback(null, request.resources) logger.log({project_id: request.project_id, user_id: request.user_id}, "full sync");
return this.saveAllResourcesToDisk(request.project_id, request.resources, basePath, function(error) {
if (error != null) { return callback(error); }
return ResourceStateManager.saveProjectState(request.syncState, request.resources, basePath, function(error) {
if (error != null) { return callback(error); }
return callback(null, request.resources);
});
});
}
},
saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) -> saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) {
@_createDirectory basePath, (error) => if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return this._createDirectory(basePath, error => {
jobs = for resource in resources if (error != null) { return callback(error); }
do (resource) => const jobs = Array.from(resources).map((resource) =>
(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback) (resource => {
async.parallelLimit jobs, parallelFileDownloads, callback return callback => this._writeResourceToDisk(project_id, resource, basePath, callback);
})(resource));
return async.parallelLimit(jobs, parallelFileDownloads, callback);
});
},
saveAllResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) -> saveAllResourcesToDisk(project_id, resources, basePath, callback) {
@_createDirectory basePath, (error) => if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return this._createDirectory(basePath, error => {
@_removeExtraneousFiles resources, basePath, (error) => if (error != null) { return callback(error); }
return callback(error) if error? return this._removeExtraneousFiles(resources, basePath, error => {
jobs = for resource in resources if (error != null) { return callback(error); }
do (resource) => const jobs = Array.from(resources).map((resource) =>
(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback) (resource => {
async.parallelLimit jobs, parallelFileDownloads, callback return callback => this._writeResourceToDisk(project_id, resource, basePath, callback);
})(resource));
return async.parallelLimit(jobs, parallelFileDownloads, callback);
});
});
},
_createDirectory: (basePath, callback = (error) ->) -> _createDirectory(basePath, callback) {
fs.mkdir basePath, (err) -> if (callback == null) { callback = function(error) {}; }
if err? return fs.mkdir(basePath, function(err) {
if err.code is 'EEXIST' if (err != null) {
return callback() if (err.code === 'EEXIST') {
else return callback();
logger.log {err: err, dir:basePath}, "error creating directory" } else {
return callback(err) logger.log({err, dir:basePath}, "error creating directory");
else return callback(err);
return callback() }
} else {
return callback();
}
});
},
_removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) -> _removeExtraneousFiles(resources, basePath, _callback) {
timer = new Metrics.Timer("unlink-output-files") if (_callback == null) { _callback = function(error, outputFiles, allFiles) {}; }
callback = (error, result...) -> const timer = new Metrics.Timer("unlink-output-files");
timer.done() const callback = function(error, ...result) {
_callback(error, result...) timer.done();
return _callback(error, ...Array.from(result));
};
OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) -> return OutputFileFinder.findOutputFiles(resources, basePath, function(error, outputFiles, allFiles) {
return callback(error) if error? if (error != null) { return callback(error); }
jobs = [] const jobs = [];
for file in outputFiles or [] for (let file of Array.from(outputFiles || [])) {
do (file) -> (function(file) {
path = file.path const { path } = file;
should_delete = true let should_delete = true;
if path.match(/^output\./) or path.match(/\.aux$/) or path.match(/^cache\//) # knitr cache if (path.match(/^output\./) || path.match(/\.aux$/) || path.match(/^cache\//)) { // knitr cache
should_delete = false should_delete = false;
if path.match(/^output-.*/) # Tikz cached figures (default case) }
should_delete = false if (path.match(/^output-.*/)) { // Tikz cached figures (default case)
if path.match(/\.(pdf|dpth|md5)$/) # Tikz cached figures (by extension) should_delete = false;
should_delete = false }
if path.match(/\.(pygtex|pygstyle)$/) or path.match(/(^|\/)_minted-[^\/]+\//) # minted files/directory if (path.match(/\.(pdf|dpth|md5)$/)) { // Tikz cached figures (by extension)
should_delete = false should_delete = false;
if path.match(/\.md\.tex$/) or path.match(/(^|\/)_markdown_[^\/]+\//) # markdown files/directory }
should_delete = false if (path.match(/\.(pygtex|pygstyle)$/) || path.match(/(^|\/)_minted-[^\/]+\//)) { // minted files/directory
if path.match(/-eps-converted-to\.pdf$/) # Epstopdf generated files should_delete = false;
should_delete = false }
if path == "output.pdf" or path == "output.dvi" or path == "output.log" or path == "output.xdv" if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) { // markdown files/directory
should_delete = true should_delete = false;
if path == "output.tex" # created by TikzManager if present in output files }
should_delete = true if (path.match(/-eps-converted-to\.pdf$/)) { // Epstopdf generated files
if should_delete should_delete = false;
jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback }
if ((path === "output.pdf") || (path === "output.dvi") || (path === "output.log") || (path === "output.xdv")) {
should_delete = true;
}
if (path === "output.tex") { // created by TikzManager if present in output files
should_delete = true;
}
if (should_delete) {
return jobs.push(callback => ResourceWriter._deleteFileIfNotDirectory(Path.join(basePath, path), callback));
}
})(file);
}
async.series jobs, (error) -> return async.series(jobs, function(error) {
return callback(error) if error? if (error != null) { return callback(error); }
callback(null, outputFiles, allFiles) return callback(null, outputFiles, allFiles);
});
});
},
_deleteFileIfNotDirectory: (path, callback = (error) ->) -> _deleteFileIfNotDirectory(path, callback) {
fs.stat path, (error, stat) -> if (callback == null) { callback = function(error) {}; }
if error? and error.code is 'ENOENT' return fs.stat(path, function(error, stat) {
return callback() if ((error != null) && (error.code === 'ENOENT')) {
else if error? return callback();
logger.err {err: error, path: path}, "error stating file in deleteFileIfNotDirectory" } else if (error != null) {
return callback(error) logger.err({err: error, path}, "error stating file in deleteFileIfNotDirectory");
else if stat.isFile() return callback(error);
fs.unlink path, (error) -> } else if (stat.isFile()) {
if error? return fs.unlink(path, function(error) {
logger.err {err: error, path: path}, "error removing file in deleteFileIfNotDirectory" if (error != null) {
callback(error) logger.err({err: error, path}, "error removing file in deleteFileIfNotDirectory");
else return callback(error);
callback() } else {
else return callback();
callback() }
});
} else {
return callback();
}
});
},
_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) -> _writeResourceToDisk(project_id, resource, basePath, callback) {
ResourceWriter.checkPath basePath, resource.path, (error, path) -> if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return ResourceWriter.checkPath(basePath, resource.path, function(error, path) {
mkdirp Path.dirname(path), (error) -> if (error != null) { return callback(error); }
return callback(error) if error? return mkdirp(Path.dirname(path), function(error) {
# TODO: Don't overwrite file if it hasn't been modified if (error != null) { return callback(error); }
if resource.url? // TODO: Don't overwrite file if it hasn't been modified
UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)-> if (resource.url != null) {
if err? return UrlCache.downloadUrlToFile(project_id, resource.url, path, resource.modified, function(err){
logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources" if (err != null) {
callback() #try and continue compiling even if http resource can not be downloaded at this time logger.err({err, project_id, path, resource_url:resource.url, modified:resource.modified}, "error downloading file for resources");
else }
process = require("process") return callback();
fs.writeFile path, resource.content, callback }); //try and continue compiling even if http resource can not be downloaded at this time
try } else {
result = fs.lstatSync(path) const process = require("process");
catch e fs.writeFile(path, resource.content, callback);
try {
let result;
return result = fs.lstatSync(path);
} catch (e) {}
}
});
});
},
checkPath: (basePath, resourcePath, callback) -> checkPath(basePath, resourcePath, callback) {
path = Path.normalize(Path.join(basePath, resourcePath)) const path = Path.normalize(Path.join(basePath, resourcePath));
if (path.slice(0, basePath.length + 1) != basePath + "/") if (path.slice(0, basePath.length + 1) !== (basePath + "/")) {
return callback new Error("resource path is outside root directory") return callback(new Error("resource path is outside root directory"));
else } else {
return callback(null, path) return callback(null, path);
}
}
});

55
app/coffee/SafeReader.js
View File

@@ -1,25 +1,40 @@
fs = require "fs" /*
logger = require "logger-sharelatex" * decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let SafeReader;
const fs = require("fs");
const logger = require("logger-sharelatex");
module.exports = SafeReader = module.exports = (SafeReader = {
# safely read up to size bytes from a file and return result as a // safely read up to size bytes from a file and return result as a
# string // string
readFile: (file, size, encoding, callback = (error, result) ->) -> readFile(file, size, encoding, callback) {
fs.open file, 'r', (err, fd) -> if (callback == null) { callback = function(error, result) {}; }
return callback() if err? and err.code is 'ENOENT' return fs.open(file, 'r', function(err, fd) {
return callback(err) if err? if ((err != null) && (err.code === 'ENOENT')) { return callback(); }
if (err != null) { return callback(err); }
# safely return always closing the file // safely return always closing the file
callbackWithClose = (err, result...) -> const callbackWithClose = (err, ...result) =>
fs.close fd, (err1) -> fs.close(fd, function(err1) {
return callback(err) if err? if (err != null) { return callback(err); }
return callback(err1) if err1? if (err1 != null) { return callback(err1); }
callback(null, result...) return callback(null, ...Array.from(result));
})
;
buff = new Buffer(size, 0) # fill with zeros const buff = new Buffer(size, 0); // fill with zeros
fs.read fd, buff, 0, buff.length, 0, (err, bytesRead, buffer) -> return fs.read(fd, buff, 0, buff.length, 0, function(err, bytesRead, buffer) {
return callbackWithClose(err) if err? if (err != null) { return callbackWithClose(err); }
result = buffer.toString(encoding, 0, bytesRead) const result = buffer.toString(encoding, 0, bytesRead);
callbackWithClose(null, result, bytesRead) return callbackWithClose(null, result, bytesRead);
});
});
}
});

103
app/coffee/StaticServerForbidSymlinks.js
View File

@@ -1,41 +1,64 @@
Path = require("path") /*
fs = require("fs") * decaffeinate suggestions:
Settings = require("settings-sharelatex") * DS101: Remove unnecessary use of Array.from
logger = require("logger-sharelatex") * DS102: Remove unnecessary code created because of implicit returns
url = require "url" * DS103: Rewrite code to no longer use __guard__
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ForbidSymlinks;
const Path = require("path");
const fs = require("fs");
const Settings = require("settings-sharelatex");
const logger = require("logger-sharelatex");
const url = require("url");
module.exports = ForbidSymlinks = (staticFn, root, options) -> module.exports = (ForbidSymlinks = function(staticFn, root, options) {
expressStatic = staticFn root, options const expressStatic = staticFn(root, options);
basePath = Path.resolve(root) const basePath = Path.resolve(root);
return (req, res, next) -> return function(req, res, next) {
path = url.parse(req.url)?.pathname let file, project_id, result;
# check that the path is of the form /project_id_or_name/path/to/file.log const path = __guard__(url.parse(req.url), x => x.pathname);
if result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/) // check that the path is of the form /project_id_or_name/path/to/file.log
project_id = result[1] if (result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/)) {
file = result[2] project_id = result[1];
else file = result[2];
logger.warn path: path, "unrecognized file request" } else {
return res.sendStatus(404) logger.warn({path}, "unrecognized file request");
# check that the file does not use a relative path return res.sendStatus(404);
for dir in file.split('/') }
if dir == '..' // check that the file does not use a relative path
logger.warn path: path, "attempt to use a relative path" for (let dir of Array.from(file.split('/'))) {
return res.sendStatus(404) if (dir === '..') {
# check that the requested path is normalized logger.warn({path}, "attempt to use a relative path");
requestedFsPath = "#{basePath}/#{project_id}/#{file}" return res.sendStatus(404);
if requestedFsPath != Path.normalize(requestedFsPath) }
logger.error path: requestedFsPath, "requestedFsPath is not normalized" }
return res.sendStatus(404) // check that the requested path is normalized
# check that the requested path is not a symlink const requestedFsPath = `${basePath}/${project_id}/${file}`;
fs.realpath requestedFsPath, (err, realFsPath)-> if (requestedFsPath !== Path.normalize(requestedFsPath)) {
if err? logger.error({path: requestedFsPath}, "requestedFsPath is not normalized");
if err.code == 'ENOENT' return res.sendStatus(404);
return res.sendStatus(404) }
else // check that the requested path is not a symlink
logger.error err:err, requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "error checking file access" return fs.realpath(requestedFsPath, function(err, realFsPath){
return res.sendStatus(500) if (err != null) {
else if requestedFsPath != realFsPath if (err.code === 'ENOENT') {
logger.warn requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "trying to access a different file (symlink), aborting" return res.sendStatus(404);
return res.sendStatus(404) } else {
else logger.error({err, requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "error checking file access");
expressStatic(req, res, next) return res.sendStatus(500);
}
} else if (requestedFsPath !== realFsPath) {
logger.warn({requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "trying to access a different file (symlink), aborting");
return res.sendStatus(404);
} else {
return expressStatic(req, res, next);
}
});
};
});
function __guard__(value, transform) {
return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined;
}

85
app/coffee/TikzManager.js
View File

@@ -1,37 +1,56 @@
fs = require "fs" /*
Path = require "path" * decaffeinate suggestions:
ResourceWriter = require "./ResourceWriter" * DS101: Remove unnecessary use of Array.from
SafeReader = require "./SafeReader" * DS102: Remove unnecessary code created because of implicit returns
logger = require "logger-sharelatex" * DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let TikzManager;
const fs = require("fs");
const Path = require("path");
const ResourceWriter = require("./ResourceWriter");
const SafeReader = require("./SafeReader");
const logger = require("logger-sharelatex");
# for \tikzexternalize or pstool to work the main file needs to match the // for \tikzexternalize or pstool to work the main file needs to match the
# jobname. Since we set the -jobname to output, we have to create a // jobname. Since we set the -jobname to output, we have to create a
# copy of the main file as 'output.tex'. // copy of the main file as 'output.tex'.
module.exports = TikzManager = module.exports = (TikzManager = {
checkMainFile: (compileDir, mainFile, resources, callback = (error, needsMainFile) ->) -> checkMainFile(compileDir, mainFile, resources, callback) {
# if there's already an output.tex file, we don't want to touch it // if there's already an output.tex file, we don't want to touch it
for resource in resources if (callback == null) { callback = function(error, needsMainFile) {}; }
if resource.path is "output.tex" for (let resource of Array.from(resources)) {
logger.log compileDir: compileDir, mainFile: mainFile, "output.tex already in resources" if (resource.path === "output.tex") {
return callback(null, false) logger.log({compileDir, mainFile}, "output.tex already in resources");
# if there's no output.tex, see if we are using tikz/pgf or pstool in the main file return callback(null, false);
ResourceWriter.checkPath compileDir, mainFile, (error, path) -> }
return callback(error) if error? }
SafeReader.readFile path, 65536, "utf8", (error, content) -> // if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
return callback(error) if error? return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) {
usesTikzExternalize = content?.indexOf("\\tikzexternalize") >= 0 if (error != null) { return callback(error); }
usesPsTool = content?.indexOf("{pstool}") >= 0 return SafeReader.readFile(path, 65536, "utf8", function(error, content) {
logger.log compileDir: compileDir, mainFile: mainFile, usesTikzExternalize:usesTikzExternalize, usesPsTool: usesPsTool, "checked for packages needing main file as output.tex" if (error != null) { return callback(error); }
needsMainFile = (usesTikzExternalize || usesPsTool) const usesTikzExternalize = (content != null ? content.indexOf("\\tikzexternalize") : undefined) >= 0;
callback null, needsMainFile const usesPsTool = (content != null ? content.indexOf("{pstool}") : undefined) >= 0;
logger.log({compileDir, mainFile, usesTikzExternalize, usesPsTool}, "checked for packages needing main file as output.tex");
const needsMainFile = (usesTikzExternalize || usesPsTool);
return callback(null, needsMainFile);
});
});
},
injectOutputFile: (compileDir, mainFile, callback = (error) ->) -> injectOutputFile(compileDir, mainFile, callback) {
ResourceWriter.checkPath compileDir, mainFile, (error, path) -> if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) {
fs.readFile path, "utf8", (error, content) -> if (error != null) { return callback(error); }
return callback(error) if error? return fs.readFile(path, "utf8", function(error, content) {
logger.log compileDir: compileDir, mainFile: mainFile, "copied file to output.tex as project uses packages which require it" if (error != null) { return callback(error); }
# use wx flag to ensure that output file does not already exist logger.log({compileDir, mainFile}, "copied file to output.tex as project uses packages which require it");
fs.writeFile Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback // use wx flag to ensure that output file does not already exist
return fs.writeFile(Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback);
});
});
}
});

281
app/coffee/UrlCache.js
View File

@@ -1,134 +1,189 @@
db = require("./db") /*
dbQueue = require "./DbQueue" * decaffeinate suggestions:
UrlFetcher = require("./UrlFetcher") * DS101: Remove unnecessary use of Array.from
Settings = require("settings-sharelatex") * DS102: Remove unnecessary code created because of implicit returns
crypto = require("crypto") * DS207: Consider shorter variations of null checks
fs = require("fs") * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
logger = require "logger-sharelatex" */
async = require "async" let UrlCache;
const db = require("./db");
const dbQueue = require("./DbQueue");
const UrlFetcher = require("./UrlFetcher");
const Settings = require("settings-sharelatex");
const crypto = require("crypto");
const fs = require("fs");
const logger = require("logger-sharelatex");
const async = require("async");
module.exports = UrlCache = module.exports = (UrlCache = {
downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) -> downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) => if (callback == null) { callback = function(error) {}; }
return callback(error) if error? return UrlCache._ensureUrlIsInCache(project_id, url, lastModified, (error, pathToCachedUrl) => {
UrlCache._copyFile pathToCachedUrl, destPath, (error) -> if (error != null) { return callback(error); }
if error? return UrlCache._copyFile(pathToCachedUrl, destPath, function(error) {
UrlCache._clearUrlDetails project_id, url, () -> if (error != null) {
callback(error) return UrlCache._clearUrlDetails(project_id, url, () => callback(error));
else } else {
callback(error) return callback(error);
}
});
});
},
clearProject: (project_id, callback = (error) ->) -> clearProject(project_id, callback) {
UrlCache._findAllUrlsInProject project_id, (error, urls) -> if (callback == null) { callback = function(error) {}; }
logger.log project_id: project_id, url_count: urls.length, "clearing project URLs" return UrlCache._findAllUrlsInProject(project_id, function(error, urls) {
return callback(error) if error? logger.log({project_id, url_count: urls.length}, "clearing project URLs");
jobs = for url in (urls or []) if (error != null) { return callback(error); }
do (url) -> const jobs = (Array.from(urls || [])).map((url) =>
(callback) -> (url =>
UrlCache._clearUrlFromCache project_id, url, (error) -> callback =>
if error? UrlCache._clearUrlFromCache(project_id, url, function(error) {
logger.error err: error, project_id: project_id, url: url, "error clearing project URL" if (error != null) {
callback() logger.error({err: error, project_id, url}, "error clearing project URL");
async.series jobs, callback }
return callback();
})
_ensureUrlIsInCache: (project_id, url, lastModified, callback = (error, pathOnDisk) ->) -> )(url));
if lastModified? return async.series(jobs, callback);
# MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds. });
# So round down to seconds },
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000)
UrlCache._doesUrlNeedDownloading project_id, url, lastModified, (error, needsDownloading) =>
return callback(error) if error?
if needsDownloading
logger.log url: url, lastModified: lastModified, "downloading URL"
UrlFetcher.pipeUrlToFile url, UrlCache._cacheFilePathForUrl(project_id, url), (error) =>
return callback(error) if error?
UrlCache._updateOrCreateUrlDetails project_id, url, lastModified, (error) =>
return callback(error) if error?
callback null, UrlCache._cacheFilePathForUrl(project_id, url)
else
logger.log url: url, lastModified: lastModified, "URL is up to date in cache"
callback null, UrlCache._cacheFilePathForUrl(project_id, url)
_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) -> _ensureUrlIsInCache(project_id, url, lastModified, callback) {
if !lastModified? if (callback == null) { callback = function(error, pathOnDisk) {}; }
return callback null, true if (lastModified != null) {
UrlCache._findUrlDetails project_id, url, (error, urlDetails) -> // MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
return callback(error) if error? // So round down to seconds
if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime() lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000);
return callback null, true }
else return UrlCache._doesUrlNeedDownloading(project_id, url, lastModified, (error, needsDownloading) => {
return callback null, false if (error != null) { return callback(error); }
if (needsDownloading) {
logger.log({url, lastModified}, "downloading URL");
return UrlFetcher.pipeUrlToFile(url, UrlCache._cacheFilePathForUrl(project_id, url), error => {
if (error != null) { return callback(error); }
return UrlCache._updateOrCreateUrlDetails(project_id, url, lastModified, error => {
if (error != null) { return callback(error); }
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url));
});
});
} else {
logger.log({url, lastModified}, "URL is up to date in cache");
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url));
}
});
},
_cacheFileNameForUrl: (project_id, url) -> _doesUrlNeedDownloading(project_id, url, lastModified, callback) {
project_id + ":" + crypto.createHash("md5").update(url).digest("hex") if (callback == null) { callback = function(error, needsDownloading) {}; }
if ((lastModified == null)) {
return callback(null, true);
}
return UrlCache._findUrlDetails(project_id, url, function(error, urlDetails) {
if (error != null) { return callback(error); }
if ((urlDetails == null) || (urlDetails.lastModified == null) || (urlDetails.lastModified.getTime() < lastModified.getTime())) {
return callback(null, true);
} else {
return callback(null, false);
}
});
},
_cacheFilePathForUrl: (project_id, url) -> _cacheFileNameForUrl(project_id, url) {
"#{Settings.path.clsiCacheDir}/#{UrlCache._cacheFileNameForUrl(project_id, url)}" return project_id + ":" + crypto.createHash("md5").update(url).digest("hex");
},
_copyFile: (from, to, _callback = (error) ->) -> _cacheFilePathForUrl(project_id, url) {
callbackOnce = (error) -> return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(project_id, url)}`;
if error? },
logger.error err: error, from:from, to:to, "error copying file from cache"
_callback(error)
_callback = () ->
writeStream = fs.createWriteStream(to)
readStream = fs.createReadStream(from)
writeStream.on "error", callbackOnce
readStream.on "error", callbackOnce
writeStream.on "close", callbackOnce
writeStream.on "open", () ->
readStream.pipe(writeStream)
_clearUrlFromCache: (project_id, url, callback = (error) ->) -> _copyFile(from, to, _callback) {
UrlCache._clearUrlDetails project_id, url, (error) -> if (_callback == null) { _callback = function(error) {}; }
return callback(error) if error? const callbackOnce = function(error) {
UrlCache._deleteUrlCacheFromDisk project_id, url, (error) -> if (error != null) {
return callback(error) if error? logger.error({err: error, from, to}, "error copying file from cache");
callback null }
_callback(error);
return _callback = function() {};
};
const writeStream = fs.createWriteStream(to);
const readStream = fs.createReadStream(from);
writeStream.on("error", callbackOnce);
readStream.on("error", callbackOnce);
writeStream.on("close", callbackOnce);
return writeStream.on("open", () => readStream.pipe(writeStream));
},
_deleteUrlCacheFromDisk: (project_id, url, callback = (error) ->) -> _clearUrlFromCache(project_id, url, callback) {
fs.unlink UrlCache._cacheFilePathForUrl(project_id, url), (error) -> if (callback == null) { callback = function(error) {}; }
if error? and error.code != 'ENOENT' # no error if the file isn't present return UrlCache._clearUrlDetails(project_id, url, function(error) {
return callback(error) if (error != null) { return callback(error); }
else return UrlCache._deleteUrlCacheFromDisk(project_id, url, function(error) {
return callback() if (error != null) { return callback(error); }
return callback(null);
});
});
},
_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) -> _deleteUrlCacheFromDisk(project_id, url, callback) {
job = (cb)-> if (callback == null) { callback = function(error) {}; }
db.UrlCache.find(where: { url: url, project_id: project_id }) return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function(error) {
.then((urlDetails) -> cb null, urlDetails) if ((error != null) && (error.code !== 'ENOENT')) { // no error if the file isn't present
.error cb return callback(error);
dbQueue.queue.push job, callback } else {
return callback();
}
});
},
_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) -> _findUrlDetails(project_id, url, callback) {
job = (cb)-> if (callback == null) { callback = function(error, urlDetails) {}; }
db.UrlCache.findOrCreate(where: {url: url, project_id: project_id}) const job = cb=>
db.UrlCache.find({where: { url, project_id }})
.then(urlDetails => cb(null, urlDetails))
.error(cb)
;
return dbQueue.queue.push(job, callback);
},
_updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
if (callback == null) { callback = function(error) {}; }
const job = cb=>
db.UrlCache.findOrCreate({where: {url, project_id}})
.spread( .spread(
(urlDetails, created) -> (urlDetails, created) =>
urlDetails.updateAttributes(lastModified: lastModified) urlDetails.updateAttributes({lastModified})
.then(() -> cb()) .then(() => cb())
.error(cb) .error(cb)
) )
.error cb .error(cb)
dbQueue.queue.push(job, callback) ;
return dbQueue.queue.push(job, callback);
},
_clearUrlDetails: (project_id, url, callback = (error) ->) -> _clearUrlDetails(project_id, url, callback) {
job = (cb)-> if (callback == null) { callback = function(error) {}; }
db.UrlCache.destroy(where: {url: url, project_id: project_id}) const job = cb=>
.then(() -> cb null) db.UrlCache.destroy({where: {url, project_id}})
.error cb .then(() => cb(null))
dbQueue.queue.push(job, callback) .error(cb)
;
return dbQueue.queue.push(job, callback);
},
_findAllUrlsInProject: (project_id, callback = (error, urls) ->) -> _findAllUrlsInProject(project_id, callback) {
job = (cb)-> if (callback == null) { callback = function(error, urls) {}; }
db.UrlCache.findAll(where: { project_id: project_id }) const job = cb=>
db.UrlCache.findAll({where: { project_id }})
.then( .then(
(urlEntries) -> urlEntries => cb(null, urlEntries.map(entry => entry.url)))
cb null, urlEntries.map((entry) -> entry.url) .error(cb)
) ;
.error cb return dbQueue.queue.push(job, callback);
dbQueue.queue.push(job, callback) }
});

136
app/coffee/UrlFetcher.js
View File

@@ -1,70 +1,88 @@
request = require("request").defaults(jar: false) /*
fs = require("fs") * decaffeinate suggestions:
logger = require "logger-sharelatex" * DS102: Remove unnecessary code created because of implicit returns
settings = require("settings-sharelatex") * DS207: Consider shorter variations of null checks
URL = require('url'); * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let UrlFetcher;
const request = require("request").defaults({jar: false});
const fs = require("fs");
const logger = require("logger-sharelatex");
const settings = require("settings-sharelatex");
const URL = require('url');
oneMinute = 60 * 1000 const oneMinute = 60 * 1000;
module.exports = UrlFetcher = module.exports = (UrlFetcher = {
pipeUrlToFile: (url, filePath, _callback = (error) ->) -> pipeUrlToFile(url, filePath, _callback) {
callbackOnce = (error) -> if (_callback == null) { _callback = function(error) {}; }
clearTimeout timeoutHandler if timeoutHandler? const callbackOnce = function(error) {
_callback(error) if (timeoutHandler != null) { clearTimeout(timeoutHandler); }
_callback = () -> _callback(error);
return _callback = function() {};
};
if settings.filestoreDomainOveride? if (settings.filestoreDomainOveride != null) {
p = URL.parse(url).path const p = URL.parse(url).path;
url = "#{settings.filestoreDomainOveride}#{p}" url = `${settings.filestoreDomainOveride}${p}`;
timeoutHandler = setTimeout () -> }
timeoutHandler = null var timeoutHandler = setTimeout(function() {
logger.error url:url, filePath: filePath, "Timed out downloading file to cache" timeoutHandler = null;
callbackOnce(new Error("Timed out downloading file to cache #{url}")) logger.error({url, filePath}, "Timed out downloading file to cache");
# FIXME: maybe need to close fileStream here return callbackOnce(new Error(`Timed out downloading file to cache ${url}`));
, 3 * oneMinute }
// FIXME: maybe need to close fileStream here
, 3 * oneMinute);
logger.log url:url, filePath: filePath, "started downloading url to cache" logger.log({url, filePath}, "started downloading url to cache");
urlStream = request.get({url: url, timeout: oneMinute}) const urlStream = request.get({url, timeout: oneMinute});
urlStream.pause() # stop data flowing until we are ready urlStream.pause(); // stop data flowing until we are ready
# attach handlers before setting up pipes // attach handlers before setting up pipes
urlStream.on "error", (error) -> urlStream.on("error", function(error) {
logger.error err: error, url:url, filePath: filePath, "error downloading url" logger.error({err: error, url, filePath}, "error downloading url");
callbackOnce(error or new Error("Something went wrong downloading the URL #{url}")) return callbackOnce(error || new Error(`Something went wrong downloading the URL ${url}`));
});
urlStream.on "end", () -> urlStream.on("end", () => logger.log({url, filePath}, "finished downloading file into cache"));
logger.log url:url, filePath: filePath, "finished downloading file into cache"
urlStream.on "response", (res) -> return urlStream.on("response", function(res) {
if res.statusCode >= 200 and res.statusCode < 300 if ((res.statusCode >= 200) && (res.statusCode < 300)) {
fileStream = fs.createWriteStream(filePath) const fileStream = fs.createWriteStream(filePath);
# attach handlers before setting up pipes // attach handlers before setting up pipes
fileStream.on 'error', (error) -> fileStream.on('error', function(error) {
logger.error err: error, url:url, filePath: filePath, "error writing file into cache" logger.error({err: error, url, filePath}, "error writing file into cache");
fs.unlink filePath, (err) -> return fs.unlink(filePath, function(err) {
if err? if (err != null) {
logger.err err: err, filePath: filePath, "error deleting file from cache" logger.err({err, filePath}, "error deleting file from cache");
callbackOnce(error) }
return callbackOnce(error);
});
});
fileStream.on 'finish', () -> fileStream.on('finish', function() {
logger.log url:url, filePath: filePath, "finished writing file into cache" logger.log({url, filePath}, "finished writing file into cache");
callbackOnce() return callbackOnce();
});
fileStream.on 'pipe', () -> fileStream.on('pipe', () => logger.log({url, filePath}, "piping into filestream"));
logger.log url:url, filePath: filePath, "piping into filestream"
urlStream.pipe(fileStream) urlStream.pipe(fileStream);
urlStream.resume() # now we are ready to handle the data return urlStream.resume(); // now we are ready to handle the data
else } else {
logger.error statusCode: res.statusCode, url:url, filePath: filePath, "unexpected status code downloading url to cache" logger.error({statusCode: res.statusCode, url, filePath}, "unexpected status code downloading url to cache");
# https://nodejs.org/api/http.html#http_class_http_clientrequest // https://nodejs.org/api/http.html#http_class_http_clientrequest
# If you add a 'response' event handler, then you must consume // If you add a 'response' event handler, then you must consume
# the data from the response object, either by calling // the data from the response object, either by calling
# response.read() whenever there is a 'readable' event, or by // response.read() whenever there is a 'readable' event, or by
# adding a 'data' handler, or by calling the .resume() // adding a 'data' handler, or by calling the .resume()
# method. Until the data is consumed, the 'end' event will not // method. Until the data is consumed, the 'end' event will not
# fire. Also, until the data is read it will consume memory // fire. Also, until the data is read it will consume memory
# that can eventually lead to a 'process out of memory' error. // that can eventually lead to a 'process out of memory' error.
urlStream.resume() # discard the data urlStream.resume(); // discard the data
callbackOnce(new Error("URL returned non-success status code: #{res.statusCode} #{url}")) return callbackOnce(new Error(`URL returned non-success status code: ${res.statusCode} ${url}`));
}
});
}
});

60
app/coffee/db.js
View File

@@ -1,55 +1,59 @@
Sequelize = require("sequelize") /*
Settings = require("settings-sharelatex") * decaffeinate suggestions:
_ = require("underscore") * DS102: Remove unnecessary code created because of implicit returns
logger = require "logger-sharelatex" * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const Sequelize = require("sequelize");
const Settings = require("settings-sharelatex");
const _ = require("underscore");
const logger = require("logger-sharelatex");
options = _.extend {logging:false}, Settings.mysql.clsi const options = _.extend({logging:false}, Settings.mysql.clsi);
logger.log dbPath:Settings.mysql.clsi.storage, "connecting to db" logger.log({dbPath:Settings.mysql.clsi.storage}, "connecting to db");
sequelize = new Sequelize( const sequelize = new Sequelize(
Settings.mysql.clsi.database, Settings.mysql.clsi.database,
Settings.mysql.clsi.username, Settings.mysql.clsi.username,
Settings.mysql.clsi.password, Settings.mysql.clsi.password,
options options
) );
if Settings.mysql.clsi.dialect == "sqlite" if (Settings.mysql.clsi.dialect === "sqlite") {
logger.log "running PRAGMA journal_mode=WAL;" logger.log("running PRAGMA journal_mode=WAL;");
sequelize.query("PRAGMA journal_mode=WAL;") sequelize.query("PRAGMA journal_mode=WAL;");
sequelize.query("PRAGMA synchronous=OFF;") sequelize.query("PRAGMA synchronous=OFF;");
sequelize.query("PRAGMA read_uncommitted = true;") sequelize.query("PRAGMA read_uncommitted = true;");
}
module.exports = module.exports = {
UrlCache: sequelize.define("UrlCache", { UrlCache: sequelize.define("UrlCache", {
url: Sequelize.STRING url: Sequelize.STRING,
project_id: Sequelize.STRING project_id: Sequelize.STRING,
lastModified: Sequelize.DATE lastModified: Sequelize.DATE
}, { }, {
indexes: [ indexes: [
{fields: ['url', 'project_id']}, {fields: ['url', 'project_id']},
{fields: ['project_id']} {fields: ['project_id']}
] ]
}) }),
Project: sequelize.define("Project", { Project: sequelize.define("Project", {
project_id: {type: Sequelize.STRING, primaryKey: true} project_id: {type: Sequelize.STRING, primaryKey: true},
lastAccessed: Sequelize.DATE lastAccessed: Sequelize.DATE
}, { }, {
indexes: [ indexes: [
{fields: ['lastAccessed']} {fields: ['lastAccessed']}
] ]
}) }),
op: Sequelize.Op op: Sequelize.Op,
sync: () -> sync() {
logger.log dbPath:Settings.mysql.clsi.storage, "syncing db schema" logger.log({dbPath:Settings.mysql.clsi.storage}, "syncing db schema");
sequelize.sync() return sequelize.sync()
.then(-> .then(() => logger.log("db sync complete")).catch(err=> console.log(err, "error syncing"));
logger.log "db sync complete" }
).catch((err)-> };
console.log err, "error syncing"
)