diff --git a/app/coffee/CommandRunner.js b/app/coffee/CommandRunner.js index 2d1c3a9..dd7210a 100644 --- a/app/coffee/CommandRunner.js +++ b/app/coffee/CommandRunner.js @@ -1,11 +1,18 @@ -Settings = require "settings-sharelatex" -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let commandRunnerPath; +const Settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); -if Settings.clsi?.dockerRunner == true - commandRunnerPath = "./DockerRunner" -else - commandRunnerPath = "./LocalCommandRunner" -logger.info commandRunnerPath:commandRunnerPath, "selecting command runner for clsi" -CommandRunner = require(commandRunnerPath) +if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) { + commandRunnerPath = "./DockerRunner"; +} else { + commandRunnerPath = "./LocalCommandRunner"; +} +logger.info({commandRunnerPath}, "selecting command runner for clsi"); +const CommandRunner = require(commandRunnerPath); -module.exports = CommandRunner +module.exports = CommandRunner; diff --git a/app/coffee/CompileController.js b/app/coffee/CompileController.js index 4952d84..cfdbcfe 100644 --- a/app/coffee/CompileController.js +++ b/app/coffee/CompileController.js @@ -1,119 +1,163 @@ -RequestParser = require "./RequestParser" -CompileManager = require "./CompileManager" -Settings = require "settings-sharelatex" -Metrics = require "./Metrics" -ProjectPersistenceManager = require "./ProjectPersistenceManager" -logger = require "logger-sharelatex" -Errors = require "./Errors" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let CompileController; +const RequestParser = require("./RequestParser"); +const CompileManager = require("./CompileManager"); +const Settings = require("settings-sharelatex"); +const Metrics = require("./Metrics"); +const ProjectPersistenceManager = require("./ProjectPersistenceManager"); +const logger = require("logger-sharelatex"); +const Errors = require("./Errors"); -module.exports = CompileController = - compile: (req, res, next = (error) ->) -> - timer = new Metrics.Timer("compile-request") - RequestParser.parse req.body, (error, request) -> - return next(error) if error? - request.project_id = req.params.project_id - request.user_id = req.params.user_id if req.params.user_id? - ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) -> - return next(error) if error? - CompileManager.doCompileWithLock request, (error, outputFiles = []) -> - if error instanceof Errors.AlreadyCompilingError - code = 423 # Http 423 Locked - status = "compile-in-progress" - else if error instanceof Errors.FilesOutOfSyncError - code = 409 # Http 409 Conflict - status = "retry" - else if error?.terminated - status = "terminated" - else if error?.validate - status = "validation-#{error.validate}" - else if error?.timedout - status = "timedout" - logger.log err: error, project_id: request.project_id, "timeout running compile" - else if error? - status = "error" - code = 500 - logger.warn err: error, project_id: request.project_id, "error running compile" - else - status = "failure" - for file in outputFiles - if file.path?.match(/output\.pdf$/) - status = "success" +module.exports = (CompileController = { + compile(req, res, next) { + if (next == null) { next = function(error) {}; } + const timer = new Metrics.Timer("compile-request"); + return RequestParser.parse(req.body, function(error, request) { + if (error != null) { return next(error); } + request.project_id = req.params.project_id; + if (req.params.user_id != null) { request.user_id = req.params.user_id; } + return ProjectPersistenceManager.markProjectAsJustAccessed(request.project_id, function(error) { + if (error != null) { return next(error); } + return CompileManager.doCompileWithLock(request, function(error, outputFiles) { + let code, status; + if (outputFiles == null) { outputFiles = []; } + if (error instanceof Errors.AlreadyCompilingError) { + code = 423; // Http 423 Locked + status = "compile-in-progress"; + } else if (error instanceof Errors.FilesOutOfSyncError) { + code = 409; // Http 409 Conflict + status = "retry"; + } else if (error != null ? error.terminated : undefined) { + status = "terminated"; + } else if (error != null ? error.validate : undefined) { + status = `validation-${error.validate}`; + } else if (error != null ? error.timedout : undefined) { + status = "timedout"; + logger.log({err: error, project_id: request.project_id}, "timeout running compile"); + } else if (error != null) { + status = "error"; + code = 500; + logger.warn({err: error, project_id: request.project_id}, "error running compile"); + } else { + let file; + status = "failure"; + for (file of Array.from(outputFiles)) { + if (file.path != null ? file.path.match(/output\.pdf$/) : undefined) { + status = "success"; + } + } - if status == "failure" - logger.warn project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated" + if (status === "failure") { + logger.warn({project_id: request.project_id, outputFiles}, "project failed to compile successfully, no output.pdf generated"); + } - # log an error if any core files are found - for file in outputFiles - if file.path is "core" - logger.error project_id:request.project_id, req:req, outputFiles:outputFiles, "core file found in output" - - if error? - outputFiles = error.outputFiles || [] - - timer.done() - res.status(code or 200).send { - compile: - status: status - error: error?.message or error - outputFiles: outputFiles.map (file) -> - url: - "#{Settings.apis.clsi.url}/project/#{request.project_id}" + - (if request.user_id? then "/user/#{request.user_id}" else "") + - (if file.build? then "/build/#{file.build}" else "") + - "/output/#{file.path}" - path: file.path - type: file.type - build: file.build + // log an error if any core files are found + for (file of Array.from(outputFiles)) { + if (file.path === "core") { + logger.error({project_id:request.project_id, req, outputFiles}, "core file found in output"); + } + } } - stopCompile: (req, res, next) -> - {project_id, user_id} = req.params - CompileManager.stopCompile project_id, user_id, (error) -> - return next(error) if error? - res.sendStatus(204) + if (error != null) { + outputFiles = error.outputFiles || []; + } - clearCache: (req, res, next = (error) ->) -> - ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) -> - return next(error) if error? - res.sendStatus(204) # No content + timer.done(); + return res.status(code || 200).send({ + compile: { + status, + error: (error != null ? error.message : undefined) || error, + outputFiles: outputFiles.map(file => + ({ + url: + `${Settings.apis.clsi.url}/project/${request.project_id}` + + ((request.user_id != null) ? `/user/${request.user_id}` : "") + + ((file.build != null) ? `/build/${file.build}` : "") + + `/output/${file.path}`, + path: file.path, + type: file.type, + build: file.build + }) + ) + } + }); + }); + }); + }); + }, - syncFromCode: (req, res, next = (error) ->) -> - file = req.query.file - line = parseInt(req.query.line, 10) - column = parseInt(req.query.column, 10) - project_id = req.params.project_id - user_id = req.params.user_id - CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) -> - return next(error) if error? - res.json { + stopCompile(req, res, next) { + const {project_id, user_id} = req.params; + return CompileManager.stopCompile(project_id, user_id, function(error) { + if (error != null) { return next(error); } + return res.sendStatus(204); + }); + }, + + clearCache(req, res, next) { + if (next == null) { next = function(error) {}; } + return ProjectPersistenceManager.clearProject(req.params.project_id, req.params.user_id, function(error) { + if (error != null) { return next(error); } + return res.sendStatus(204); + }); + }, // No content + + syncFromCode(req, res, next) { + if (next == null) { next = function(error) {}; } + const { file } = req.query; + const line = parseInt(req.query.line, 10); + const column = parseInt(req.query.column, 10); + const { project_id } = req.params; + const { user_id } = req.params; + return CompileManager.syncFromCode(project_id, user_id, file, line, column, function(error, pdfPositions) { + if (error != null) { return next(error); } + return res.json({ pdf: pdfPositions - } + }); + }); + }, - syncFromPdf: (req, res, next = (error) ->) -> - page = parseInt(req.query.page, 10) - h = parseFloat(req.query.h) - v = parseFloat(req.query.v) - project_id = req.params.project_id - user_id = req.params.user_id - CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) -> - return next(error) if error? - res.json { + syncFromPdf(req, res, next) { + if (next == null) { next = function(error) {}; } + const page = parseInt(req.query.page, 10); + const h = parseFloat(req.query.h); + const v = parseFloat(req.query.v); + const { project_id } = req.params; + const { user_id } = req.params; + return CompileManager.syncFromPdf(project_id, user_id, page, h, v, function(error, codePositions) { + if (error != null) { return next(error); } + return res.json({ code: codePositions - } + }); + }); + }, - wordcount: (req, res, next = (error) ->) -> - file = req.query.file || "main.tex" - project_id = req.params.project_id - user_id = req.params.user_id - image = req.query.image - logger.log {image, file, project_id}, "word count request" + wordcount(req, res, next) { + if (next == null) { next = function(error) {}; } + const file = req.query.file || "main.tex"; + const { project_id } = req.params; + const { user_id } = req.params; + const { image } = req.query; + logger.log({image, file, project_id}, "word count request"); - CompileManager.wordcount project_id, user_id, file, image, (error, result) -> - return next(error) if error? - res.json { + return CompileManager.wordcount(project_id, user_id, file, image, function(error, result) { + if (error != null) { return next(error); } + return res.json({ texcount: result - } + }); + }); + }, - status: (req, res, next = (error)-> )-> - res.send("OK") + status(req, res, next ){ + if (next == null) { next = function(error){}; } + return res.send("OK"); + } +}); diff --git a/app/coffee/CompileManager.js b/app/coffee/CompileManager.js index 792beb8..82dafd1 100644 --- a/app/coffee/CompileManager.js +++ b/app/coffee/CompileManager.js @@ -1,345 +1,454 @@ -ResourceWriter = require "./ResourceWriter" -LatexRunner = require "./LatexRunner" -OutputFileFinder = require "./OutputFileFinder" -OutputCacheManager = require "./OutputCacheManager" -Settings = require("settings-sharelatex") -Path = require "path" -logger = require "logger-sharelatex" -Metrics = require "./Metrics" -child_process = require "child_process" -DraftModeManager = require "./DraftModeManager" -TikzManager = require "./TikzManager" -LockManager = require "./LockManager" -fs = require("fs") -fse = require "fs-extra" -os = require("os") -async = require "async" -Errors = require './Errors' -CommandRunner = require "./CommandRunner" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let CompileManager; +const ResourceWriter = require("./ResourceWriter"); +const LatexRunner = require("./LatexRunner"); +const OutputFileFinder = require("./OutputFileFinder"); +const OutputCacheManager = require("./OutputCacheManager"); +const Settings = require("settings-sharelatex"); +const Path = require("path"); +const logger = require("logger-sharelatex"); +const Metrics = require("./Metrics"); +const child_process = require("child_process"); +const DraftModeManager = require("./DraftModeManager"); +const TikzManager = require("./TikzManager"); +const LockManager = require("./LockManager"); +const fs = require("fs"); +const fse = require("fs-extra"); +const os = require("os"); +const async = require("async"); +const Errors = require('./Errors'); +const CommandRunner = require("./CommandRunner"); -getCompileName = (project_id, user_id) -> - if user_id? then "#{project_id}-#{user_id}" else project_id +const getCompileName = function(project_id, user_id) { + if (user_id != null) { return `${project_id}-${user_id}`; } else { return project_id; } +}; -getCompileDir = (project_id, user_id) -> - Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)) +const getCompileDir = (project_id, user_id) => Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)); -module.exports = CompileManager = +module.exports = (CompileManager = { - doCompileWithLock: (request, callback = (error, outputFiles) ->) -> - compileDir = getCompileDir(request.project_id, request.user_id) - lockFile = Path.join(compileDir, ".project-lock") - # use a .project-lock file in the compile directory to prevent - # simultaneous compiles - fse.ensureDir compileDir, (error) -> - return callback(error) if error? - LockManager.runWithLock lockFile, (releaseLock) -> - CompileManager.doCompile(request, releaseLock) - , callback + doCompileWithLock(request, callback) { + if (callback == null) { callback = function(error, outputFiles) {}; } + const compileDir = getCompileDir(request.project_id, request.user_id); + const lockFile = Path.join(compileDir, ".project-lock"); + // use a .project-lock file in the compile directory to prevent + // simultaneous compiles + return fse.ensureDir(compileDir, function(error) { + if (error != null) { return callback(error); } + return LockManager.runWithLock(lockFile, releaseLock => CompileManager.doCompile(request, releaseLock) + , callback); + }); + }, - doCompile: (request, callback = (error, outputFiles) ->) -> - compileDir = getCompileDir(request.project_id, request.user_id) - timer = new Metrics.Timer("write-to-disk") - logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk" - ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) -> - # NOTE: resourceList is insecure, it should only be used to exclude files from the output list - if error? and error instanceof Errors.FilesOutOfSyncError - logger.warn project_id: request.project_id, user_id: request.user_id, "files out of sync, please retry" - return callback(error) - else if error? - logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk" - return callback(error) - logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk" - timer.done() + doCompile(request, callback) { + if (callback == null) { callback = function(error, outputFiles) {}; } + const compileDir = getCompileDir(request.project_id, request.user_id); + let timer = new Metrics.Timer("write-to-disk"); + logger.log({project_id: request.project_id, user_id: request.user_id}, "syncing resources to disk"); + return ResourceWriter.syncResourcesToDisk(request, compileDir, function(error, resourceList) { + // NOTE: resourceList is insecure, it should only be used to exclude files from the output list + if ((error != null) && error instanceof Errors.FilesOutOfSyncError) { + logger.warn({project_id: request.project_id, user_id: request.user_id}, "files out of sync, please retry"); + return callback(error); + } else if (error != null) { + logger.err({err:error, project_id: request.project_id, user_id: request.user_id}, "error writing resources to disk"); + return callback(error); + } + logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start}, "written files to disk"); + timer.done(); - injectDraftModeIfRequired = (callback) -> - if request.draft - DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback - else - callback() + const injectDraftModeIfRequired = function(callback) { + if (request.draft) { + return DraftModeManager.injectDraftMode(Path.join(compileDir, request.rootResourcePath), callback); + } else { + return callback(); + } + }; - createTikzFileIfRequired = (callback) -> - TikzManager.checkMainFile compileDir, request.rootResourcePath, resourceList, (error, needsMainFile) -> - return callback(error) if error? - if needsMainFile - TikzManager.injectOutputFile compileDir, request.rootResourcePath, callback - else - callback() + const createTikzFileIfRequired = callback => + TikzManager.checkMainFile(compileDir, request.rootResourcePath, resourceList, function(error, needsMainFile) { + if (error != null) { return callback(error); } + if (needsMainFile) { + return TikzManager.injectOutputFile(compileDir, request.rootResourcePath, callback); + } else { + return callback(); + } + }) + ; - # set up environment variables for chktex - env = {} - # only run chktex on LaTeX files (not knitr .Rtex files or any others) - isLaTeXFile = request.rootResourcePath?.match(/\.tex$/i) - if request.check? and isLaTeXFile - env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16' - env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000' - if request.check is 'error' - env['CHKTEX_EXIT_ON_ERROR'] = 1 - if request.check is 'validate' - env['CHKTEX_VALIDATE'] = 1 + // set up environment variables for chktex + const env = {}; + // only run chktex on LaTeX files (not knitr .Rtex files or any others) + const isLaTeXFile = request.rootResourcePath != null ? request.rootResourcePath.match(/\.tex$/i) : undefined; + if ((request.check != null) && isLaTeXFile) { + env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16'; + env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000'; + if (request.check === 'error') { + env['CHKTEX_EXIT_ON_ERROR'] = 1; + } + if (request.check === 'validate') { + env['CHKTEX_VALIDATE'] = 1; + } + } - # apply a series of file modifications/creations for draft mode and tikz - async.series [injectDraftModeIfRequired, createTikzFileIfRequired], (error) -> - return callback(error) if error? - timer = new Metrics.Timer("run-compile") - # find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite) - tag = request.imageName?.match(/:(.*)/)?[1]?.replace(/\./g,'-') or "default" - tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test - Metrics.inc("compiles") - Metrics.inc("compiles-with-image.#{tag}") - compileName = getCompileName(request.project_id, request.user_id) - LatexRunner.runLatex compileName, { - directory: compileDir - mainFile: request.rootResourcePath - compiler: request.compiler - timeout: request.timeout - image: request.imageName - flags: request.flags + // apply a series of file modifications/creations for draft mode and tikz + return async.series([injectDraftModeIfRequired, createTikzFileIfRequired], function(error) { + if (error != null) { return callback(error); } + timer = new Metrics.Timer("run-compile"); + // find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite) + let tag = __guard__(__guard__(request.imageName != null ? request.imageName.match(/:(.*)/) : undefined, x1 => x1[1]), x => x.replace(/\./g,'-')) || "default"; + if (!request.project_id.match(/^[0-9a-f]{24}$/)) { tag = "other"; } // exclude smoke test + Metrics.inc("compiles"); + Metrics.inc(`compiles-with-image.${tag}`); + const compileName = getCompileName(request.project_id, request.user_id); + return LatexRunner.runLatex(compileName, { + directory: compileDir, + mainFile: request.rootResourcePath, + compiler: request.compiler, + timeout: request.timeout, + image: request.imageName, + flags: request.flags, environment: env - }, (error, output, stats, timings) -> - # request was for validation only - if request.check is "validate" - result = if error?.code then "fail" else "pass" - error = new Error("validation") - error.validate = result - # request was for compile, and failed on validation - if request.check is "error" and error?.message is 'exited' - error = new Error("compilation") - error.validate = "fail" - # compile was killed by user, was a validation, or a compile which failed validation - if error?.terminated or error?.validate or error?.timedout - OutputFileFinder.findOutputFiles resourceList, compileDir, (err, outputFiles) -> - return callback(err) if err? - error.outputFiles = outputFiles # return output files so user can check logs - callback(error) - return - # compile completed normally - return callback(error) if error? - Metrics.inc("compiles-succeeded") - for metric_key, metric_value of stats or {} - Metrics.count(metric_key, metric_value) - for metric_key, metric_value of timings or {} - Metrics.timing(metric_key, metric_value) - loadavg = os.loadavg?() - Metrics.gauge("load-avg", loadavg[0]) if loadavg? - ts = timer.done() - logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" - if stats?["latex-runs"] > 0 - Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]) - if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0 - Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]) + }, function(error, output, stats, timings) { + // request was for validation only + let metric_key, metric_value; + if (request.check === "validate") { + const result = (error != null ? error.code : undefined) ? "fail" : "pass"; + error = new Error("validation"); + error.validate = result; + } + // request was for compile, and failed on validation + if ((request.check === "error") && ((error != null ? error.message : undefined) === 'exited')) { + error = new Error("compilation"); + error.validate = "fail"; + } + // compile was killed by user, was a validation, or a compile which failed validation + if ((error != null ? error.terminated : undefined) || (error != null ? error.validate : undefined) || (error != null ? error.timedout : undefined)) { + OutputFileFinder.findOutputFiles(resourceList, compileDir, function(err, outputFiles) { + if (err != null) { return callback(err); } + error.outputFiles = outputFiles; // return output files so user can check logs + return callback(error); + }); + return; + } + // compile completed normally + if (error != null) { return callback(error); } + Metrics.inc("compiles-succeeded"); + const object = stats || {}; + for (metric_key in object) { + metric_value = object[metric_key]; + Metrics.count(metric_key, metric_value); + } + const object1 = timings || {}; + for (metric_key in object1) { + metric_value = object1[metric_key]; + Metrics.timing(metric_key, metric_value); + } + const loadavg = typeof os.loadavg === 'function' ? os.loadavg() : undefined; + if (loadavg != null) { Metrics.gauge("load-avg", loadavg[0]); } + const ts = timer.done(); + logger.log({project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats, timings, loadavg}, "done compile"); + if ((stats != null ? stats["latex-runs"] : undefined) > 0) { + Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]); + } + if (((stats != null ? stats["latex-runs"] : undefined) > 0) && ((timings != null ? timings["cpu-time"] : undefined) > 0)) { + Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"]); + } - OutputFileFinder.findOutputFiles resourceList, compileDir, (error, outputFiles) -> - return callback(error) if error? - OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> - callback null, newOutputFiles + return OutputFileFinder.findOutputFiles(resourceList, compileDir, function(error, outputFiles) { + if (error != null) { return callback(error); } + return OutputCacheManager.saveOutputFiles(outputFiles, compileDir, (error, newOutputFiles) => callback(null, newOutputFiles)); + }); + }); + }); + }); + }, - stopCompile: (project_id, user_id, callback = (error) ->) -> - compileName = getCompileName(project_id, user_id) - LatexRunner.killLatex compileName, callback + stopCompile(project_id, user_id, callback) { + if (callback == null) { callback = function(error) {}; } + const compileName = getCompileName(project_id, user_id); + return LatexRunner.killLatex(compileName, callback); + }, - clearProject: (project_id, user_id, _callback = (error) ->) -> - callback = (error) -> - _callback(error) - _callback = () -> + clearProject(project_id, user_id, _callback) { + if (_callback == null) { _callback = function(error) {}; } + const callback = function(error) { + _callback(error); + return _callback = function() {}; + }; - compileDir = getCompileDir(project_id, user_id) + const compileDir = getCompileDir(project_id, user_id); - CompileManager._checkDirectory compileDir, (err, exists) -> - return callback(err) if err? - return callback() if not exists # skip removal if no directory present + return CompileManager._checkDirectory(compileDir, function(err, exists) { + if (err != null) { return callback(err); } + if (!exists) { return callback(); } // skip removal if no directory present - proc = child_process.spawn "rm", ["-r", compileDir] + const proc = child_process.spawn("rm", ["-r", compileDir]); - proc.on "error", callback + proc.on("error", callback); - stderr = "" - proc.stderr.on "data", (chunk) -> stderr += chunk.toString() + let stderr = ""; + proc.stderr.on("data", chunk => stderr += chunk.toString()); - proc.on "close", (code) -> - if code == 0 - return callback(null) - else - return callback(new Error("rm -r #{compileDir} failed: #{stderr}")) + return proc.on("close", function(code) { + if (code === 0) { + return callback(null); + } else { + return callback(new Error(`rm -r ${compileDir} failed: ${stderr}`)); + } + }); + }); + }, - _findAllDirs: (callback = (error, allDirs) ->) -> - root = Settings.path.compilesDir - fs.readdir root, (err, files) -> - return callback(err) if err? - allDirs = (Path.join(root, file) for file in files) - callback(null, allDirs) + _findAllDirs(callback) { + if (callback == null) { callback = function(error, allDirs) {}; } + const root = Settings.path.compilesDir; + return fs.readdir(root, function(err, files) { + if (err != null) { return callback(err); } + const allDirs = (Array.from(files).map((file) => Path.join(root, file))); + return callback(null, allDirs); + }); + }, - clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) -> - now = Date.now() - # action for each directory - expireIfNeeded = (checkDir, cb) -> - fs.stat checkDir, (err, stats) -> - return cb() if err? # ignore errors checking directory - age = now - stats.mtime - hasExpired = (age > max_cache_age_ms) - if hasExpired then fse.remove(checkDir, cb) else cb() - # iterate over all project directories - CompileManager._findAllDirs (error, allDirs) -> - return callback() if error? - async.eachSeries allDirs, expireIfNeeded, callback + clearExpiredProjects(max_cache_age_ms, callback) { + if (callback == null) { callback = function(error) {}; } + const now = Date.now(); + // action for each directory + const expireIfNeeded = (checkDir, cb) => + fs.stat(checkDir, function(err, stats) { + if (err != null) { return cb(); } // ignore errors checking directory + const age = now - stats.mtime; + const hasExpired = (age > max_cache_age_ms); + if (hasExpired) { return fse.remove(checkDir, cb); } else { return cb(); } + }) + ; + // iterate over all project directories + return CompileManager._findAllDirs(function(error, allDirs) { + if (error != null) { return callback(); } + return async.eachSeries(allDirs, expireIfNeeded, callback); + }); + }, - _checkDirectory: (compileDir, callback = (error, exists) ->) -> - fs.lstat compileDir, (err, stats) -> - if err?.code is 'ENOENT' - return callback(null, false) # directory does not exist - else if err? - logger.err {dir: compileDir, err:err}, "error on stat of project directory for removal" - return callback(err) - else if not stats?.isDirectory() - logger.err {dir: compileDir, stats:stats}, "bad project directory for removal" - return callback new Error("project directory is not directory") - else - callback(null, true) # directory exists + _checkDirectory(compileDir, callback) { + if (callback == null) { callback = function(error, exists) {}; } + return fs.lstat(compileDir, function(err, stats) { + if ((err != null ? err.code : undefined) === 'ENOENT') { + return callback(null, false); // directory does not exist + } else if (err != null) { + logger.err({dir: compileDir, err}, "error on stat of project directory for removal"); + return callback(err); + } else if (!(stats != null ? stats.isDirectory() : undefined)) { + logger.err({dir: compileDir, stats}, "bad project directory for removal"); + return callback(new Error("project directory is not directory")); + } else { + return callback(null, true); + } + }); + }, // directory exists - syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) -> - # If LaTeX was run in a virtual environment, the file path that synctex expects - # might not match the file path on the host. The .synctex.gz file however, will be accessed - # wherever it is on the host. - compileName = getCompileName(project_id, user_id) - base_dir = Settings.path.synctexBaseDir(compileName) - file_path = base_dir + "/" + file_name - compileDir = getCompileDir(project_id, user_id) - synctex_path = "#{base_dir}/output.pdf" - command = ["code", synctex_path, file_path, line, column] - fse.ensureDir compileDir, (error) -> - if error? - logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code" - return callback(error) - CompileManager._runSynctex project_id, user_id, command, (error, stdout) -> - return callback(error) if error? - logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output" - callback null, CompileManager._parseSynctexFromCodeOutput(stdout) + syncFromCode(project_id, user_id, file_name, line, column, callback) { + // If LaTeX was run in a virtual environment, the file path that synctex expects + // might not match the file path on the host. The .synctex.gz file however, will be accessed + // wherever it is on the host. + if (callback == null) { callback = function(error, pdfPositions) {}; } + const compileName = getCompileName(project_id, user_id); + const base_dir = Settings.path.synctexBaseDir(compileName); + const file_path = base_dir + "/" + file_name; + const compileDir = getCompileDir(project_id, user_id); + const synctex_path = `${base_dir}/output.pdf`; + const command = ["code", synctex_path, file_path, line, column]; + return fse.ensureDir(compileDir, function(error) { + if (error != null) { + logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code"); + return callback(error); + } + return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) { + if (error != null) { return callback(error); } + logger.log({project_id, user_id, file_name, line, column, command, stdout}, "synctex code output"); + return callback(null, CompileManager._parseSynctexFromCodeOutput(stdout)); + }); + }); + }, - syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) -> - compileName = getCompileName(project_id, user_id) - compileDir = getCompileDir(project_id, user_id) - base_dir = Settings.path.synctexBaseDir(compileName) - synctex_path = "#{base_dir}/output.pdf" - command = ["pdf", synctex_path, page, h, v] - fse.ensureDir compileDir, (error) -> - if error? - logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync to code" - return callback(error) - CompileManager._runSynctex project_id, user_id, command, (error, stdout) -> - return callback(error) if error? - logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" - callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir) + syncFromPdf(project_id, user_id, page, h, v, callback) { + if (callback == null) { callback = function(error, filePositions) {}; } + const compileName = getCompileName(project_id, user_id); + const compileDir = getCompileDir(project_id, user_id); + const base_dir = Settings.path.synctexBaseDir(compileName); + const synctex_path = `${base_dir}/output.pdf`; + const command = ["pdf", synctex_path, page, h, v]; + return fse.ensureDir(compileDir, function(error) { + if (error != null) { + logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync to code"); + return callback(error); + } + return CompileManager._runSynctex(project_id, user_id, command, function(error, stdout) { + if (error != null) { return callback(error); } + logger.log({project_id, user_id, page, h, v, stdout}, "synctex pdf output"); + return callback(null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)); + }); + }); + }, - _checkFileExists: (path, callback = (error) ->) -> - synctexDir = Path.dirname(path) - synctexFile = Path.join(synctexDir, "output.synctex.gz") - fs.stat synctexDir, (error, stats) -> - if error?.code is 'ENOENT' - return callback(new Errors.NotFoundError("called synctex with no output directory")) - return callback(error) if error? - fs.stat synctexFile, (error, stats) -> - if error?.code is 'ENOENT' - return callback(new Errors.NotFoundError("called synctex with no output file")) - return callback(error) if error? - return callback(new Error("not a file")) if not stats?.isFile() - callback() + _checkFileExists(path, callback) { + if (callback == null) { callback = function(error) {}; } + const synctexDir = Path.dirname(path); + const synctexFile = Path.join(synctexDir, "output.synctex.gz"); + return fs.stat(synctexDir, function(error, stats) { + if ((error != null ? error.code : undefined) === 'ENOENT') { + return callback(new Errors.NotFoundError("called synctex with no output directory")); + } + if (error != null) { return callback(error); } + return fs.stat(synctexFile, function(error, stats) { + if ((error != null ? error.code : undefined) === 'ENOENT') { + return callback(new Errors.NotFoundError("called synctex with no output file")); + } + if (error != null) { return callback(error); } + if (!(stats != null ? stats.isFile() : undefined)) { return callback(new Error("not a file")); } + return callback(); + }); + }); + }, - _runSynctex: (project_id, user_id, command, callback = (error, stdout) ->) -> - seconds = 1000 + _runSynctex(project_id, user_id, command, callback) { + if (callback == null) { callback = function(error, stdout) {}; } + const seconds = 1000; - command.unshift("/opt/synctex") + command.unshift("/opt/synctex"); - directory = getCompileDir(project_id, user_id) - timeout = 60 * 1000 # increased to allow for large projects - compileName = getCompileName(project_id, user_id) - CommandRunner.run compileName, command, directory, Settings.clsi?.docker.image, timeout, {}, (error, output) -> - if error? - logger.err err:error, command:command, project_id:project_id, user_id:user_id, "error running synctex" - return callback(error) - callback(null, output.stdout) + const directory = getCompileDir(project_id, user_id); + const timeout = 60 * 1000; // increased to allow for large projects + const compileName = getCompileName(project_id, user_id); + return CommandRunner.run(compileName, command, directory, Settings.clsi != null ? Settings.clsi.docker.image : undefined, timeout, {}, function(error, output) { + if (error != null) { + logger.err({err:error, command, project_id, user_id}, "error running synctex"); + return callback(error); + } + return callback(null, output.stdout); + }); + }, - _parseSynctexFromCodeOutput: (output) -> - results = [] - for line in output.split("\n") - [node, page, h, v, width, height] = line.split("\t") - if node == "NODE" - results.push { - page: parseInt(page, 10) - h: parseFloat(h) - v: parseFloat(v) - height: parseFloat(height) + _parseSynctexFromCodeOutput(output) { + const results = []; + for (let line of Array.from(output.split("\n"))) { + const [node, page, h, v, width, height] = Array.from(line.split("\t")); + if (node === "NODE") { + results.push({ + page: parseInt(page, 10), + h: parseFloat(h), + v: parseFloat(v), + height: parseFloat(height), width: parseFloat(width) - } - return results - - _parseSynctexFromPdfOutput: (output, base_dir) -> - results = [] - for line in output.split("\n") - [node, file_path, line, column] = line.split("\t") - if node == "NODE" - file = file_path.slice(base_dir.length + 1) - results.push { - file: file - line: parseInt(line, 10) - column: parseInt(column, 10) - } - return results - - - wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) -> - logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount" - file_path = "$COMPILE_DIR/" + file_name - command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"] - compileDir = getCompileDir(project_id, user_id) - timeout = 60 * 1000 - compileName = getCompileName(project_id, user_id) - fse.ensureDir compileDir, (error) -> - if error? - logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code" - return callback(error) - CommandRunner.run compileName, command, compileDir, image, timeout, {}, (error) -> - return callback(error) if error? - fs.readFile compileDir + "/" + file_name + ".wc", "utf-8", (err, stdout) -> - if err? - #call it node_err so sentry doesn't use random path error as unique id so it can't be ignored - logger.err node_err:err, command:command, compileDir:compileDir, project_id:project_id, user_id:user_id, "error reading word count output" - return callback(err) - results = CompileManager._parseWordcountFromOutput(stdout) - logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results" - callback null, results - - _parseWordcountFromOutput: (output) -> - results = { - encode: "" - textWords: 0 - headWords: 0 - outside: 0 - headers: 0 - elements: 0 - mathInline: 0 - mathDisplay: 0 - errors: 0 - messages: "" + }); + } } - for line in output.split("\n") - [data, info] = line.split(":") - if data.indexOf("Encoding") > -1 - results['encode'] = info.trim() - if data.indexOf("in text") > -1 - results['textWords'] = parseInt(info, 10) - if data.indexOf("in head") > -1 - results['headWords'] = parseInt(info, 10) - if data.indexOf("outside") > -1 - results['outside'] = parseInt(info, 10) - if data.indexOf("of head") > -1 - results['headers'] = parseInt(info, 10) - if data.indexOf("Number of floats/tables/figures") > -1 - results['elements'] = parseInt(info, 10) - if data.indexOf("Number of math inlines") > -1 - results['mathInline'] = parseInt(info, 10) - if data.indexOf("Number of math displayed") > -1 - results['mathDisplay'] = parseInt(info, 10) - if data is "(errors" # errors reported as (errors:123) - results['errors'] = parseInt(info, 10) - if line.indexOf("!!! ") > -1 # errors logged as !!! message !!! - results['messages'] += line + "\n" - return results + return results; + }, + + _parseSynctexFromPdfOutput(output, base_dir) { + const results = []; + for (let line of Array.from(output.split("\n"))) { + let column, file_path, node; + [node, file_path, line, column] = Array.from(line.split("\t")); + if (node === "NODE") { + const file = file_path.slice(base_dir.length + 1); + results.push({ + file, + line: parseInt(line, 10), + column: parseInt(column, 10) + }); + } + } + return results; + }, + + + wordcount(project_id, user_id, file_name, image, callback) { + if (callback == null) { callback = function(error, pdfPositions) {}; } + logger.log({project_id, user_id, file_name, image}, "running wordcount"); + const file_path = `$COMPILE_DIR/${file_name}`; + const command = [ "texcount", '-nocol', '-inc', file_path, `-out=${file_path}.wc`]; + const compileDir = getCompileDir(project_id, user_id); + const timeout = 60 * 1000; + const compileName = getCompileName(project_id, user_id); + return fse.ensureDir(compileDir, function(error) { + if (error != null) { + logger.err({error, project_id, user_id, file_name}, "error ensuring dir for sync from code"); + return callback(error); + } + return CommandRunner.run(compileName, command, compileDir, image, timeout, {}, function(error) { + if (error != null) { return callback(error); } + return fs.readFile(compileDir + "/" + file_name + ".wc", "utf-8", function(err, stdout) { + if (err != null) { + //call it node_err so sentry doesn't use random path error as unique id so it can't be ignored + logger.err({node_err:err, command, compileDir, project_id, user_id}, "error reading word count output"); + return callback(err); + } + const results = CompileManager._parseWordcountFromOutput(stdout); + logger.log({project_id, user_id, wordcount: results}, "word count results"); + return callback(null, results); + }); + }); + }); + }, + + _parseWordcountFromOutput(output) { + const results = { + encode: "", + textWords: 0, + headWords: 0, + outside: 0, + headers: 0, + elements: 0, + mathInline: 0, + mathDisplay: 0, + errors: 0, + messages: "" + }; + for (let line of Array.from(output.split("\n"))) { + const [data, info] = Array.from(line.split(":")); + if (data.indexOf("Encoding") > -1) { + results['encode'] = info.trim(); + } + if (data.indexOf("in text") > -1) { + results['textWords'] = parseInt(info, 10); + } + if (data.indexOf("in head") > -1) { + results['headWords'] = parseInt(info, 10); + } + if (data.indexOf("outside") > -1) { + results['outside'] = parseInt(info, 10); + } + if (data.indexOf("of head") > -1) { + results['headers'] = parseInt(info, 10); + } + if (data.indexOf("Number of floats/tables/figures") > -1) { + results['elements'] = parseInt(info, 10); + } + if (data.indexOf("Number of math inlines") > -1) { + results['mathInline'] = parseInt(info, 10); + } + if (data.indexOf("Number of math displayed") > -1) { + results['mathDisplay'] = parseInt(info, 10); + } + if (data === "(errors") { // errors reported as (errors:123) + results['errors'] = parseInt(info, 10); + } + if (line.indexOf("!!! ") > -1) { // errors logged as !!! message !!! + results['messages'] += line + "\n"; + } + } + return results; + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/app/coffee/ContentTypeMapper.js b/app/coffee/ContentTypeMapper.js index 68b2d14..c57057f 100644 --- a/app/coffee/ContentTypeMapper.js +++ b/app/coffee/ContentTypeMapper.js @@ -1,24 +1,28 @@ -Path = require 'path' +let ContentTypeMapper; +const Path = require('path'); -# here we coerce html, css and js to text/plain, -# otherwise choose correct mime type based on file extension, -# falling back to octet-stream -module.exports = ContentTypeMapper = - map: (path) -> - switch Path.extname(path) - when '.txt', '.html', '.js', '.css', '.svg' - return 'text/plain' - when '.csv' - return 'text/csv' - when '.pdf' - return 'application/pdf' - when '.png' - return 'image/png' - when '.jpg', '.jpeg' - return 'image/jpeg' - when '.tiff' - return 'image/tiff' - when '.gif' - return 'image/gif' - else - return 'application/octet-stream' +// here we coerce html, css and js to text/plain, +// otherwise choose correct mime type based on file extension, +// falling back to octet-stream +module.exports = (ContentTypeMapper = { + map(path) { + switch (Path.extname(path)) { + case '.txt': case '.html': case '.js': case '.css': case '.svg': + return 'text/plain'; + case '.csv': + return 'text/csv'; + case '.pdf': + return 'application/pdf'; + case '.png': + return 'image/png'; + case '.jpg': case '.jpeg': + return 'image/jpeg'; + case '.tiff': + return 'image/tiff'; + case '.gif': + return 'image/gif'; + default: + return 'application/octet-stream'; + } + } +}); diff --git a/app/coffee/DbQueue.js b/app/coffee/DbQueue.js index a3593fd..0f1f8cf 100644 --- a/app/coffee/DbQueue.js +++ b/app/coffee/DbQueue.js @@ -1,13 +1,16 @@ -async = require "async" -Settings = require "settings-sharelatex" -logger = require("logger-sharelatex") -queue = async.queue((task, cb)-> - task(cb) - , Settings.parallelSqlQueryLimit) +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const async = require("async"); +const Settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const queue = async.queue((task, cb)=> task(cb) + , Settings.parallelSqlQueryLimit); -queue.drain = ()-> - logger.debug('all items have been processed') +queue.drain = ()=> logger.debug('all items have been processed'); module.exports = - queue: queue + {queue}; diff --git a/app/coffee/DockerLockManager.js b/app/coffee/DockerLockManager.js index bf90f02..9c7deff 100644 --- a/app/coffee/DockerLockManager.js +++ b/app/coffee/DockerLockManager.js @@ -1,56 +1,84 @@ -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let LockManager; +const logger = require("logger-sharelatex"); -LockState = {} # locks for docker container operations, by container name +const LockState = {}; // locks for docker container operations, by container name -module.exports = LockManager = +module.exports = (LockManager = { - MAX_LOCK_HOLD_TIME: 15000 # how long we can keep a lock - MAX_LOCK_WAIT_TIME: 10000 # how long we wait for a lock - LOCK_TEST_INTERVAL: 1000 # retry time + MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock + MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock + LOCK_TEST_INTERVAL: 1000, // retry time - tryLock: (key, callback = (err, gotLock) ->) -> - existingLock = LockState[key] - if existingLock? # the lock is already taken, check how old it is - lockAge = Date.now() - existingLock.created - if lockAge < LockManager.MAX_LOCK_HOLD_TIME - return callback(null, false) # we didn't get the lock, bail out - else - logger.error {key: key, lock: existingLock, age:lockAge}, "taking old lock by force" - # take the lock - LockState[key] = lockValue = {created: Date.now()} - callback(null, true, lockValue) + tryLock(key, callback) { + let lockValue; + if (callback == null) { callback = function(err, gotLock) {}; } + const existingLock = LockState[key]; + if (existingLock != null) { // the lock is already taken, check how old it is + const lockAge = Date.now() - existingLock.created; + if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) { + return callback(null, false); // we didn't get the lock, bail out + } else { + logger.error({key, lock: existingLock, age:lockAge}, "taking old lock by force"); + } + } + // take the lock + LockState[key] = (lockValue = {created: Date.now()}); + return callback(null, true, lockValue); + }, - getLock: (key, callback = (error, lockValue) ->) -> - startTime = Date.now() - do attempt = () -> - LockManager.tryLock key, (error, gotLock, lockValue) -> - return callback(error) if error? - if gotLock - callback(null, lockValue) - else if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME - e = new Error("Lock timeout") - e.key = key - return callback(e) - else - setTimeout attempt, LockManager.LOCK_TEST_INTERVAL + getLock(key, callback) { + let attempt; + if (callback == null) { callback = function(error, lockValue) {}; } + const startTime = Date.now(); + return (attempt = () => + LockManager.tryLock(key, function(error, gotLock, lockValue) { + if (error != null) { return callback(error); } + if (gotLock) { + return callback(null, lockValue); + } else if ((Date.now() - startTime) > LockManager.MAX_LOCK_WAIT_TIME) { + const e = new Error("Lock timeout"); + e.key = key; + return callback(e); + } else { + return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL); + } + }) + )(); + }, - releaseLock: (key, lockValue, callback = (error) ->) -> - existingLock = LockState[key] - if existingLock is lockValue # lockValue is an object, so we can test by reference - delete LockState[key] # our lock, so we can free it - callback() - else if existingLock? # lock exists but doesn't match ours - logger.error {key:key, lock: existingLock}, "tried to release lock taken by force" - callback() - else - logger.error {key:key, lock: existingLock}, "tried to release lock that has gone" - callback() + releaseLock(key, lockValue, callback) { + if (callback == null) { callback = function(error) {}; } + const existingLock = LockState[key]; + if (existingLock === lockValue) { // lockValue is an object, so we can test by reference + delete LockState[key]; // our lock, so we can free it + return callback(); + } else if (existingLock != null) { // lock exists but doesn't match ours + logger.error({key, lock: existingLock}, "tried to release lock taken by force"); + return callback(); + } else { + logger.error({key, lock: existingLock}, "tried to release lock that has gone"); + return callback(); + } + }, - runWithLock: (key, runner, callback = ( (error) -> )) -> - LockManager.getLock key, (error, lockValue) -> - return callback(error) if error? - runner (error1, args...) -> - LockManager.releaseLock key, lockValue, (error2) -> - error = error1 or error2 - return callback(error) if error? - callback(null, args...) + runWithLock(key, runner, callback) { + if (callback == null) { callback = function(error) {}; } + return LockManager.getLock(key, function(error, lockValue) { + if (error != null) { return callback(error); } + return runner((error1, ...args) => + LockManager.releaseLock(key, lockValue, function(error2) { + error = error1 || error2; + if (error != null) { return callback(error); } + return callback(null, ...Array.from(args)); + }) + ); + }); + } +}); diff --git a/app/coffee/DockerRunner.js b/app/coffee/DockerRunner.js index 6ea929f..ab78419 100644 --- a/app/coffee/DockerRunner.js +++ b/app/coffee/DockerRunner.js @@ -1,358 +1,475 @@ -Settings = require "settings-sharelatex" -logger = require "logger-sharelatex" -Docker = require("dockerode") -dockerode = new Docker() -crypto = require "crypto" -async = require "async" -LockManager = require "./DockerLockManager" -fs = require "fs" -Path = require 'path' -_ = require "underscore" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let DockerRunner, oneHour; +const Settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const Docker = require("dockerode"); +const dockerode = new Docker(); +const crypto = require("crypto"); +const async = require("async"); +const LockManager = require("./DockerLockManager"); +const fs = require("fs"); +const Path = require('path'); +const _ = require("underscore"); -logger.info "using docker runner" +logger.info("using docker runner"); -usingSiblingContainers = () -> - Settings?.path?.sandboxedCompilesHostDir? +const usingSiblingContainers = () => __guard__(Settings != null ? Settings.path : undefined, x => x.sandboxedCompilesHostDir) != null; -module.exports = DockerRunner = - ERR_NOT_DIRECTORY: new Error("not a directory") - ERR_TERMINATED: new Error("terminated") - ERR_EXITED: new Error("exited") - ERR_TIMED_OUT: new Error("container timed out") +module.exports = (DockerRunner = { + ERR_NOT_DIRECTORY: new Error("not a directory"), + ERR_TERMINATED: new Error("terminated"), + ERR_EXITED: new Error("exited"), + ERR_TIMED_OUT: new Error("container timed out"), - run: (project_id, command, directory, image, timeout, environment, callback = (error, output) ->) -> + run(project_id, command, directory, image, timeout, environment, callback) { - if usingSiblingContainers() - _newPath = Settings.path.sandboxedCompilesHostDir - logger.log {path: _newPath}, "altering bind path for sibling containers" - # Server Pro, example: - # '/var/lib/sharelatex/data/compiles/' - # ... becomes ... - # '/opt/sharelatex_data/data/compiles/' - directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory)) + let name; + if (callback == null) { callback = function(error, output) {}; } + if (usingSiblingContainers()) { + const _newPath = Settings.path.sandboxedCompilesHostDir; + logger.log({path: _newPath}, "altering bind path for sibling containers"); + // Server Pro, example: + // '/var/lib/sharelatex/data/compiles/' + // ... becomes ... + // '/opt/sharelatex_data/data/compiles/' + directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory)); + } - volumes = {} - volumes[directory] = "/compile" + const volumes = {}; + volumes[directory] = "/compile"; - command = (arg.toString().replace?('$COMPILE_DIR', "/compile") for arg in command) - if !image? - image = Settings.clsi.docker.image + command = (Array.from(command).map((arg) => __guardMethod__(arg.toString(), 'replace', o => o.replace('$COMPILE_DIR', "/compile")))); + if ((image == null)) { + ({ image } = Settings.clsi.docker); + } - if Settings.texliveImageNameOveride? - img = image.split("/") - image = "#{Settings.texliveImageNameOveride}/#{img[2]}" + if (Settings.texliveImageNameOveride != null) { + const img = image.split("/"); + image = `${Settings.texliveImageNameOveride}/${img[2]}`; + } - options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment) - fingerprint = DockerRunner._fingerprintContainer(options) - options.name = name = "project-#{project_id}-#{fingerprint}" + const options = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment); + const fingerprint = DockerRunner._fingerprintContainer(options); + options.name = (name = `project-${project_id}-${fingerprint}`); - # logOptions = _.clone(options) - # logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging" - logger.log project_id: project_id, "running docker container" - DockerRunner._runAndWaitForContainer options, volumes, timeout, (error, output) -> - if error?.message?.match("HTTP code is 500") - logger.log err: error, project_id: project_id, "error running container so destroying and retrying" - DockerRunner.destroyContainer name, null, true, (error) -> - return callback(error) if error? - DockerRunner._runAndWaitForContainer options, volumes, timeout, callback - else - callback(error, output) + // logOptions = _.clone(options) + // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging" + logger.log({project_id}, "running docker container"); + DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(error, output) { + if (__guard__(error != null ? error.message : undefined, x => x.match("HTTP code is 500"))) { + logger.log({err: error, project_id}, "error running container so destroying and retrying"); + return DockerRunner.destroyContainer(name, null, true, function(error) { + if (error != null) { return callback(error); } + return DockerRunner._runAndWaitForContainer(options, volumes, timeout, callback); + }); + } else { + return callback(error, output); + } + }); - return name # pass back the container name to allow it to be killed + return name; + }, // pass back the container name to allow it to be killed - kill: (container_id, callback = (error) ->) -> - logger.log container_id: container_id, "sending kill signal to container" - container = dockerode.getContainer(container_id) - container.kill (error) -> - if error? and error?.message?.match?(/Cannot kill container .* is not running/) - logger.warn err: error, container_id: container_id, "container not running, continuing" - error = null - if error? - logger.error err: error, container_id: container_id, "error killing container" - return callback(error) - else - callback() + kill(container_id, callback) { + if (callback == null) { callback = function(error) {}; } + logger.log({container_id}, "sending kill signal to container"); + const container = dockerode.getContainer(container_id); + return container.kill(function(error) { + if ((error != null) && __guardMethod__(error != null ? error.message : undefined, 'match', o => o.match(/Cannot kill container .* is not running/))) { + logger.warn({err: error, container_id}, "container not running, continuing"); + error = null; + } + if (error != null) { + logger.error({err: error, container_id}, "error killing container"); + return callback(error); + } else { + return callback(); + } + }); + }, - _runAndWaitForContainer: (options, volumes, timeout, _callback = (error, output) ->) -> - callback = (args...) -> - _callback(args...) - # Only call the callback once - _callback = () -> + _runAndWaitForContainer(options, volumes, timeout, _callback) { + if (_callback == null) { _callback = function(error, output) {}; } + const callback = function(...args) { + _callback(...Array.from(args || [])); + // Only call the callback once + return _callback = function() {}; + }; - name = options.name + const { name } = options; - streamEnded = false - containerReturned = false - output = {} + let streamEnded = false; + let containerReturned = false; + let output = {}; - callbackIfFinished = () -> - if streamEnded and containerReturned - callback(null, output) + const callbackIfFinished = function() { + if (streamEnded && containerReturned) { + return callback(null, output); + } + }; - attachStreamHandler = (error, _output) -> - return callback(error) if error? - output = _output - streamEnded = true - callbackIfFinished() + const attachStreamHandler = function(error, _output) { + if (error != null) { return callback(error); } + output = _output; + streamEnded = true; + return callbackIfFinished(); + }; - DockerRunner.startContainer options, volumes, attachStreamHandler, (error, containerId) -> - return callback(error) if error? + return DockerRunner.startContainer(options, volumes, attachStreamHandler, function(error, containerId) { + if (error != null) { return callback(error); } - DockerRunner.waitForContainer name, timeout, (error, exitCode) -> - return callback(error) if error? - if exitCode is 137 # exit status from kill -9 - err = DockerRunner.ERR_TERMINATED - err.terminated = true - return callback(err) - if exitCode is 1 # exit status from chktex - err = DockerRunner.ERR_EXITED - err.code = exitCode - return callback(err) - containerReturned = true - options?.HostConfig?.SecurityOpt = null #small log line - logger.log err:err, exitCode:exitCode, options:options, "docker container has exited" - callbackIfFinished() - - _getContainerOptions: (command, image, volumes, timeout, environment) -> - timeoutInSeconds = timeout / 1000 - - dockerVolumes = {} - for hostVol, dockerVol of volumes - dockerVolumes[dockerVol] = {} - - if volumes[hostVol].slice(-3).indexOf(":r") == -1 - volumes[hostVol] = "#{dockerVol}:rw" - - # merge settings and environment parameter - env = {} - for src in [Settings.clsi.docker.env, environment or {}] - env[key] = value for key, value of src - # set the path based on the image year - if m = image.match /:([0-9]+)\.[0-9]+/ - year = m[1] - else - year = "2014" - env['PATH'] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/#{year}/bin/x86_64-linux/" - options = - "Cmd" : command, - "Image" : image - "Volumes" : dockerVolumes - "WorkingDir" : "/compile" - "NetworkDisabled" : true - "Memory" : 1024 * 1024 * 1024 * 1024 # 1 Gb - "User" : Settings.clsi.docker.user - "Env" : ("#{key}=#{value}" for key, value of env) # convert the environment hash to an array - "HostConfig" : - "Binds": ("#{hostVol}:#{dockerVol}" for hostVol, dockerVol of volumes) - "LogConfig": {"Type": "none", "Config": {}} - "Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}] - "CapDrop": "ALL" - "SecurityOpt": ["no-new-privileges"] - - - if Settings.path?.synctexBinHostPath? - options["HostConfig"]["Binds"].push("#{Settings.path.synctexBinHostPath}:/opt/synctex:ro") - - if Settings.clsi.docker.seccomp_profile? - options.HostConfig.SecurityOpt.push "seccomp=#{Settings.clsi.docker.seccomp_profile}" - - return options - - _fingerprintContainer: (containerOptions) -> - # Yay, Hashing! - json = JSON.stringify(containerOptions) - return crypto.createHash("md5").update(json).digest("hex") - - startContainer: (options, volumes, attachStreamHandler, callback) -> - LockManager.runWithLock options.name, (releaseLock) -> - # Check that volumes exist before starting the container. - # When a container is started with volume pointing to a - # non-existent directory then docker creates the directory but - # with root ownership. - DockerRunner._checkVolumes options, volumes, (err) -> - return releaseLock(err) if err? - DockerRunner._startContainer options, volumes, attachStreamHandler, releaseLock - , callback - - # Check that volumes exist and are directories - _checkVolumes: (options, volumes, callback = (error, containerName) ->) -> - if usingSiblingContainers() - # Server Pro, with sibling-containers active, skip checks - return callback(null) - - checkVolume = (path, cb) -> - fs.stat path, (err, stats) -> - return cb(err) if err? - return cb(DockerRunner.ERR_NOT_DIRECTORY) if not stats?.isDirectory() - cb() - jobs = [] - for vol of volumes - do (vol) -> - jobs.push (cb) -> checkVolume(vol, cb) - async.series jobs, callback - - _startContainer: (options, volumes, attachStreamHandler, callback = ((error, output) ->)) -> - callback = _.once(callback) - name = options.name - - logger.log {container_name: name}, "starting container" - container = dockerode.getContainer(name) - - createAndStartContainer = -> - dockerode.createContainer options, (error, container) -> - return callback(error) if error? - startExistingContainer() - - startExistingContainer = -> - DockerRunner.attachToContainer options.name, attachStreamHandler, (error)-> - return callback(error) if error? - container.start (error) -> - if error? and error?.statusCode != 304 #already running - return callback(error) - else - callback() - - container.inspect (error, stats)-> - if error?.statusCode == 404 - createAndStartContainer() - else if error? - logger.err {container_name: name, error:error}, "unable to inspect container to start" - return callback(error) - else - startExistingContainer() - - - attachToContainer: (containerId, attachStreamHandler, attachStartCallback) -> - container = dockerode.getContainer(containerId) - container.attach {stdout: 1, stderr: 1, stream: 1}, (error, stream) -> - if error? - logger.error err: error, container_id: containerId, "error attaching to container" - return attachStartCallback(error) - else - attachStartCallback() - - - logger.log container_id: containerId, "attached to container" - - MAX_OUTPUT = 1024 * 1024 # limit output to 1MB - createStringOutputStream = (name) -> - return { - data: "" - overflowed: false - write: (data) -> - return if @overflowed - if @data.length < MAX_OUTPUT - @data += data - else - logger.error container_id: containerId, length: @data.length, maxLen: MAX_OUTPUT, "#{name} exceeds max size" - @data += "(...truncated at #{MAX_OUTPUT} chars...)" - @overflowed = true - # kill container if too much output - # docker.containers.kill(containerId, () ->) + return DockerRunner.waitForContainer(name, timeout, function(error, exitCode) { + let err; + if (error != null) { return callback(error); } + if (exitCode === 137) { // exit status from kill -9 + err = DockerRunner.ERR_TERMINATED; + err.terminated = true; + return callback(err); } + if (exitCode === 1) { // exit status from chktex + err = DockerRunner.ERR_EXITED; + err.code = exitCode; + return callback(err); + } + containerReturned = true; + __guard__(options != null ? options.HostConfig : undefined, x => x.SecurityOpt = null); //small log line + logger.log({err, exitCode, options}, "docker container has exited"); + return callbackIfFinished(); + }); + }); + }, - stdout = createStringOutputStream "stdout" - stderr = createStringOutputStream "stderr" + _getContainerOptions(command, image, volumes, timeout, environment) { + let m, year; + let key, value, hostVol, dockerVol; + const timeoutInSeconds = timeout / 1000; - container.modem.demuxStream(stream, stdout, stderr) + const dockerVolumes = {}; + for (hostVol in volumes) { + dockerVol = volumes[hostVol]; + dockerVolumes[dockerVol] = {}; - stream.on "error", (err) -> - logger.error err: err, container_id: containerId, "error reading from container stream" + if (volumes[hostVol].slice(-3).indexOf(":r") === -1) { + volumes[hostVol] = `${dockerVol}:rw`; + } + } - stream.on "end", () -> - attachStreamHandler null, {stdout: stdout.data, stderr: stderr.data} + // merge settings and environment parameter + const env = {}; + for (let src of [Settings.clsi.docker.env, environment || {}]) { + for (key in src) { value = src[key]; env[key] = value; } + } + // set the path based on the image year + if ((m = image.match(/:([0-9]+)\.[0-9]+/))) { + year = m[1]; + } else { + year = "2014"; + } + env['PATH'] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`; + const options = { + "Cmd" : command, + "Image" : image, + "Volumes" : dockerVolumes, + "WorkingDir" : "/compile", + "NetworkDisabled" : true, + "Memory" : 1024 * 1024 * 1024 * 1024, // 1 Gb + "User" : Settings.clsi.docker.user, + "Env" : (((() => { + const result = []; + for (key in env) { + value = env[key]; + result.push(`${key}=${value}`); + } + return result; + })())), // convert the environment hash to an array + "HostConfig" : { + "Binds": (((() => { + const result1 = []; + for (hostVol in volumes) { + dockerVol = volumes[hostVol]; + result1.push(`${hostVol}:${dockerVol}`); + } + return result1; + })())), + "LogConfig": {"Type": "none", "Config": {}}, + "Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}], + "CapDrop": "ALL", + "SecurityOpt": ["no-new-privileges"] + } + }; - waitForContainer: (containerId, timeout, _callback = (error, exitCode) ->) -> - callback = (args...) -> - _callback(args...) - # Only call the callback once - _callback = () -> - container = dockerode.getContainer(containerId) + if ((Settings.path != null ? Settings.path.synctexBinHostPath : undefined) != null) { + options["HostConfig"]["Binds"].push(`${Settings.path.synctexBinHostPath}:/opt/synctex:ro`); + } - timedOut = false - timeoutId = setTimeout () -> - timedOut = true - logger.log container_id: containerId, "timeout reached, killing container" - container.kill(() ->) - , timeout + if (Settings.clsi.docker.seccomp_profile != null) { + options.HostConfig.SecurityOpt.push(`seccomp=${Settings.clsi.docker.seccomp_profile}`); + } - logger.log container_id: containerId, "waiting for docker container" - container.wait (error, res) -> - if error? - clearTimeout timeoutId - logger.error err: error, container_id: containerId, "error waiting for container" - return callback(error) - if timedOut - logger.log containerId: containerId, "docker container timed out" - error = DockerRunner.ERR_TIMED_OUT - error.timedout = true - callback error - else - clearTimeout timeoutId - logger.log container_id: containerId, exitCode: res.StatusCode, "docker container returned" - callback null, res.StatusCode + return options; + }, - destroyContainer: (containerName, containerId, shouldForce, callback = (error) ->) -> - # We want the containerName for the lock and, ideally, the - # containerId to delete. There is a bug in the docker.io module - # where if you delete by name and there is an error, it throws an - # async exception, but if you delete by id it just does a normal - # error callback. We fall back to deleting by name if no id is - # supplied. - LockManager.runWithLock containerName, (releaseLock) -> - DockerRunner._destroyContainer containerId or containerName, shouldForce, releaseLock - , callback + _fingerprintContainer(containerOptions) { + // Yay, Hashing! + const json = JSON.stringify(containerOptions); + return crypto.createHash("md5").update(json).digest("hex"); + }, - _destroyContainer: (containerId, shouldForce, callback = (error) ->) -> - logger.log container_id: containerId, "destroying docker container" - container = dockerode.getContainer(containerId) - container.remove {force: shouldForce == true}, (error) -> - if error? and error?.statusCode == 404 - logger.warn err: error, container_id: containerId, "container not found, continuing" - error = null - if error? - logger.error err: error, container_id: containerId, "error destroying container" - else - logger.log container_id: containerId, "destroyed container" - callback(error) + startContainer(options, volumes, attachStreamHandler, callback) { + return LockManager.runWithLock(options.name, releaseLock => + // Check that volumes exist before starting the container. + // When a container is started with volume pointing to a + // non-existent directory then docker creates the directory but + // with root ownership. + DockerRunner._checkVolumes(options, volumes, function(err) { + if (err != null) { return releaseLock(err); } + return DockerRunner._startContainer(options, volumes, attachStreamHandler, releaseLock); + }) + + , callback); + }, - # handle expiry of docker containers + // Check that volumes exist and are directories + _checkVolumes(options, volumes, callback) { + if (callback == null) { callback = function(error, containerName) {}; } + if (usingSiblingContainers()) { + // Server Pro, with sibling-containers active, skip checks + return callback(null); + } - MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge or oneHour = 60 * 60 * 1000 + const checkVolume = (path, cb) => + fs.stat(path, function(err, stats) { + if (err != null) { return cb(err); } + if (!(stats != null ? stats.isDirectory() : undefined)) { return cb(DockerRunner.ERR_NOT_DIRECTORY); } + return cb(); + }) + ; + const jobs = []; + for (let vol in volumes) { + (vol => jobs.push(cb => checkVolume(vol, cb)))(vol); + } + return async.series(jobs, callback); + }, - examineOldContainer: (container, callback = (error, name, id, ttl)->) -> - name = container.Name or container.Names?[0] - created = container.Created * 1000 # creation time is returned in seconds - now = Date.now() - age = now - created - maxAge = DockerRunner.MAX_CONTAINER_AGE - ttl = maxAge - age - logger.log {containerName: name, created: created, now: now, age: age, maxAge: maxAge, ttl: ttl}, "checking whether to destroy container" - callback(null, name, container.Id, ttl) + _startContainer(options, volumes, attachStreamHandler, callback) { + if (callback == null) { callback = function(error, output) {}; } + callback = _.once(callback); + const { name } = options; - destroyOldContainers: (callback = (error) ->) -> - dockerode.listContainers all: true, (error, containers) -> - return callback(error) if error? - jobs = [] - for container in containers or [] - do (container) -> - DockerRunner.examineOldContainer container, (err, name, id, ttl) -> - if name.slice(0, 9) == '/project-' && ttl <= 0 - jobs.push (cb) -> - DockerRunner.destroyContainer name, id, false, () -> cb() - # Ignore errors because some containers get stuck but - # will be destroyed next time - async.series jobs, callback + logger.log({container_name: name}, "starting container"); + const container = dockerode.getContainer(name); - startContainerMonitor: () -> - logger.log {maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry" - # randomise the start time - randomDelay = Math.floor(Math.random() * 5 * 60 * 1000) - setTimeout () -> - setInterval () -> - DockerRunner.destroyOldContainers() - , oneHour = 60 * 60 * 1000 - , randomDelay + const createAndStartContainer = () => + dockerode.createContainer(options, function(error, container) { + if (error != null) { return callback(error); } + return startExistingContainer(); + }) + ; -DockerRunner.startContainerMonitor() + var startExistingContainer = () => + DockerRunner.attachToContainer(options.name, attachStreamHandler, function(error){ + if (error != null) { return callback(error); } + return container.start(function(error) { + if ((error != null) && ((error != null ? error.statusCode : undefined) !== 304)) { //already running + return callback(error); + } else { + return callback(); + } + }); + }) + ; + + return container.inspect(function(error, stats){ + if ((error != null ? error.statusCode : undefined) === 404) { + return createAndStartContainer(); + } else if (error != null) { + logger.err({container_name: name, error}, "unable to inspect container to start"); + return callback(error); + } else { + return startExistingContainer(); + } + }); + }, + + + attachToContainer(containerId, attachStreamHandler, attachStartCallback) { + const container = dockerode.getContainer(containerId); + return container.attach({stdout: 1, stderr: 1, stream: 1}, function(error, stream) { + if (error != null) { + logger.error({err: error, container_id: containerId}, "error attaching to container"); + return attachStartCallback(error); + } else { + attachStartCallback(); + } + + + logger.log({container_id: containerId}, "attached to container"); + + const MAX_OUTPUT = 1024 * 1024; // limit output to 1MB + const createStringOutputStream = function(name) { + return { + data: "", + overflowed: false, + write(data) { + if (this.overflowed) { return; } + if (this.data.length < MAX_OUTPUT) { + return this.data += data; + } else { + logger.error({container_id: containerId, length: this.data.length, maxLen: MAX_OUTPUT}, `${name} exceeds max size`); + this.data += `(...truncated at ${MAX_OUTPUT} chars...)`; + return this.overflowed = true; + } + } + // kill container if too much output + // docker.containers.kill(containerId, () ->) + }; + }; + + const stdout = createStringOutputStream("stdout"); + const stderr = createStringOutputStream("stderr"); + + container.modem.demuxStream(stream, stdout, stderr); + + stream.on("error", err => logger.error({err, container_id: containerId}, "error reading from container stream")); + + return stream.on("end", () => attachStreamHandler(null, {stdout: stdout.data, stderr: stderr.data})); + }); + }, + + waitForContainer(containerId, timeout, _callback) { + if (_callback == null) { _callback = function(error, exitCode) {}; } + const callback = function(...args) { + _callback(...Array.from(args || [])); + // Only call the callback once + return _callback = function() {}; + }; + + const container = dockerode.getContainer(containerId); + + let timedOut = false; + const timeoutId = setTimeout(function() { + timedOut = true; + logger.log({container_id: containerId}, "timeout reached, killing container"); + return container.kill(function() {}); + } + , timeout); + + logger.log({container_id: containerId}, "waiting for docker container"); + return container.wait(function(error, res) { + if (error != null) { + clearTimeout(timeoutId); + logger.error({err: error, container_id: containerId}, "error waiting for container"); + return callback(error); + } + if (timedOut) { + logger.log({containerId}, "docker container timed out"); + error = DockerRunner.ERR_TIMED_OUT; + error.timedout = true; + return callback(error); + } else { + clearTimeout(timeoutId); + logger.log({container_id: containerId, exitCode: res.StatusCode}, "docker container returned"); + return callback(null, res.StatusCode); + } + }); + }, + + destroyContainer(containerName, containerId, shouldForce, callback) { + // We want the containerName for the lock and, ideally, the + // containerId to delete. There is a bug in the docker.io module + // where if you delete by name and there is an error, it throws an + // async exception, but if you delete by id it just does a normal + // error callback. We fall back to deleting by name if no id is + // supplied. + if (callback == null) { callback = function(error) {}; } + return LockManager.runWithLock(containerName, releaseLock => DockerRunner._destroyContainer(containerId || containerName, shouldForce, releaseLock) + , callback); + }, + + _destroyContainer(containerId, shouldForce, callback) { + if (callback == null) { callback = function(error) {}; } + logger.log({container_id: containerId}, "destroying docker container"); + const container = dockerode.getContainer(containerId); + return container.remove({force: shouldForce === true}, function(error) { + if ((error != null) && ((error != null ? error.statusCode : undefined) === 404)) { + logger.warn({err: error, container_id: containerId}, "container not found, continuing"); + error = null; + } + if (error != null) { + logger.error({err: error, container_id: containerId}, "error destroying container"); + } else { + logger.log({container_id: containerId}, "destroyed container"); + } + return callback(error); + }); + }, + + // handle expiry of docker containers + + MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000), + + examineOldContainer(container, callback) { + if (callback == null) { callback = function(error, name, id, ttl){}; } + const name = container.Name || (container.Names != null ? container.Names[0] : undefined); + const created = container.Created * 1000; // creation time is returned in seconds + const now = Date.now(); + const age = now - created; + const maxAge = DockerRunner.MAX_CONTAINER_AGE; + const ttl = maxAge - age; + logger.log({containerName: name, created, now, age, maxAge, ttl}, "checking whether to destroy container"); + return callback(null, name, container.Id, ttl); + }, + + destroyOldContainers(callback) { + if (callback == null) { callback = function(error) {}; } + return dockerode.listContainers({all: true}, function(error, containers) { + if (error != null) { return callback(error); } + const jobs = []; + for (let container of Array.from(containers || [])) { + (container => + DockerRunner.examineOldContainer(container, function(err, name, id, ttl) { + if ((name.slice(0, 9) === '/project-') && (ttl <= 0)) { + return jobs.push(cb => DockerRunner.destroyContainer(name, id, false, () => cb())); + } + }) + )(container); + } + // Ignore errors because some containers get stuck but + // will be destroyed next time + return async.series(jobs, callback); + }); + }, + + startContainerMonitor() { + logger.log({maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry"); + // randomise the start time + const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000); + return setTimeout(() => + setInterval(() => DockerRunner.destroyOldContainers() + , (oneHour = 60 * 60 * 1000)) + + , randomDelay); + } +}); + +DockerRunner.startContainerMonitor(); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} +function __guardMethod__(obj, methodName, transform) { + if (typeof obj !== 'undefined' && obj !== null && typeof obj[methodName] === 'function') { + return transform(obj, methodName); + } else { + return undefined; + } +} \ No newline at end of file diff --git a/app/coffee/DraftModeManager.js b/app/coffee/DraftModeManager.js index 2f9e931..8ddbbd0 100644 --- a/app/coffee/DraftModeManager.js +++ b/app/coffee/DraftModeManager.js @@ -1,24 +1,37 @@ -fs = require "fs" -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let DraftModeManager; +const fs = require("fs"); +const logger = require("logger-sharelatex"); -module.exports = DraftModeManager = - injectDraftMode: (filename, callback = (error) ->) -> - fs.readFile filename, "utf8", (error, content) -> - return callback(error) if error? - # avoid adding draft mode more than once - if content?.indexOf("\\documentclass\[draft") >= 0 - return callback() - modified_content = DraftModeManager._injectDraftOption content - logger.log { - content: content.slice(0,1024), # \documentclass is normally v near the top +module.exports = (DraftModeManager = { + injectDraftMode(filename, callback) { + if (callback == null) { callback = function(error) {}; } + return fs.readFile(filename, "utf8", function(error, content) { + if (error != null) { return callback(error); } + // avoid adding draft mode more than once + if ((content != null ? content.indexOf("\\documentclass\[draft") : undefined) >= 0) { + return callback(); + } + const modified_content = DraftModeManager._injectDraftOption(content); + logger.log({ + content: content.slice(0,1024), // \documentclass is normally v near the top modified_content: modified_content.slice(0,1024), filename - }, "injected draft class" - fs.writeFile filename, modified_content, callback + }, "injected draft class"); + return fs.writeFile(filename, modified_content, callback); + }); + }, - _injectDraftOption: (content) -> - content - # With existing options (must be first, otherwise both are applied) + _injectDraftOption(content) { + return content + // With existing options (must be first, otherwise both are applied) .replace(/\\documentclass\[/g, "\\documentclass[draft,") - # Without existing options - .replace(/\\documentclass\{/g, "\\documentclass[draft]{") + // Without existing options + .replace(/\\documentclass\{/g, "\\documentclass[draft]{"); + } +}); diff --git a/app/coffee/Errors.js b/app/coffee/Errors.js index b375513..3a9ef22 100644 --- a/app/coffee/Errors.js +++ b/app/coffee/Errors.js @@ -1,25 +1,30 @@ -NotFoundError = (message) -> - error = new Error(message) - error.name = "NotFoundError" - error.__proto__ = NotFoundError.prototype - return error -NotFoundError.prototype.__proto__ = Error.prototype +let Errors; +var NotFoundError = function(message) { + const error = new Error(message); + error.name = "NotFoundError"; + error.__proto__ = NotFoundError.prototype; + return error; +}; +NotFoundError.prototype.__proto__ = Error.prototype; -FilesOutOfSyncError = (message) -> - error = new Error(message) - error.name = "FilesOutOfSyncError" - error.__proto__ = FilesOutOfSyncError.prototype - return error -FilesOutOfSyncError.prototype.__proto__ = Error.prototype +var FilesOutOfSyncError = function(message) { + const error = new Error(message); + error.name = "FilesOutOfSyncError"; + error.__proto__ = FilesOutOfSyncError.prototype; + return error; +}; +FilesOutOfSyncError.prototype.__proto__ = Error.prototype; -AlreadyCompilingError = (message) -> - error = new Error(message) - error.name = "AlreadyCompilingError" - error.__proto__ = AlreadyCompilingError.prototype - return error -AlreadyCompilingError.prototype.__proto__ = Error.prototype +var AlreadyCompilingError = function(message) { + const error = new Error(message); + error.name = "AlreadyCompilingError"; + error.__proto__ = AlreadyCompilingError.prototype; + return error; +}; +AlreadyCompilingError.prototype.__proto__ = Error.prototype; -module.exports = Errors = - NotFoundError: NotFoundError - FilesOutOfSyncError: FilesOutOfSyncError - AlreadyCompilingError: AlreadyCompilingError +module.exports = (Errors = { + NotFoundError, + FilesOutOfSyncError, + AlreadyCompilingError +}); diff --git a/app/coffee/LatexRunner.js b/app/coffee/LatexRunner.js index 29433f8..4c83e08 100644 --- a/app/coffee/LatexRunner.js +++ b/app/coffee/LatexRunner.js @@ -1,95 +1,123 @@ -Path = require "path" -Settings = require "settings-sharelatex" -logger = require "logger-sharelatex" -Metrics = require "./Metrics" -CommandRunner = require "./CommandRunner" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let LatexRunner; +const Path = require("path"); +const Settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const Metrics = require("./Metrics"); +const CommandRunner = require("./CommandRunner"); -ProcessTable = {} # table of currently running jobs (pids or docker container names) +const ProcessTable = {}; // table of currently running jobs (pids or docker container names) -module.exports = LatexRunner = - runLatex: (project_id, options, callback = (error) ->) -> - {directory, mainFile, compiler, timeout, image, environment, flags} = options - compiler ||= "pdflatex" - timeout ||= 60000 # milliseconds +module.exports = (LatexRunner = { + runLatex(project_id, options, callback) { + let command; + if (callback == null) { callback = function(error) {}; } + let {directory, mainFile, compiler, timeout, image, environment, flags} = options; + if (!compiler) { compiler = "pdflatex"; } + if (!timeout) { timeout = 60000; } // milliseconds - logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, environment: environment, flags:flags, "starting compile" + logger.log({directory, compiler, timeout, mainFile, environment, flags}, "starting compile"); - # We want to run latexmk on the tex file which we will automatically - # generate from the Rtex/Rmd/md file. - mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex") + // We want to run latexmk on the tex file which we will automatically + // generate from the Rtex/Rmd/md file. + mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex"); - if compiler == "pdflatex" - command = LatexRunner._pdflatexCommand mainFile, flags - else if compiler == "latex" - command = LatexRunner._latexCommand mainFile, flags - else if compiler == "xelatex" - command = LatexRunner._xelatexCommand mainFile, flags - else if compiler == "lualatex" - command = LatexRunner._lualatexCommand mainFile, flags - else - return callback new Error("unknown compiler: #{compiler}") + if (compiler === "pdflatex") { + command = LatexRunner._pdflatexCommand(mainFile, flags); + } else if (compiler === "latex") { + command = LatexRunner._latexCommand(mainFile, flags); + } else if (compiler === "xelatex") { + command = LatexRunner._xelatexCommand(mainFile, flags); + } else if (compiler === "lualatex") { + command = LatexRunner._lualatexCommand(mainFile, flags); + } else { + return callback(new Error(`unknown compiler: ${compiler}`)); + } - if Settings.clsi?.strace - command = ["strace", "-o", "strace", "-ff"].concat(command) + if (Settings.clsi != null ? Settings.clsi.strace : undefined) { + command = ["strace", "-o", "strace", "-ff"].concat(command); + } - id = "#{project_id}" # record running project under this id + const id = `${project_id}`; // record running project under this id - ProcessTable[id] = CommandRunner.run project_id, command, directory, image, timeout, environment, (error, output) -> - delete ProcessTable[id] - return callback(error) if error? - runs = output?.stderr?.match(/^Run number \d+ of .*latex/mg)?.length or 0 - failed = if output?.stdout?.match(/^Latexmk: Errors/m)? then 1 else 0 - # counters from latexmk output - stats = {} - stats["latexmk-errors"] = failed - stats["latex-runs"] = runs - stats["latex-runs-with-errors"] = if failed then runs else 0 - stats["latex-runs-#{runs}"] = 1 - stats["latex-runs-with-errors-#{runs}"] = if failed then 1 else 0 - # timing information from /usr/bin/time - timings = {} - stderr = output?.stderr - timings["cpu-percent"] = stderr?.match(/Percent of CPU this job got: (\d+)/m)?[1] or 0 - timings["cpu-time"] = stderr?.match(/User time.*: (\d+.\d+)/m)?[1] or 0 - timings["sys-time"] = stderr?.match(/System time.*: (\d+.\d+)/m)?[1] or 0 - callback error, output, stats, timings + return ProcessTable[id] = CommandRunner.run(project_id, command, directory, image, timeout, environment, function(error, output) { + delete ProcessTable[id]; + if (error != null) { return callback(error); } + const runs = __guard__(__guard__(output != null ? output.stderr : undefined, x1 => x1.match(/^Run number \d+ of .*latex/mg)), x => x.length) || 0; + const failed = (__guard__(output != null ? output.stdout : undefined, x2 => x2.match(/^Latexmk: Errors/m)) != null) ? 1 : 0; + // counters from latexmk output + const stats = {}; + stats["latexmk-errors"] = failed; + stats["latex-runs"] = runs; + stats["latex-runs-with-errors"] = failed ? runs : 0; + stats[`latex-runs-${runs}`] = 1; + stats[`latex-runs-with-errors-${runs}`] = failed ? 1 : 0; + // timing information from /usr/bin/time + const timings = {}; + const stderr = output != null ? output.stderr : undefined; + timings["cpu-percent"] = __guard__(stderr != null ? stderr.match(/Percent of CPU this job got: (\d+)/m) : undefined, x3 => x3[1]) || 0; + timings["cpu-time"] = __guard__(stderr != null ? stderr.match(/User time.*: (\d+.\d+)/m) : undefined, x4 => x4[1]) || 0; + timings["sys-time"] = __guard__(stderr != null ? stderr.match(/System time.*: (\d+.\d+)/m) : undefined, x5 => x5[1]) || 0; + return callback(error, output, stats, timings); + }); + }, - killLatex: (project_id, callback = (error) ->) -> - id = "#{project_id}" - logger.log {id:id}, "killing running compile" - if not ProcessTable[id]? - logger.warn {id}, "no such project to kill" - return callback(null) - else - CommandRunner.kill ProcessTable[id], callback + killLatex(project_id, callback) { + if (callback == null) { callback = function(error) {}; } + const id = `${project_id}`; + logger.log({id}, "killing running compile"); + if ((ProcessTable[id] == null)) { + logger.warn({id}, "no such project to kill"); + return callback(null); + } else { + return CommandRunner.kill(ProcessTable[id], callback); + } + }, - _latexmkBaseCommand: (flags) -> - args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"] - if flags - args = args.concat(flags) - (Settings?.clsi?.latexmkCommandPrefix || []).concat(args) + _latexmkBaseCommand(flags) { + let args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"]; + if (flags) { + args = args.concat(flags); + } + return (__guard__(Settings != null ? Settings.clsi : undefined, x => x.latexmkCommandPrefix) || []).concat(args); + }, - _pdflatexCommand: (mainFile, flags) -> - LatexRunner._latexmkBaseCommand(flags).concat [ + _pdflatexCommand(mainFile, flags) { + return LatexRunner._latexmkBaseCommand(flags).concat([ "-pdf", Path.join("$COMPILE_DIR", mainFile) - ] + ]); + }, - _latexCommand: (mainFile, flags) -> - LatexRunner._latexmkBaseCommand(flags).concat [ + _latexCommand(mainFile, flags) { + return LatexRunner._latexmkBaseCommand(flags).concat([ "-pdfdvi", Path.join("$COMPILE_DIR", mainFile) - ] + ]); + }, - _xelatexCommand: (mainFile, flags) -> - LatexRunner._latexmkBaseCommand(flags).concat [ + _xelatexCommand(mainFile, flags) { + return LatexRunner._latexmkBaseCommand(flags).concat([ "-xelatex", Path.join("$COMPILE_DIR", mainFile) - ] + ]); + }, - _lualatexCommand: (mainFile, flags) -> - LatexRunner._latexmkBaseCommand(flags).concat [ + _lualatexCommand(mainFile, flags) { + return LatexRunner._latexmkBaseCommand(flags).concat([ "-lualatex", Path.join("$COMPILE_DIR", mainFile) - ] + ]); + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/app/coffee/LocalCommandRunner.js b/app/coffee/LocalCommandRunner.js index c5ef3c6..405c51b 100644 --- a/app/coffee/LocalCommandRunner.js +++ b/app/coffee/LocalCommandRunner.js @@ -1,48 +1,66 @@ -spawn = require("child_process").spawn -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let CommandRunner; +const { spawn } = require("child_process"); +const logger = require("logger-sharelatex"); -logger.info "using standard command runner" +logger.info("using standard command runner"); -module.exports = CommandRunner = - run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) -> - command = (arg.toString().replace('$COMPILE_DIR', directory) for arg in command) - logger.log project_id: project_id, command: command, directory: directory, "running command" - logger.warn "timeouts and sandboxing are not enabled with CommandRunner" +module.exports = (CommandRunner = { + run(project_id, command, directory, image, timeout, environment, callback) { + let key, value; + if (callback == null) { callback = function(error) {}; } + command = (Array.from(command).map((arg) => arg.toString().replace('$COMPILE_DIR', directory))); + logger.log({project_id, command, directory}, "running command"); + logger.warn("timeouts and sandboxing are not enabled with CommandRunner"); - # merge environment settings - env = {} - env[key] = value for key, value of process.env - env[key] = value for key, value of environment + // merge environment settings + const env = {}; + for (key in process.env) { value = process.env[key]; env[key] = value; } + for (key in environment) { value = environment[key]; env[key] = value; } - # run command as detached process so it has its own process group (which can be killed if needed) - proc = spawn command[0], command.slice(1), cwd: directory, env: env + // run command as detached process so it has its own process group (which can be killed if needed) + const proc = spawn(command[0], command.slice(1), {cwd: directory, env}); - stdout = "" - proc.stdout.on "data", (data)-> - stdout += data + let stdout = ""; + proc.stdout.on("data", data=> stdout += data); - proc.on "error", (err)-> - logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command" - callback(err) + proc.on("error", function(err){ + logger.err({err, project_id, command, directory}, "error running command"); + return callback(err); + }); - proc.on "close", (code, signal) -> - logger.info code:code, signal:signal, project_id:project_id, "command exited" - if signal is 'SIGTERM' # signal from kill method below - err = new Error("terminated") - err.terminated = true - return callback(err) - else if code is 1 # exit status from chktex - err = new Error("exited") - err.code = code - return callback(err) - else - callback(null, {"stdout": stdout}) + proc.on("close", function(code, signal) { + let err; + logger.info({code, signal, project_id}, "command exited"); + if (signal === 'SIGTERM') { // signal from kill method below + err = new Error("terminated"); + err.terminated = true; + return callback(err); + } else if (code === 1) { // exit status from chktex + err = new Error("exited"); + err.code = code; + return callback(err); + } else { + return callback(null, {"stdout": stdout}); + } + }); - return proc.pid # return process id to allow job to be killed if necessary + return proc.pid; + }, // return process id to allow job to be killed if necessary - kill: (pid, callback = (error) ->) -> - try - process.kill -pid # kill all processes in group - catch err - return callback(err) - callback() + kill(pid, callback) { + if (callback == null) { callback = function(error) {}; } + try { + process.kill(-pid); // kill all processes in group + } catch (err) { + return callback(err); + } + return callback(); + } +}); diff --git a/app/coffee/LockManager.js b/app/coffee/LockManager.js index 5d9fe26..2405e8a 100644 --- a/app/coffee/LockManager.js +++ b/app/coffee/LockManager.js @@ -1,31 +1,50 @@ -Settings = require('settings-sharelatex') -logger = require "logger-sharelatex" -Lockfile = require('lockfile') # from https://github.com/npm/lockfile -Errors = require "./Errors" -fs = require("fs") -Path = require("path") -module.exports = LockManager = - LOCK_TEST_INTERVAL: 1000 # 50ms between each test of the lock - MAX_LOCK_WAIT_TIME: 15000 # 10s maximum time to spend trying to get the lock - LOCK_STALE: 5*60*1000 # 5 mins time until lock auto expires +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let LockManager; +const Settings = require('settings-sharelatex'); +const logger = require("logger-sharelatex"); +const Lockfile = require('lockfile'); // from https://github.com/npm/lockfile +const Errors = require("./Errors"); +const fs = require("fs"); +const Path = require("path"); +module.exports = (LockManager = { + LOCK_TEST_INTERVAL: 1000, // 50ms between each test of the lock + MAX_LOCK_WAIT_TIME: 15000, // 10s maximum time to spend trying to get the lock + LOCK_STALE: 5*60*1000, // 5 mins time until lock auto expires - runWithLock: (path, runner, callback = ((error) ->)) -> - lockOpts = - wait: @MAX_LOCK_WAIT_TIME - pollPeriod: @LOCK_TEST_INTERVAL - stale: @LOCK_STALE - Lockfile.lock path, lockOpts, (error) -> - if error?.code is 'EEXIST' - return callback new Errors.AlreadyCompilingError("compile in progress") - else if error? - fs.lstat path, (statLockErr, statLock)-> - fs.lstat Path.dirname(path), (statDirErr, statDir)-> - fs.readdir Path.dirname(path), (readdirErr, readdirDir)-> - logger.err error:error, path:path, statLock:statLock, statLockErr:statLockErr, statDir:statDir, statDirErr: statDirErr, readdirErr:readdirErr, readdirDir:readdirDir, "unable to get lock" - return callback(error) - else - runner (error1, args...) -> - Lockfile.unlock path, (error2) -> - error = error1 or error2 - return callback(error) if error? - callback(null, args...) + runWithLock(path, runner, callback) { + if (callback == null) { callback = function(error) {}; } + const lockOpts = { + wait: this.MAX_LOCK_WAIT_TIME, + pollPeriod: this.LOCK_TEST_INTERVAL, + stale: this.LOCK_STALE + }; + return Lockfile.lock(path, lockOpts, function(error) { + if ((error != null ? error.code : undefined) === 'EEXIST') { + return callback(new Errors.AlreadyCompilingError("compile in progress")); + } else if (error != null) { + return fs.lstat(path, (statLockErr, statLock)=> + fs.lstat(Path.dirname(path), (statDirErr, statDir)=> + fs.readdir(Path.dirname(path), function(readdirErr, readdirDir){ + logger.err({error, path, statLock, statLockErr, statDir, statDirErr, readdirErr, readdirDir}, "unable to get lock"); + return callback(error); + }) + ) + ); + } else { + return runner((error1, ...args) => + Lockfile.unlock(path, function(error2) { + error = error1 || error2; + if (error != null) { return callback(error); } + return callback(null, ...Array.from(args)); + }) + ); + } + }); + } +}); diff --git a/app/coffee/Metrics.js b/app/coffee/Metrics.js index 9965b25..8148d66 100644 --- a/app/coffee/Metrics.js +++ b/app/coffee/Metrics.js @@ -1,2 +1,2 @@ -module.exports = require "metrics-sharelatex" +module.exports = require("metrics-sharelatex"); diff --git a/app/coffee/OutputCacheManager.js b/app/coffee/OutputCacheManager.js index 5ef92ec..6d03a10 100644 --- a/app/coffee/OutputCacheManager.js +++ b/app/coffee/OutputCacheManager.js @@ -1,199 +1,270 @@ -async = require "async" -fs = require "fs" -fse = require "fs-extra" -Path = require "path" -logger = require "logger-sharelatex" -_ = require "underscore" -Settings = require "settings-sharelatex" -crypto = require "crypto" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS104: Avoid inline assignments + * DS204: Change includes calls to have a more natural evaluation order + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let OutputCacheManager; +const async = require("async"); +const fs = require("fs"); +const fse = require("fs-extra"); +const Path = require("path"); +const logger = require("logger-sharelatex"); +const _ = require("underscore"); +const Settings = require("settings-sharelatex"); +const crypto = require("crypto"); -OutputFileOptimiser = require "./OutputFileOptimiser" +const OutputFileOptimiser = require("./OutputFileOptimiser"); -module.exports = OutputCacheManager = - CACHE_SUBDIR: '.cache/clsi' - ARCHIVE_SUBDIR: '.archive/clsi' - # build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes - # for backwards compatibility, make the randombytes part optional - BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/ - CACHE_LIMIT: 2 # maximum number of cache directories - CACHE_AGE: 60*60*1000 # up to one hour old +module.exports = (OutputCacheManager = { + CACHE_SUBDIR: '.cache/clsi', + ARCHIVE_SUBDIR: '.archive/clsi', + // build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes + // for backwards compatibility, make the randombytes part optional + BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/, + CACHE_LIMIT: 2, // maximum number of cache directories + CACHE_AGE: 60*60*1000, // up to one hour old - path: (buildId, file) -> - # used by static server, given build id return '.cache/clsi/buildId' - if buildId.match OutputCacheManager.BUILD_REGEX - return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file) - else - # for invalid build id, return top level - return file + path(buildId, file) { + // used by static server, given build id return '.cache/clsi/buildId' + if (buildId.match(OutputCacheManager.BUILD_REGEX)) { + return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file); + } else { + // for invalid build id, return top level + return file; + } + }, - generateBuildId: (callback = (error, buildId) ->) -> - # generate a secure build id from Date.now() and 8 random bytes in hex - crypto.randomBytes 8, (err, buf) -> - return callback(err) if err? - random = buf.toString('hex') - date = Date.now().toString(16) - callback err, "#{date}-#{random}" + generateBuildId(callback) { + // generate a secure build id from Date.now() and 8 random bytes in hex + if (callback == null) { callback = function(error, buildId) {}; } + return crypto.randomBytes(8, function(err, buf) { + if (err != null) { return callback(err); } + const random = buf.toString('hex'); + const date = Date.now().toString(16); + return callback(err, `${date}-${random}`); + }); + }, - saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) -> - OutputCacheManager.generateBuildId (err, buildId) -> - return callback(err) if err? - OutputCacheManager.saveOutputFilesInBuildDir outputFiles, compileDir, buildId, callback + saveOutputFiles(outputFiles, compileDir, callback) { + if (callback == null) { callback = function(error) {}; } + return OutputCacheManager.generateBuildId(function(err, buildId) { + if (err != null) { return callback(err); } + return OutputCacheManager.saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback); + }); + }, - saveOutputFilesInBuildDir: (outputFiles, compileDir, buildId, callback = (error) ->) -> - # make a compileDir/CACHE_SUBDIR/build_id directory and - # copy all the output files into it - cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR) - # Put the files into a new cache subdirectory - cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId) - # Is it a per-user compile? check if compile directory is PROJECTID-USERID - perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/) + saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback) { + // make a compileDir/CACHE_SUBDIR/build_id directory and + // copy all the output files into it + if (callback == null) { callback = function(error) {}; } + const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR); + // Put the files into a new cache subdirectory + const cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId); + // Is it a per-user compile? check if compile directory is PROJECTID-USERID + const perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/); - # Archive logs in background - if Settings.clsi?.archive_logs or Settings.clsi?.strace - OutputCacheManager.archiveLogs outputFiles, compileDir, buildId, (err) -> - if err? - logger.warn err:err, "erroring archiving log files" + // Archive logs in background + if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) || (Settings.clsi != null ? Settings.clsi.strace : undefined)) { + OutputCacheManager.archiveLogs(outputFiles, compileDir, buildId, function(err) { + if (err != null) { + return logger.warn({err}, "erroring archiving log files"); + } + }); + } - # make the new cache directory - fse.ensureDir cacheDir, (err) -> - if err? - logger.error err: err, directory: cacheDir, "error creating cache directory" - callback(err, outputFiles) - else - # copy all the output files into the new cache directory - results = [] - async.mapSeries outputFiles, (file, cb) -> - # don't send dot files as output, express doesn't serve them - if OutputCacheManager._fileIsHidden(file.path) - logger.debug compileDir: compileDir, path: file.path, "ignoring dotfile in output" - return cb() - # copy other files into cache directory if valid - newFile = _.clone(file) - [src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)] - OutputCacheManager._checkFileIsSafe src, (err, isSafe) -> - return cb(err) if err? - if !isSafe - return cb() - OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) -> - return cb(err) if err? - if !shouldCopy - return cb() - OutputCacheManager._copyFile src, dst, (err) -> - return cb(err) if err? - newFile.build = buildId # attach a build id if we cached the file - results.push newFile - cb() - , (err) -> - if err? - # pass back the original files if we encountered *any* error - callback(err, outputFiles) - # clean up the directory we just created - fse.remove cacheDir, (err) -> - if err? - logger.error err: err, dir: cacheDir, "error removing cache dir after failure" - else - # pass back the list of new files in the cache - callback(err, results) - # let file expiry run in the background, expire all previous files if per-user - OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 1 else null} + // make the new cache directory + return fse.ensureDir(cacheDir, function(err) { + if (err != null) { + logger.error({err, directory: cacheDir}, "error creating cache directory"); + return callback(err, outputFiles); + } else { + // copy all the output files into the new cache directory + const results = []; + return async.mapSeries(outputFiles, function(file, cb) { + // don't send dot files as output, express doesn't serve them + if (OutputCacheManager._fileIsHidden(file.path)) { + logger.debug({compileDir, path: file.path}, "ignoring dotfile in output"); + return cb(); + } + // copy other files into cache directory if valid + const newFile = _.clone(file); + const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]); + return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) { + if (err != null) { return cb(err); } + if (!isSafe) { + return cb(); + } + return OutputCacheManager._checkIfShouldCopy(src, function(err, shouldCopy) { + if (err != null) { return cb(err); } + if (!shouldCopy) { + return cb(); + } + return OutputCacheManager._copyFile(src, dst, function(err) { + if (err != null) { return cb(err); } + newFile.build = buildId; // attach a build id if we cached the file + results.push(newFile); + return cb(); + }); + }); + }); + } + , function(err) { + if (err != null) { + // pass back the original files if we encountered *any* error + callback(err, outputFiles); + // clean up the directory we just created + return fse.remove(cacheDir, function(err) { + if (err != null) { + return logger.error({err, dir: cacheDir}, "error removing cache dir after failure"); + } + }); + } else { + // pass back the list of new files in the cache + callback(err, results); + // let file expiry run in the background, expire all previous files if per-user + return OutputCacheManager.expireOutputFiles(cacheRoot, {keep: buildId, limit: perUser ? 1 : null}); + } + }); + } + }); + }, - archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) -> - archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId) - logger.log {dir: archiveDir}, "archiving log files for project" - fse.ensureDir archiveDir, (err) -> - return callback(err) if err? - async.mapSeries outputFiles, (file, cb) -> - [src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)] - OutputCacheManager._checkFileIsSafe src, (err, isSafe) -> - return cb(err) if err? - return cb() if !isSafe - OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) -> - return cb(err) if err? - return cb() if !shouldArchive - OutputCacheManager._copyFile src, dst, cb - , callback + archiveLogs(outputFiles, compileDir, buildId, callback) { + if (callback == null) { callback = function(error) {}; } + const archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId); + logger.log({dir: archiveDir}, "archiving log files for project"); + return fse.ensureDir(archiveDir, function(err) { + if (err != null) { return callback(err); } + return async.mapSeries(outputFiles, function(file, cb) { + const [src, dst] = Array.from([Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]); + return OutputCacheManager._checkFileIsSafe(src, function(err, isSafe) { + if (err != null) { return cb(err); } + if (!isSafe) { return cb(); } + return OutputCacheManager._checkIfShouldArchive(src, function(err, shouldArchive) { + if (err != null) { return cb(err); } + if (!shouldArchive) { return cb(); } + return OutputCacheManager._copyFile(src, dst, cb); + }); + }); + } + , callback); + }); + }, - expireOutputFiles: (cacheRoot, options, callback = (error) ->) -> - # look in compileDir for build dirs and delete if > N or age of mod time > T - fs.readdir cacheRoot, (err, results) -> - if err? - return callback(null) if err.code == 'ENOENT' # cache directory is empty - logger.error err: err, project_id: cacheRoot, "error clearing cache" - return callback(err) + expireOutputFiles(cacheRoot, options, callback) { + // look in compileDir for build dirs and delete if > N or age of mod time > T + if (callback == null) { callback = function(error) {}; } + return fs.readdir(cacheRoot, function(err, results) { + if (err != null) { + if (err.code === 'ENOENT') { return callback(null); } // cache directory is empty + logger.error({err, project_id: cacheRoot}, "error clearing cache"); + return callback(err); + } - dirs = results.sort().reverse() - currentTime = Date.now() + const dirs = results.sort().reverse(); + const currentTime = Date.now(); - isExpired = (dir, index) -> - return false if options?.keep == dir - # remove any directories over the requested (non-null) limit - return true if options?.limit? and index > options.limit - # remove any directories over the hard limit - return true if index > OutputCacheManager.CACHE_LIMIT - # we can get the build time from the first part of the directory name DDDD-RRRR - # DDDD is date and RRRR is random bytes - dirTime = parseInt(dir.split('-')?[0], 16) - age = currentTime - dirTime - return age > OutputCacheManager.CACHE_AGE + const isExpired = function(dir, index) { + if ((options != null ? options.keep : undefined) === dir) { return false; } + // remove any directories over the requested (non-null) limit + if (((options != null ? options.limit : undefined) != null) && (index > options.limit)) { return true; } + // remove any directories over the hard limit + if (index > OutputCacheManager.CACHE_LIMIT) { return true; } + // we can get the build time from the first part of the directory name DDDD-RRRR + // DDDD is date and RRRR is random bytes + const dirTime = parseInt(__guard__(dir.split('-'), x => x[0]), 16); + const age = currentTime - dirTime; + return age > OutputCacheManager.CACHE_AGE; + }; - toRemove = _.filter(dirs, isExpired) + const toRemove = _.filter(dirs, isExpired); - removeDir = (dir, cb) -> - fse.remove Path.join(cacheRoot, dir), (err, result) -> - logger.log cache: cacheRoot, dir: dir, "removed expired cache dir" - if err? - logger.error err: err, dir: dir, "cache remove error" - cb(err, result) + const removeDir = (dir, cb) => + fse.remove(Path.join(cacheRoot, dir), function(err, result) { + logger.log({cache: cacheRoot, dir}, "removed expired cache dir"); + if (err != null) { + logger.error({err, dir}, "cache remove error"); + } + return cb(err, result); + }) + ; - async.eachSeries toRemove, (dir, cb) -> - removeDir dir, cb - , callback + return async.eachSeries(toRemove, (dir, cb) => removeDir(dir, cb) + , callback); + }); + }, - _fileIsHidden: (path) -> - return path?.match(/^\.|\/\./)? + _fileIsHidden(path) { + return ((path != null ? path.match(/^\.|\/\./) : undefined) != null); + }, - _checkFileIsSafe: (src, callback = (error, isSafe) ->) -> - # check if we have a valid file to copy into the cache - fs.stat src, (err, stats) -> - if err?.code is 'ENOENT' - logger.warn err: err, file: src, "file has disappeared before copying to build cache" - callback(err, false) - else if err? - # some other problem reading the file - logger.error err: err, file: src, "stat error for file in cache" - callback(err, false) - else if not stats.isFile() - # other filetype - reject it - logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache" - callback(null, false) - else - # it's a plain file, ok to copy - callback(null, true) + _checkFileIsSafe(src, callback) { + // check if we have a valid file to copy into the cache + if (callback == null) { callback = function(error, isSafe) {}; } + return fs.stat(src, function(err, stats) { + if ((err != null ? err.code : undefined) === 'ENOENT') { + logger.warn({err, file: src}, "file has disappeared before copying to build cache"); + return callback(err, false); + } else if (err != null) { + // some other problem reading the file + logger.error({err, file: src}, "stat error for file in cache"); + return callback(err, false); + } else if (!stats.isFile()) { + // other filetype - reject it + logger.warn({src, stat: stats}, "nonfile output - refusing to copy to cache"); + return callback(null, false); + } else { + // it's a plain file, ok to copy + return callback(null, true); + } + }); + }, - _copyFile: (src, dst, callback) -> - # copy output file into the cache - fse.copy src, dst, (err) -> - if err?.code is 'ENOENT' - logger.warn err: err, file: src, "file has disappeared when copying to build cache" - callback(err, false) - else if err? - logger.error err: err, src: src, dst: dst, "copy error for file in cache" - callback(err) - else - if Settings.clsi?.optimiseInDocker - # don't run any optimisations on the pdf when they are done - # in the docker container - callback() - else - # call the optimiser for the file too - OutputFileOptimiser.optimiseFile src, dst, callback + _copyFile(src, dst, callback) { + // copy output file into the cache + return fse.copy(src, dst, function(err) { + if ((err != null ? err.code : undefined) === 'ENOENT') { + logger.warn({err, file: src}, "file has disappeared when copying to build cache"); + return callback(err, false); + } else if (err != null) { + logger.error({err, src, dst}, "copy error for file in cache"); + return callback(err); + } else { + if ((Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined)) { + // don't run any optimisations on the pdf when they are done + // in the docker container + return callback(); + } else { + // call the optimiser for the file too + return OutputFileOptimiser.optimiseFile(src, dst, callback); + } + } + }); + }, - _checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) -> - return callback(null, !Path.basename(src).match(/^strace/)) + _checkIfShouldCopy(src, callback) { + if (callback == null) { callback = function(err, shouldCopy) {}; } + return callback(null, !Path.basename(src).match(/^strace/)); + }, - _checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) -> - if Path.basename(src).match(/^strace/) - return callback(null, true) - if Settings.clsi?.archive_logs and Path.basename(src) in ["output.log", "output.blg"] - return callback(null, true) - return callback(null, false) + _checkIfShouldArchive(src, callback) { + let needle; + if (callback == null) { callback = function(err, shouldCopy) {}; } + if (Path.basename(src).match(/^strace/)) { + return callback(null, true); + } + if ((Settings.clsi != null ? Settings.clsi.archive_logs : undefined) && (needle = Path.basename(src), ["output.log", "output.blg"].includes(needle))) { + return callback(null, true); + } + return callback(null, false); + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/app/coffee/OutputFileFinder.js b/app/coffee/OutputFileFinder.js index 662440b..f0f837c 100644 --- a/app/coffee/OutputFileFinder.js +++ b/app/coffee/OutputFileFinder.js @@ -1,50 +1,78 @@ -async = require "async" -fs = require "fs" -Path = require "path" -spawn = require("child_process").spawn -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let OutputFileFinder; +const async = require("async"); +const fs = require("fs"); +const Path = require("path"); +const { spawn } = require("child_process"); +const logger = require("logger-sharelatex"); -module.exports = OutputFileFinder = - findOutputFiles: (resources, directory, callback = (error, outputFiles, allFiles) ->) -> - incomingResources = {} - for resource in resources - incomingResources[resource.path] = true +module.exports = (OutputFileFinder = { + findOutputFiles(resources, directory, callback) { + if (callback == null) { callback = function(error, outputFiles, allFiles) {}; } + const incomingResources = {}; + for (let resource of Array.from(resources)) { + incomingResources[resource.path] = true; + } - OutputFileFinder._getAllFiles directory, (error, allFiles = []) -> - if error? - logger.err err:error, "error finding all output files" - return callback(error) - outputFiles = [] - for file in allFiles - if !incomingResources[file] - outputFiles.push { - path: file - type: file.match(/\.([^\.]+)$/)?[1] - } - callback null, outputFiles, allFiles + return OutputFileFinder._getAllFiles(directory, function(error, allFiles) { + if (allFiles == null) { allFiles = []; } + if (error != null) { + logger.err({err:error}, "error finding all output files"); + return callback(error); + } + const outputFiles = []; + for (let file of Array.from(allFiles)) { + if (!incomingResources[file]) { + outputFiles.push({ + path: file, + type: __guard__(file.match(/\.([^\.]+)$/), x => x[1]) + }); + } + } + return callback(null, outputFiles, allFiles); + }); + }, - _getAllFiles: (directory, _callback = (error, fileList) ->) -> - callback = (error, fileList) -> - _callback(error, fileList) - _callback = () -> + _getAllFiles(directory, _callback) { + if (_callback == null) { _callback = function(error, fileList) {}; } + const callback = function(error, fileList) { + _callback(error, fileList); + return _callback = function() {}; + }; - # don't include clsi-specific files/directories in the output list - EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"] - args = [directory, "(", EXCLUDE_DIRS..., ")", "-prune", "-o", "-type", "f", "-print"] - logger.log args: args, "running find command" + // don't include clsi-specific files/directories in the output list + const EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"]; + const args = [directory, "(", ...Array.from(EXCLUDE_DIRS), ")", "-prune", "-o", "-type", "f", "-print"]; + logger.log({args}, "running find command"); - proc = spawn("find", args) - stdout = "" - proc.stdout.on "data", (chunk) -> - stdout += chunk.toString() - proc.on "error", callback - proc.on "close", (code) -> - if code != 0 - logger.warn {directory, code}, "find returned error, directory likely doesn't exist" - return callback null, [] - fileList = stdout.trim().split("\n") - fileList = fileList.map (file) -> - # Strip leading directory - path = Path.relative(directory, file) - return callback null, fileList + const proc = spawn("find", args); + let stdout = ""; + proc.stdout.on("data", chunk => stdout += chunk.toString()); + proc.on("error", callback); + return proc.on("close", function(code) { + if (code !== 0) { + logger.warn({directory, code}, "find returned error, directory likely doesn't exist"); + return callback(null, []); + } + let fileList = stdout.trim().split("\n"); + fileList = fileList.map(function(file) { + // Strip leading directory + let path; + return path = Path.relative(directory, file); + }); + return callback(null, fileList); + }); + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/app/coffee/OutputFileOptimiser.js b/app/coffee/OutputFileOptimiser.js index b702f36..f8302aa 100644 --- a/app/coffee/OutputFileOptimiser.js +++ b/app/coffee/OutputFileOptimiser.js @@ -1,55 +1,77 @@ -fs = require "fs" -Path = require "path" -spawn = require("child_process").spawn -logger = require "logger-sharelatex" -Metrics = require "./Metrics" -_ = require "underscore" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let OutputFileOptimiser; +const fs = require("fs"); +const Path = require("path"); +const { spawn } = require("child_process"); +const logger = require("logger-sharelatex"); +const Metrics = require("./Metrics"); +const _ = require("underscore"); -module.exports = OutputFileOptimiser = +module.exports = (OutputFileOptimiser = { - optimiseFile: (src, dst, callback = (error) ->) -> - # check output file (src) and see if we can optimise it, storing - # the result in the build directory (dst) - if src.match(/\/output\.pdf$/) - OutputFileOptimiser.checkIfPDFIsOptimised src, (err, isOptimised) -> - return callback(null) if err? or isOptimised - OutputFileOptimiser.optimisePDF src, dst, callback - else - callback (null) + optimiseFile(src, dst, callback) { + // check output file (src) and see if we can optimise it, storing + // the result in the build directory (dst) + if (callback == null) { callback = function(error) {}; } + if (src.match(/\/output\.pdf$/)) { + return OutputFileOptimiser.checkIfPDFIsOptimised(src, function(err, isOptimised) { + if ((err != null) || isOptimised) { return callback(null); } + return OutputFileOptimiser.optimisePDF(src, dst, callback); + }); + } else { + return callback((null)); + } + }, - checkIfPDFIsOptimised: (file, callback) -> - SIZE = 16*1024 # check the header of the pdf - result = new Buffer(SIZE) - result.fill(0) # prevent leakage of uninitialised buffer - fs.open file, "r", (err, fd) -> - return callback(err) if err? - fs.read fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) -> - fs.close fd, (errClose) -> - return callback(errRead) if errRead? - return callback(errClose) if errReadClose? - isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0 - callback(null, isOptimised) + checkIfPDFIsOptimised(file, callback) { + const SIZE = 16*1024; // check the header of the pdf + const result = new Buffer(SIZE); + result.fill(0); // prevent leakage of uninitialised buffer + return fs.open(file, "r", function(err, fd) { + if (err != null) { return callback(err); } + return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) => + fs.close(fd, function(errClose) { + if (errRead != null) { return callback(errRead); } + if (typeof errReadClose !== 'undefined' && errReadClose !== null) { return callback(errClose); } + const isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0; + return callback(null, isOptimised); + }) + ); + }); + }, - optimisePDF: (src, dst, callback = (error) ->) -> - tmpOutput = dst + '.opt' - args = ["--linearize", src, tmpOutput] - logger.log args: args, "running qpdf command" + optimisePDF(src, dst, callback) { + if (callback == null) { callback = function(error) {}; } + const tmpOutput = dst + '.opt'; + const args = ["--linearize", src, tmpOutput]; + logger.log({args}, "running qpdf command"); - timer = new Metrics.Timer("qpdf") - proc = spawn("qpdf", args) - stdout = "" - proc.stdout.on "data", (chunk) -> - stdout += chunk.toString() - callback = _.once(callback) # avoid double call back for error and close event - proc.on "error", (err) -> - logger.warn {err, args}, "qpdf failed" - callback(null) # ignore the error - proc.on "close", (code) -> - timer.done() - if code != 0 - logger.warn {code, args}, "qpdf returned error" - return callback(null) # ignore the error - fs.rename tmpOutput, dst, (err) -> - if err? - logger.warn {tmpOutput, dst}, "failed to rename output of qpdf command" - callback(null) # ignore the error + const timer = new Metrics.Timer("qpdf"); + const proc = spawn("qpdf", args); + let stdout = ""; + proc.stdout.on("data", chunk => stdout += chunk.toString()); + callback = _.once(callback); // avoid double call back for error and close event + proc.on("error", function(err) { + logger.warn({err, args}, "qpdf failed"); + return callback(null); + }); // ignore the error + return proc.on("close", function(code) { + timer.done(); + if (code !== 0) { + logger.warn({code, args}, "qpdf returned error"); + return callback(null); // ignore the error + } + return fs.rename(tmpOutput, dst, function(err) { + if (err != null) { + logger.warn({tmpOutput, dst}, "failed to rename output of qpdf command"); + } + return callback(null); + }); + }); + } // ignore the error +}); diff --git a/app/coffee/ProjectPersistenceManager.js b/app/coffee/ProjectPersistenceManager.js index 4ea02bf..7b3d5ee 100644 --- a/app/coffee/ProjectPersistenceManager.js +++ b/app/coffee/ProjectPersistenceManager.js @@ -1,84 +1,117 @@ -UrlCache = require "./UrlCache" -CompileManager = require "./CompileManager" -db = require "./db" -dbQueue = require "./DbQueue" -async = require "async" -logger = require "logger-sharelatex" -oneDay = 24 * 60 * 60 * 1000 -Settings = require "settings-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ProjectPersistenceManager; +const UrlCache = require("./UrlCache"); +const CompileManager = require("./CompileManager"); +const db = require("./db"); +const dbQueue = require("./DbQueue"); +const async = require("async"); +const logger = require("logger-sharelatex"); +const oneDay = 24 * 60 * 60 * 1000; +const Settings = require("settings-sharelatex"); -module.exports = ProjectPersistenceManager = +module.exports = (ProjectPersistenceManager = { - EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5 + EXPIRY_TIMEOUT: Settings.project_cache_length_ms || (oneDay * 2.5), - markProjectAsJustAccessed: (project_id, callback = (error) ->) -> - job = (cb)-> - db.Project.findOrCreate(where: {project_id: project_id}) + markProjectAsJustAccessed(project_id, callback) { + if (callback == null) { callback = function(error) {}; } + const job = cb=> + db.Project.findOrCreate({where: {project_id}}) .spread( - (project, created) -> - project.updateAttributes(lastAccessed: new Date()) - .then(() -> cb()) - .error cb + (project, created) => + project.updateAttributes({lastAccessed: new Date()}) + .then(() => cb()) + .error(cb) ) - .error cb - dbQueue.queue.push(job, callback) + .error(cb) + ; + return dbQueue.queue.push(job, callback); + }, - clearExpiredProjects: (callback = (error) ->) -> - ProjectPersistenceManager._findExpiredProjectIds (error, project_ids) -> - return callback(error) if error? - logger.log project_ids: project_ids, "clearing expired projects" - jobs = for project_id in (project_ids or []) - do (project_id) -> - (callback) -> - ProjectPersistenceManager.clearProjectFromCache project_id, (err) -> - if err? - logger.error err: err, project_id: project_id, "error clearing project" - callback() - async.series jobs, (error) -> - return callback(error) if error? - CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) -> - callback() # ignore any errors from deleting directories + clearExpiredProjects(callback) { + if (callback == null) { callback = function(error) {}; } + return ProjectPersistenceManager._findExpiredProjectIds(function(error, project_ids) { + if (error != null) { return callback(error); } + logger.log({project_ids}, "clearing expired projects"); + const jobs = (Array.from(project_ids || [])).map((project_id) => + (project_id => + callback => + ProjectPersistenceManager.clearProjectFromCache(project_id, function(err) { + if (err != null) { + logger.error({err, project_id}, "error clearing project"); + } + return callback(); + }) + + )(project_id)); + return async.series(jobs, function(error) { + if (error != null) { return callback(error); } + return CompileManager.clearExpiredProjects(ProjectPersistenceManager.EXPIRY_TIMEOUT, error => callback()); + }); + }); + }, // ignore any errors from deleting directories - clearProject: (project_id, user_id, callback = (error) ->) -> - logger.log project_id: project_id, user_id:user_id, "clearing project for user" - CompileManager.clearProject project_id, user_id, (error) -> - return callback(error) if error? - ProjectPersistenceManager.clearProjectFromCache project_id, (error) -> - return callback(error) if error? - callback() + clearProject(project_id, user_id, callback) { + if (callback == null) { callback = function(error) {}; } + logger.log({project_id, user_id}, "clearing project for user"); + return CompileManager.clearProject(project_id, user_id, function(error) { + if (error != null) { return callback(error); } + return ProjectPersistenceManager.clearProjectFromCache(project_id, function(error) { + if (error != null) { return callback(error); } + return callback(); + }); + }); + }, - clearProjectFromCache: (project_id, callback = (error) ->) -> - logger.log project_id: project_id, "clearing project from cache" - UrlCache.clearProject project_id, (error) -> - if error? - logger.err error:error, project_id: project_id, "error clearing project from cache" - return callback(error) - ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> - if error? - logger.err error:error, project_id:project_id, "error clearing project from database" - callback(error) + clearProjectFromCache(project_id, callback) { + if (callback == null) { callback = function(error) {}; } + logger.log({project_id}, "clearing project from cache"); + return UrlCache.clearProject(project_id, function(error) { + if (error != null) { + logger.err({error, project_id}, "error clearing project from cache"); + return callback(error); + } + return ProjectPersistenceManager._clearProjectFromDatabase(project_id, function(error) { + if (error != null) { + logger.err({error, project_id}, "error clearing project from database"); + } + return callback(error); + }); + }); + }, - _clearProjectFromDatabase: (project_id, callback = (error) ->) -> - logger.log project_id:project_id, "clearing project from database" - job = (cb)-> - db.Project.destroy(where: {project_id: project_id}) - .then(() -> cb()) - .error cb - dbQueue.queue.push(job, callback) + _clearProjectFromDatabase(project_id, callback) { + if (callback == null) { callback = function(error) {}; } + logger.log({project_id}, "clearing project from database"); + const job = cb=> + db.Project.destroy({where: {project_id}}) + .then(() => cb()) + .error(cb) + ; + return dbQueue.queue.push(job, callback); + }, - _findExpiredProjectIds: (callback = (error, project_ids) ->) -> - job = (cb)-> - keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT) - q = {} - q[db.op.lt] = keepProjectsFrom - db.Project.findAll(where:{lastAccessed:q}) - .then((projects) -> - cb null, projects.map((project) -> project.project_id) - ).error cb + _findExpiredProjectIds(callback) { + if (callback == null) { callback = function(error, project_ids) {}; } + const job = function(cb){ + const keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT); + const q = {}; + q[db.op.lt] = keepProjectsFrom; + return db.Project.findAll({where:{lastAccessed:q}}) + .then(projects => cb(null, projects.map(project => project.project_id))).error(cb); + }; - dbQueue.queue.push(job, callback) + return dbQueue.queue.push(job, callback); + } +}); -logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout" +logger.log({EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout"); diff --git a/app/coffee/RequestParser.js b/app/coffee/RequestParser.js index 9b94712..fdfb8bf 100644 --- a/app/coffee/RequestParser.js +++ b/app/coffee/RequestParser.js @@ -1,128 +1,182 @@ -settings = require("settings-sharelatex") +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let RequestParser; +const settings = require("settings-sharelatex"); -module.exports = RequestParser = - VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"] - MAX_TIMEOUT: 600 +module.exports = (RequestParser = { + VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"], + MAX_TIMEOUT: 600, - parse: (body, callback = (error, data) ->) -> - response = {} + parse(body, callback) { + let resource; + if (callback == null) { callback = function(error, data) {}; } + const response = {}; - if !body.compile? - return callback "top level object should have a compile attribute" - - compile = body.compile - compile.options ||= {} - - try - response.compiler = @_parseAttribute "compiler", - compile.options.compiler, - validValues: @VALID_COMPILERS - default: "pdflatex" - type: "string" - response.timeout = @_parseAttribute "timeout", - compile.options.timeout - default: RequestParser.MAX_TIMEOUT - type: "number" - response.imageName = @_parseAttribute "imageName", - compile.options.imageName, - type: "string" - response.draft = @_parseAttribute "draft", - compile.options.draft, - default: false, - type: "boolean" - response.check = @_parseAttribute "check", - compile.options.check, - type: "string" - response.flags = @_parseAttribute "flags", - compile.options.flags, - default: [], - type: "object" - - # The syncType specifies whether the request contains all - # resources (full) or only those resources to be updated - # in-place (incremental). - response.syncType = @_parseAttribute "syncType", - compile.options.syncType, - validValues: ["full", "incremental"] - type: "string" - - # The syncState is an identifier passed in with the request - # which has the property that it changes when any resource is - # added, deleted, moved or renamed. - # - # on syncType full the syncState identifier is passed in and - # stored - # - # on syncType incremental the syncState identifier must match - # the stored value - response.syncState = @_parseAttribute "syncState", - compile.options.syncState, - type: "string" - - if response.timeout > RequestParser.MAX_TIMEOUT - response.timeout = RequestParser.MAX_TIMEOUT - response.timeout = response.timeout * 1000 # milliseconds - - response.resources = (@_parseResource(resource) for resource in (compile.resources or [])) - - rootResourcePath = @_parseAttribute "rootResourcePath", - compile.rootResourcePath - default: "main.tex" - type: "string" - originalRootResourcePath = rootResourcePath - sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath) - response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath) - - for resource in response.resources - if resource.path == originalRootResourcePath - resource.path = sanitizedRootResourcePath - catch error - return callback error - - callback null, response - - _parseResource: (resource) -> - if !resource.path? or typeof resource.path != "string" - throw "all resources should have a path attribute" - - if resource.modified? - modified = new Date(resource.modified) - if isNaN(modified.getTime()) - throw "resource modified date could not be understood: #{resource.modified}" - - if !resource.url? and !resource.content? - throw "all resources should have either a url or content attribute" - if resource.content? and typeof resource.content != "string" - throw "content attribute should be a string" - if resource.url? and typeof resource.url != "string" - throw "url attribute should be a string" - - return { - path: resource.path - modified: modified - url: resource.url - content: resource.content + if ((body.compile == null)) { + return callback("top level object should have a compile attribute"); } - _parseAttribute: (name, attribute, options) -> - if attribute? - if options.validValues? - if options.validValues.indexOf(attribute) == -1 - throw "#{name} attribute should be one of: #{options.validValues.join(", ")}" - if options.type? - if typeof attribute != options.type - throw "#{name} attribute should be a #{options.type}" - else - return options.default if options.default? - return attribute + const { compile } = body; + if (!compile.options) { compile.options = {}; } - _sanitizePath: (path) -> - # See http://php.net/manual/en/function.escapeshellcmd.php - path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "") + try { + response.compiler = this._parseAttribute("compiler", + compile.options.compiler, { + validValues: this.VALID_COMPILERS, + default: "pdflatex", + type: "string" + } + ); + response.timeout = this._parseAttribute("timeout", + compile.options.timeout, { + default: RequestParser.MAX_TIMEOUT, + type: "number" + } + ); + response.imageName = this._parseAttribute("imageName", + compile.options.imageName, + {type: "string"}); + response.draft = this._parseAttribute("draft", + compile.options.draft, { + default: false, + type: "boolean" + } + ); + response.check = this._parseAttribute("check", + compile.options.check, + {type: "string"}); + response.flags = this._parseAttribute("flags", + compile.options.flags, { + default: [], + type: "object" + } + ); - _checkPath: (path) -> - # check that the request does not use a relative path - for dir in path.split('/') - if dir == '..' - throw "relative path in root resource" - return path + // The syncType specifies whether the request contains all + // resources (full) or only those resources to be updated + // in-place (incremental). + response.syncType = this._parseAttribute("syncType", + compile.options.syncType, { + validValues: ["full", "incremental"], + type: "string" + } + ); + + // The syncState is an identifier passed in with the request + // which has the property that it changes when any resource is + // added, deleted, moved or renamed. + // + // on syncType full the syncState identifier is passed in and + // stored + // + // on syncType incremental the syncState identifier must match + // the stored value + response.syncState = this._parseAttribute("syncState", + compile.options.syncState, + {type: "string"}); + + if (response.timeout > RequestParser.MAX_TIMEOUT) { + response.timeout = RequestParser.MAX_TIMEOUT; + } + response.timeout = response.timeout * 1000; // milliseconds + + response.resources = ((() => { + const result = []; + for (resource of Array.from((compile.resources || []))) { result.push(this._parseResource(resource)); + } + return result; + })()); + + const rootResourcePath = this._parseAttribute("rootResourcePath", + compile.rootResourcePath, { + default: "main.tex", + type: "string" + } + ); + const originalRootResourcePath = rootResourcePath; + const sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath); + response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath); + + for (resource of Array.from(response.resources)) { + if (resource.path === originalRootResourcePath) { + resource.path = sanitizedRootResourcePath; + } + } + } catch (error1) { + const error = error1; + return callback(error); + } + + return callback(null, response); + }, + + _parseResource(resource) { + let modified; + if ((resource.path == null) || (typeof resource.path !== "string")) { + throw "all resources should have a path attribute"; + } + + if (resource.modified != null) { + modified = new Date(resource.modified); + if (isNaN(modified.getTime())) { + throw `resource modified date could not be understood: ${resource.modified}`; + } + } + + if ((resource.url == null) && (resource.content == null)) { + throw "all resources should have either a url or content attribute"; + } + if ((resource.content != null) && (typeof resource.content !== "string")) { + throw "content attribute should be a string"; + } + if ((resource.url != null) && (typeof resource.url !== "string")) { + throw "url attribute should be a string"; + } + + return { + path: resource.path, + modified, + url: resource.url, + content: resource.content + }; + }, + + _parseAttribute(name, attribute, options) { + if (attribute != null) { + if (options.validValues != null) { + if (options.validValues.indexOf(attribute) === -1) { + throw `${name} attribute should be one of: ${options.validValues.join(", ")}`; + } + } + if (options.type != null) { + if (typeof attribute !== options.type) { + throw `${name} attribute should be a ${options.type}`; + } + } + } else { + if (options.default != null) { return options.default; } + } + return attribute; + }, + + _sanitizePath(path) { + // See http://php.net/manual/en/function.escapeshellcmd.php + return path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, ""); + }, + + _checkPath(path) { + // check that the request does not use a relative path + for (let dir of Array.from(path.split('/'))) { + if (dir === '..') { + throw "relative path in root resource"; + } + } + return path; + } +}); diff --git a/app/coffee/ResourceStateManager.js b/app/coffee/ResourceStateManager.js index 19fea47..f430c8f 100644 --- a/app/coffee/ResourceStateManager.js +++ b/app/coffee/ResourceStateManager.js @@ -1,72 +1,108 @@ -Path = require "path" -fs = require "fs" -logger = require "logger-sharelatex" -settings = require("settings-sharelatex") -Errors = require "./Errors" -SafeReader = require "./SafeReader" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS201: Simplify complex destructure assignments + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ResourceStateManager; +const Path = require("path"); +const fs = require("fs"); +const logger = require("logger-sharelatex"); +const settings = require("settings-sharelatex"); +const Errors = require("./Errors"); +const SafeReader = require("./SafeReader"); -module.exports = ResourceStateManager = +module.exports = (ResourceStateManager = { - # The sync state is an identifier which must match for an - # incremental update to be allowed. - # - # The initial value is passed in and stored on a full - # compile, along with the list of resources.. - # - # Subsequent incremental compiles must come with the same value - if - # not they will be rejected with a 409 Conflict response. The - # previous list of resources is returned. - # - # An incremental compile can only update existing files with new - # content. The sync state identifier must change if any docs or - # files are moved, added, deleted or renamed. + // The sync state is an identifier which must match for an + // incremental update to be allowed. + // + // The initial value is passed in and stored on a full + // compile, along with the list of resources.. + // + // Subsequent incremental compiles must come with the same value - if + // not they will be rejected with a 409 Conflict response. The + // previous list of resources is returned. + // + // An incremental compile can only update existing files with new + // content. The sync state identifier must change if any docs or + // files are moved, added, deleted or renamed. - SYNC_STATE_FILE: ".project-sync-state" - SYNC_STATE_MAX_SIZE: 128*1024 + SYNC_STATE_FILE: ".project-sync-state", + SYNC_STATE_MAX_SIZE: 128*1024, - saveProjectState: (state, resources, basePath, callback = (error) ->) -> - stateFile = Path.join(basePath, @SYNC_STATE_FILE) - if not state? # remove the file if no state passed in - logger.log state:state, basePath:basePath, "clearing sync state" - fs.unlink stateFile, (err) -> - if err? and err.code isnt 'ENOENT' - return callback(err) - else - return callback() - else - logger.log state:state, basePath:basePath, "writing sync state" - resourceList = (resource.path for resource in resources) - fs.writeFile stateFile, [resourceList..., "stateHash:#{state}"].join("\n"), callback + saveProjectState(state, resources, basePath, callback) { + if (callback == null) { callback = function(error) {}; } + const stateFile = Path.join(basePath, this.SYNC_STATE_FILE); + if ((state == null)) { // remove the file if no state passed in + logger.log({state, basePath}, "clearing sync state"); + return fs.unlink(stateFile, function(err) { + if ((err != null) && (err.code !== 'ENOENT')) { + return callback(err); + } else { + return callback(); + } + }); + } else { + logger.log({state, basePath}, "writing sync state"); + const resourceList = (Array.from(resources).map((resource) => resource.path)); + return fs.writeFile(stateFile, [...Array.from(resourceList), `stateHash:${state}`].join("\n"), callback); + } + }, - checkProjectStateMatches: (state, basePath, callback = (error, resources) ->) -> - stateFile = Path.join(basePath, @SYNC_STATE_FILE) - size = @SYNC_STATE_MAX_SIZE - SafeReader.readFile stateFile, size, 'utf8', (err, result, bytesRead) -> - return callback(err) if err? - if bytesRead is size - logger.error file:stateFile, size:size, bytesRead:bytesRead, "project state file truncated" - [resourceList..., oldState] = result?.toString()?.split("\n") or [] - newState = "stateHash:#{state}" - logger.log state:state, oldState: oldState, basePath:basePath, stateMatches: (newState is oldState), "checking sync state" - if newState isnt oldState - return callback new Errors.FilesOutOfSyncError("invalid state for incremental update") - else - resources = ({path: path} for path in resourceList) - callback(null, resources) + checkProjectStateMatches(state, basePath, callback) { + if (callback == null) { callback = function(error, resources) {}; } + const stateFile = Path.join(basePath, this.SYNC_STATE_FILE); + const size = this.SYNC_STATE_MAX_SIZE; + return SafeReader.readFile(stateFile, size, 'utf8', function(err, result, bytesRead) { + if (err != null) { return callback(err); } + if (bytesRead === size) { + logger.error({file:stateFile, size, bytesRead}, "project state file truncated"); + } + const array = __guard__(result != null ? result.toString() : undefined, x => x.split("\n")) || [], + adjustedLength = Math.max(array.length, 1), + resourceList = array.slice(0, adjustedLength - 1), + oldState = array[adjustedLength - 1]; + const newState = `stateHash:${state}`; + logger.log({state, oldState, basePath, stateMatches: (newState === oldState)}, "checking sync state"); + if (newState !== oldState) { + return callback(new Errors.FilesOutOfSyncError("invalid state for incremental update")); + } else { + const resources = (Array.from(resourceList).map((path) => ({path}))); + return callback(null, resources); + } + }); + }, - checkResourceFiles: (resources, allFiles, basePath, callback = (error) ->) -> - # check the paths are all relative to current directory - for file in resources or [] - for dir in file?.path?.split('/') - if dir == '..' - return callback new Error("relative path in resource file list") - # check if any of the input files are not present in list of files - seenFile = {} - for file in allFiles - seenFile[file] = true - missingFiles = (resource.path for resource in resources when not seenFile[resource.path]) - if missingFiles?.length > 0 - logger.err missingFiles:missingFiles, basePath:basePath, allFiles:allFiles, resources:resources, "missing input files for project" - return callback new Errors.FilesOutOfSyncError("resource files missing in incremental update") - else - callback() + checkResourceFiles(resources, allFiles, basePath, callback) { + // check the paths are all relative to current directory + let file; + if (callback == null) { callback = function(error) {}; } + for (file of Array.from(resources || [])) { + for (let dir of Array.from(__guard__(file != null ? file.path : undefined, x => x.split('/')))) { + if (dir === '..') { + return callback(new Error("relative path in resource file list")); + } + } + } + // check if any of the input files are not present in list of files + const seenFile = {}; + for (file of Array.from(allFiles)) { + seenFile[file] = true; + } + const missingFiles = (Array.from(resources).filter((resource) => !seenFile[resource.path]).map((resource) => resource.path)); + if ((missingFiles != null ? missingFiles.length : undefined) > 0) { + logger.err({missingFiles, basePath, allFiles, resources}, "missing input files for project"); + return callback(new Errors.FilesOutOfSyncError("resource files missing in incremental update")); + } else { + return callback(); + } + } +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/app/coffee/ResourceWriter.js b/app/coffee/ResourceWriter.js index f3a4bd0..0044ad9 100644 --- a/app/coffee/ResourceWriter.js +++ b/app/coffee/ResourceWriter.js @@ -1,142 +1,206 @@ -UrlCache = require "./UrlCache" -Path = require "path" -fs = require "fs" -async = require "async" -mkdirp = require "mkdirp" -OutputFileFinder = require "./OutputFileFinder" -ResourceStateManager = require "./ResourceStateManager" -Metrics = require "./Metrics" -logger = require "logger-sharelatex" -settings = require("settings-sharelatex") +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ResourceWriter; +const UrlCache = require("./UrlCache"); +const Path = require("path"); +const fs = require("fs"); +const async = require("async"); +const mkdirp = require("mkdirp"); +const OutputFileFinder = require("./OutputFileFinder"); +const ResourceStateManager = require("./ResourceStateManager"); +const Metrics = require("./Metrics"); +const logger = require("logger-sharelatex"); +const settings = require("settings-sharelatex"); -parallelFileDownloads = settings.parallelFileDownloads or 1 +const parallelFileDownloads = settings.parallelFileDownloads || 1; -module.exports = ResourceWriter = +module.exports = (ResourceWriter = { - syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) -> - if request.syncType is "incremental" - logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync" - ResourceStateManager.checkProjectStateMatches request.syncState, basePath, (error, resourceList) -> - return callback(error) if error? - ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) -> - return callback(error) if error? - ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) -> - return callback(error) if error? - ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) -> - return callback(error) if error? - callback(null, resourceList) - else - logger.log project_id: request.project_id, user_id: request.user_id, "full sync" - @saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) -> - return callback(error) if error? - ResourceStateManager.saveProjectState request.syncState, request.resources, basePath, (error) -> - return callback(error) if error? - callback(null, request.resources) + syncResourcesToDisk(request, basePath, callback) { + if (callback == null) { callback = function(error, resourceList) {}; } + if (request.syncType === "incremental") { + logger.log({project_id: request.project_id, user_id: request.user_id}, "incremental sync"); + return ResourceStateManager.checkProjectStateMatches(request.syncState, basePath, function(error, resourceList) { + if (error != null) { return callback(error); } + return ResourceWriter._removeExtraneousFiles(resourceList, basePath, function(error, outputFiles, allFiles) { + if (error != null) { return callback(error); } + return ResourceStateManager.checkResourceFiles(resourceList, allFiles, basePath, function(error) { + if (error != null) { return callback(error); } + return ResourceWriter.saveIncrementalResourcesToDisk(request.project_id, request.resources, basePath, function(error) { + if (error != null) { return callback(error); } + return callback(null, resourceList); + }); + }); + }); + }); + } else { + logger.log({project_id: request.project_id, user_id: request.user_id}, "full sync"); + return this.saveAllResourcesToDisk(request.project_id, request.resources, basePath, function(error) { + if (error != null) { return callback(error); } + return ResourceStateManager.saveProjectState(request.syncState, request.resources, basePath, function(error) { + if (error != null) { return callback(error); } + return callback(null, request.resources); + }); + }); + } + }, - saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) -> - @_createDirectory basePath, (error) => - return callback(error) if error? - jobs = for resource in resources - do (resource) => - (callback) => @_writeResourceToDisk(project_id, resource, basePath, callback) - async.parallelLimit jobs, parallelFileDownloads, callback + saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) { + if (callback == null) { callback = function(error) {}; } + return this._createDirectory(basePath, error => { + if (error != null) { return callback(error); } + const jobs = Array.from(resources).map((resource) => + (resource => { + return callback => this._writeResourceToDisk(project_id, resource, basePath, callback); + })(resource)); + return async.parallelLimit(jobs, parallelFileDownloads, callback); + }); + }, - saveAllResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) -> - @_createDirectory basePath, (error) => - return callback(error) if error? - @_removeExtraneousFiles resources, basePath, (error) => - return callback(error) if error? - jobs = for resource in resources - do (resource) => - (callback) => @_writeResourceToDisk(project_id, resource, basePath, callback) - async.parallelLimit jobs, parallelFileDownloads, callback + saveAllResourcesToDisk(project_id, resources, basePath, callback) { + if (callback == null) { callback = function(error) {}; } + return this._createDirectory(basePath, error => { + if (error != null) { return callback(error); } + return this._removeExtraneousFiles(resources, basePath, error => { + if (error != null) { return callback(error); } + const jobs = Array.from(resources).map((resource) => + (resource => { + return callback => this._writeResourceToDisk(project_id, resource, basePath, callback); + })(resource)); + return async.parallelLimit(jobs, parallelFileDownloads, callback); + }); + }); + }, - _createDirectory: (basePath, callback = (error) ->) -> - fs.mkdir basePath, (err) -> - if err? - if err.code is 'EEXIST' - return callback() - else - logger.log {err: err, dir:basePath}, "error creating directory" - return callback(err) - else - return callback() + _createDirectory(basePath, callback) { + if (callback == null) { callback = function(error) {}; } + return fs.mkdir(basePath, function(err) { + if (err != null) { + if (err.code === 'EEXIST') { + return callback(); + } else { + logger.log({err, dir:basePath}, "error creating directory"); + return callback(err); + } + } else { + return callback(); + } + }); + }, - _removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) -> - timer = new Metrics.Timer("unlink-output-files") - callback = (error, result...) -> - timer.done() - _callback(error, result...) + _removeExtraneousFiles(resources, basePath, _callback) { + if (_callback == null) { _callback = function(error, outputFiles, allFiles) {}; } + const timer = new Metrics.Timer("unlink-output-files"); + const callback = function(error, ...result) { + timer.done(); + return _callback(error, ...Array.from(result)); + }; - OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) -> - return callback(error) if error? + return OutputFileFinder.findOutputFiles(resources, basePath, function(error, outputFiles, allFiles) { + if (error != null) { return callback(error); } - jobs = [] - for file in outputFiles or [] - do (file) -> - path = file.path - should_delete = true - if path.match(/^output\./) or path.match(/\.aux$/) or path.match(/^cache\//) # knitr cache - should_delete = false - if path.match(/^output-.*/) # Tikz cached figures (default case) - should_delete = false - if path.match(/\.(pdf|dpth|md5)$/) # Tikz cached figures (by extension) - should_delete = false - if path.match(/\.(pygtex|pygstyle)$/) or path.match(/(^|\/)_minted-[^\/]+\//) # minted files/directory - should_delete = false - if path.match(/\.md\.tex$/) or path.match(/(^|\/)_markdown_[^\/]+\//) # markdown files/directory - should_delete = false - if path.match(/-eps-converted-to\.pdf$/) # Epstopdf generated files - should_delete = false - if path == "output.pdf" or path == "output.dvi" or path == "output.log" or path == "output.xdv" - should_delete = true - if path == "output.tex" # created by TikzManager if present in output files - should_delete = true - if should_delete - jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback + const jobs = []; + for (let file of Array.from(outputFiles || [])) { + (function(file) { + const { path } = file; + let should_delete = true; + if (path.match(/^output\./) || path.match(/\.aux$/) || path.match(/^cache\//)) { // knitr cache + should_delete = false; + } + if (path.match(/^output-.*/)) { // Tikz cached figures (default case) + should_delete = false; + } + if (path.match(/\.(pdf|dpth|md5)$/)) { // Tikz cached figures (by extension) + should_delete = false; + } + if (path.match(/\.(pygtex|pygstyle)$/) || path.match(/(^|\/)_minted-[^\/]+\//)) { // minted files/directory + should_delete = false; + } + if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) { // markdown files/directory + should_delete = false; + } + if (path.match(/-eps-converted-to\.pdf$/)) { // Epstopdf generated files + should_delete = false; + } + if ((path === "output.pdf") || (path === "output.dvi") || (path === "output.log") || (path === "output.xdv")) { + should_delete = true; + } + if (path === "output.tex") { // created by TikzManager if present in output files + should_delete = true; + } + if (should_delete) { + return jobs.push(callback => ResourceWriter._deleteFileIfNotDirectory(Path.join(basePath, path), callback)); + } + })(file); + } - async.series jobs, (error) -> - return callback(error) if error? - callback(null, outputFiles, allFiles) + return async.series(jobs, function(error) { + if (error != null) { return callback(error); } + return callback(null, outputFiles, allFiles); + }); + }); + }, - _deleteFileIfNotDirectory: (path, callback = (error) ->) -> - fs.stat path, (error, stat) -> - if error? and error.code is 'ENOENT' - return callback() - else if error? - logger.err {err: error, path: path}, "error stating file in deleteFileIfNotDirectory" - return callback(error) - else if stat.isFile() - fs.unlink path, (error) -> - if error? - logger.err {err: error, path: path}, "error removing file in deleteFileIfNotDirectory" - callback(error) - else - callback() - else - callback() + _deleteFileIfNotDirectory(path, callback) { + if (callback == null) { callback = function(error) {}; } + return fs.stat(path, function(error, stat) { + if ((error != null) && (error.code === 'ENOENT')) { + return callback(); + } else if (error != null) { + logger.err({err: error, path}, "error stating file in deleteFileIfNotDirectory"); + return callback(error); + } else if (stat.isFile()) { + return fs.unlink(path, function(error) { + if (error != null) { + logger.err({err: error, path}, "error removing file in deleteFileIfNotDirectory"); + return callback(error); + } else { + return callback(); + } + }); + } else { + return callback(); + } + }); + }, - _writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) -> - ResourceWriter.checkPath basePath, resource.path, (error, path) -> - return callback(error) if error? - mkdirp Path.dirname(path), (error) -> - return callback(error) if error? - # TODO: Don't overwrite file if it hasn't been modified - if resource.url? - UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)-> - if err? - logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources" - callback() #try and continue compiling even if http resource can not be downloaded at this time - else - process = require("process") - fs.writeFile path, resource.content, callback - try - result = fs.lstatSync(path) - catch e + _writeResourceToDisk(project_id, resource, basePath, callback) { + if (callback == null) { callback = function(error) {}; } + return ResourceWriter.checkPath(basePath, resource.path, function(error, path) { + if (error != null) { return callback(error); } + return mkdirp(Path.dirname(path), function(error) { + if (error != null) { return callback(error); } + // TODO: Don't overwrite file if it hasn't been modified + if (resource.url != null) { + return UrlCache.downloadUrlToFile(project_id, resource.url, path, resource.modified, function(err){ + if (err != null) { + logger.err({err, project_id, path, resource_url:resource.url, modified:resource.modified}, "error downloading file for resources"); + } + return callback(); + }); //try and continue compiling even if http resource can not be downloaded at this time + } else { + const process = require("process"); + fs.writeFile(path, resource.content, callback); + try { + let result; + return result = fs.lstatSync(path); + } catch (e) {} + } + }); + }); + }, - checkPath: (basePath, resourcePath, callback) -> - path = Path.normalize(Path.join(basePath, resourcePath)) - if (path.slice(0, basePath.length + 1) != basePath + "/") - return callback new Error("resource path is outside root directory") - else - return callback(null, path) + checkPath(basePath, resourcePath, callback) { + const path = Path.normalize(Path.join(basePath, resourcePath)); + if (path.slice(0, basePath.length + 1) !== (basePath + "/")) { + return callback(new Error("resource path is outside root directory")); + } else { + return callback(null, path); + } + } +}); diff --git a/app/coffee/SafeReader.js b/app/coffee/SafeReader.js index adb96b1..f1a6349 100644 --- a/app/coffee/SafeReader.js +++ b/app/coffee/SafeReader.js @@ -1,25 +1,40 @@ -fs = require "fs" -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let SafeReader; +const fs = require("fs"); +const logger = require("logger-sharelatex"); -module.exports = SafeReader = +module.exports = (SafeReader = { - # safely read up to size bytes from a file and return result as a - # string + // safely read up to size bytes from a file and return result as a + // string - readFile: (file, size, encoding, callback = (error, result) ->) -> - fs.open file, 'r', (err, fd) -> - return callback() if err? and err.code is 'ENOENT' - return callback(err) if err? + readFile(file, size, encoding, callback) { + if (callback == null) { callback = function(error, result) {}; } + return fs.open(file, 'r', function(err, fd) { + if ((err != null) && (err.code === 'ENOENT')) { return callback(); } + if (err != null) { return callback(err); } - # safely return always closing the file - callbackWithClose = (err, result...) -> - fs.close fd, (err1) -> - return callback(err) if err? - return callback(err1) if err1? - callback(null, result...) + // safely return always closing the file + const callbackWithClose = (err, ...result) => + fs.close(fd, function(err1) { + if (err != null) { return callback(err); } + if (err1 != null) { return callback(err1); } + return callback(null, ...Array.from(result)); + }) + ; - buff = new Buffer(size, 0) # fill with zeros - fs.read fd, buff, 0, buff.length, 0, (err, bytesRead, buffer) -> - return callbackWithClose(err) if err? - result = buffer.toString(encoding, 0, bytesRead) - callbackWithClose(null, result, bytesRead) + const buff = new Buffer(size, 0); // fill with zeros + return fs.read(fd, buff, 0, buff.length, 0, function(err, bytesRead, buffer) { + if (err != null) { return callbackWithClose(err); } + const result = buffer.toString(encoding, 0, bytesRead); + return callbackWithClose(null, result, bytesRead); + }); + }); + } +}); diff --git a/app/coffee/StaticServerForbidSymlinks.js b/app/coffee/StaticServerForbidSymlinks.js index 1b3cd45..90a8879 100644 --- a/app/coffee/StaticServerForbidSymlinks.js +++ b/app/coffee/StaticServerForbidSymlinks.js @@ -1,41 +1,64 @@ -Path = require("path") -fs = require("fs") -Settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -url = require "url" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ForbidSymlinks; +const Path = require("path"); +const fs = require("fs"); +const Settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const url = require("url"); -module.exports = ForbidSymlinks = (staticFn, root, options) -> - expressStatic = staticFn root, options - basePath = Path.resolve(root) - return (req, res, next) -> - path = url.parse(req.url)?.pathname - # check that the path is of the form /project_id_or_name/path/to/file.log - if result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/) - project_id = result[1] - file = result[2] - else - logger.warn path: path, "unrecognized file request" - return res.sendStatus(404) - # check that the file does not use a relative path - for dir in file.split('/') - if dir == '..' - logger.warn path: path, "attempt to use a relative path" - return res.sendStatus(404) - # check that the requested path is normalized - requestedFsPath = "#{basePath}/#{project_id}/#{file}" - if requestedFsPath != Path.normalize(requestedFsPath) - logger.error path: requestedFsPath, "requestedFsPath is not normalized" - return res.sendStatus(404) - # check that the requested path is not a symlink - fs.realpath requestedFsPath, (err, realFsPath)-> - if err? - if err.code == 'ENOENT' - return res.sendStatus(404) - else - logger.error err:err, requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "error checking file access" - return res.sendStatus(500) - else if requestedFsPath != realFsPath - logger.warn requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "trying to access a different file (symlink), aborting" - return res.sendStatus(404) - else - expressStatic(req, res, next) +module.exports = (ForbidSymlinks = function(staticFn, root, options) { + const expressStatic = staticFn(root, options); + const basePath = Path.resolve(root); + return function(req, res, next) { + let file, project_id, result; + const path = __guard__(url.parse(req.url), x => x.pathname); + // check that the path is of the form /project_id_or_name/path/to/file.log + if (result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/)) { + project_id = result[1]; + file = result[2]; + } else { + logger.warn({path}, "unrecognized file request"); + return res.sendStatus(404); + } + // check that the file does not use a relative path + for (let dir of Array.from(file.split('/'))) { + if (dir === '..') { + logger.warn({path}, "attempt to use a relative path"); + return res.sendStatus(404); + } + } + // check that the requested path is normalized + const requestedFsPath = `${basePath}/${project_id}/${file}`; + if (requestedFsPath !== Path.normalize(requestedFsPath)) { + logger.error({path: requestedFsPath}, "requestedFsPath is not normalized"); + return res.sendStatus(404); + } + // check that the requested path is not a symlink + return fs.realpath(requestedFsPath, function(err, realFsPath){ + if (err != null) { + if (err.code === 'ENOENT') { + return res.sendStatus(404); + } else { + logger.error({err, requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "error checking file access"); + return res.sendStatus(500); + } + } else if (requestedFsPath !== realFsPath) { + logger.warn({requestedFsPath, realFsPath, path: req.params[0], project_id: req.params.project_id}, "trying to access a different file (symlink), aborting"); + return res.sendStatus(404); + } else { + return expressStatic(req, res, next); + } + }); + }; +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/app/coffee/TikzManager.js b/app/coffee/TikzManager.js index 22def27..fb52644 100644 --- a/app/coffee/TikzManager.js +++ b/app/coffee/TikzManager.js @@ -1,37 +1,56 @@ -fs = require "fs" -Path = require "path" -ResourceWriter = require "./ResourceWriter" -SafeReader = require "./SafeReader" -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let TikzManager; +const fs = require("fs"); +const Path = require("path"); +const ResourceWriter = require("./ResourceWriter"); +const SafeReader = require("./SafeReader"); +const logger = require("logger-sharelatex"); -# for \tikzexternalize or pstool to work the main file needs to match the -# jobname. Since we set the -jobname to output, we have to create a -# copy of the main file as 'output.tex'. +// for \tikzexternalize or pstool to work the main file needs to match the +// jobname. Since we set the -jobname to output, we have to create a +// copy of the main file as 'output.tex'. -module.exports = TikzManager = +module.exports = (TikzManager = { - checkMainFile: (compileDir, mainFile, resources, callback = (error, needsMainFile) ->) -> - # if there's already an output.tex file, we don't want to touch it - for resource in resources - if resource.path is "output.tex" - logger.log compileDir: compileDir, mainFile: mainFile, "output.tex already in resources" - return callback(null, false) - # if there's no output.tex, see if we are using tikz/pgf or pstool in the main file - ResourceWriter.checkPath compileDir, mainFile, (error, path) -> - return callback(error) if error? - SafeReader.readFile path, 65536, "utf8", (error, content) -> - return callback(error) if error? - usesTikzExternalize = content?.indexOf("\\tikzexternalize") >= 0 - usesPsTool = content?.indexOf("{pstool}") >= 0 - logger.log compileDir: compileDir, mainFile: mainFile, usesTikzExternalize:usesTikzExternalize, usesPsTool: usesPsTool, "checked for packages needing main file as output.tex" - needsMainFile = (usesTikzExternalize || usesPsTool) - callback null, needsMainFile + checkMainFile(compileDir, mainFile, resources, callback) { + // if there's already an output.tex file, we don't want to touch it + if (callback == null) { callback = function(error, needsMainFile) {}; } + for (let resource of Array.from(resources)) { + if (resource.path === "output.tex") { + logger.log({compileDir, mainFile}, "output.tex already in resources"); + return callback(null, false); + } + } + // if there's no output.tex, see if we are using tikz/pgf or pstool in the main file + return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) { + if (error != null) { return callback(error); } + return SafeReader.readFile(path, 65536, "utf8", function(error, content) { + if (error != null) { return callback(error); } + const usesTikzExternalize = (content != null ? content.indexOf("\\tikzexternalize") : undefined) >= 0; + const usesPsTool = (content != null ? content.indexOf("{pstool}") : undefined) >= 0; + logger.log({compileDir, mainFile, usesTikzExternalize, usesPsTool}, "checked for packages needing main file as output.tex"); + const needsMainFile = (usesTikzExternalize || usesPsTool); + return callback(null, needsMainFile); + }); + }); + }, - injectOutputFile: (compileDir, mainFile, callback = (error) ->) -> - ResourceWriter.checkPath compileDir, mainFile, (error, path) -> - return callback(error) if error? - fs.readFile path, "utf8", (error, content) -> - return callback(error) if error? - logger.log compileDir: compileDir, mainFile: mainFile, "copied file to output.tex as project uses packages which require it" - # use wx flag to ensure that output file does not already exist - fs.writeFile Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback + injectOutputFile(compileDir, mainFile, callback) { + if (callback == null) { callback = function(error) {}; } + return ResourceWriter.checkPath(compileDir, mainFile, function(error, path) { + if (error != null) { return callback(error); } + return fs.readFile(path, "utf8", function(error, content) { + if (error != null) { return callback(error); } + logger.log({compileDir, mainFile}, "copied file to output.tex as project uses packages which require it"); + // use wx flag to ensure that output file does not already exist + return fs.writeFile(Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback); + }); + }); + } +}); diff --git a/app/coffee/UrlCache.js b/app/coffee/UrlCache.js index d44479a..9a19968 100644 --- a/app/coffee/UrlCache.js +++ b/app/coffee/UrlCache.js @@ -1,134 +1,189 @@ -db = require("./db") -dbQueue = require "./DbQueue" -UrlFetcher = require("./UrlFetcher") -Settings = require("settings-sharelatex") -crypto = require("crypto") -fs = require("fs") -logger = require "logger-sharelatex" -async = require "async" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let UrlCache; +const db = require("./db"); +const dbQueue = require("./DbQueue"); +const UrlFetcher = require("./UrlFetcher"); +const Settings = require("settings-sharelatex"); +const crypto = require("crypto"); +const fs = require("fs"); +const logger = require("logger-sharelatex"); +const async = require("async"); -module.exports = UrlCache = - downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) -> - UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) => - return callback(error) if error? - UrlCache._copyFile pathToCachedUrl, destPath, (error) -> - if error? - UrlCache._clearUrlDetails project_id, url, () -> - callback(error) - else - callback(error) +module.exports = (UrlCache = { + downloadUrlToFile(project_id, url, destPath, lastModified, callback) { + if (callback == null) { callback = function(error) {}; } + return UrlCache._ensureUrlIsInCache(project_id, url, lastModified, (error, pathToCachedUrl) => { + if (error != null) { return callback(error); } + return UrlCache._copyFile(pathToCachedUrl, destPath, function(error) { + if (error != null) { + return UrlCache._clearUrlDetails(project_id, url, () => callback(error)); + } else { + return callback(error); + } + }); + }); + }, - clearProject: (project_id, callback = (error) ->) -> - UrlCache._findAllUrlsInProject project_id, (error, urls) -> - logger.log project_id: project_id, url_count: urls.length, "clearing project URLs" - return callback(error) if error? - jobs = for url in (urls or []) - do (url) -> - (callback) -> - UrlCache._clearUrlFromCache project_id, url, (error) -> - if error? - logger.error err: error, project_id: project_id, url: url, "error clearing project URL" - callback() - async.series jobs, callback + clearProject(project_id, callback) { + if (callback == null) { callback = function(error) {}; } + return UrlCache._findAllUrlsInProject(project_id, function(error, urls) { + logger.log({project_id, url_count: urls.length}, "clearing project URLs"); + if (error != null) { return callback(error); } + const jobs = (Array.from(urls || [])).map((url) => + (url => + callback => + UrlCache._clearUrlFromCache(project_id, url, function(error) { + if (error != null) { + logger.error({err: error, project_id, url}, "error clearing project URL"); + } + return callback(); + }) + + )(url)); + return async.series(jobs, callback); + }); + }, - _ensureUrlIsInCache: (project_id, url, lastModified, callback = (error, pathOnDisk) ->) -> - if lastModified? - # MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds. - # So round down to seconds - lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000) - UrlCache._doesUrlNeedDownloading project_id, url, lastModified, (error, needsDownloading) => - return callback(error) if error? - if needsDownloading - logger.log url: url, lastModified: lastModified, "downloading URL" - UrlFetcher.pipeUrlToFile url, UrlCache._cacheFilePathForUrl(project_id, url), (error) => - return callback(error) if error? - UrlCache._updateOrCreateUrlDetails project_id, url, lastModified, (error) => - return callback(error) if error? - callback null, UrlCache._cacheFilePathForUrl(project_id, url) - else - logger.log url: url, lastModified: lastModified, "URL is up to date in cache" - callback null, UrlCache._cacheFilePathForUrl(project_id, url) + _ensureUrlIsInCache(project_id, url, lastModified, callback) { + if (callback == null) { callback = function(error, pathOnDisk) {}; } + if (lastModified != null) { + // MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds. + // So round down to seconds + lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000); + } + return UrlCache._doesUrlNeedDownloading(project_id, url, lastModified, (error, needsDownloading) => { + if (error != null) { return callback(error); } + if (needsDownloading) { + logger.log({url, lastModified}, "downloading URL"); + return UrlFetcher.pipeUrlToFile(url, UrlCache._cacheFilePathForUrl(project_id, url), error => { + if (error != null) { return callback(error); } + return UrlCache._updateOrCreateUrlDetails(project_id, url, lastModified, error => { + if (error != null) { return callback(error); } + return callback(null, UrlCache._cacheFilePathForUrl(project_id, url)); + }); + }); + } else { + logger.log({url, lastModified}, "URL is up to date in cache"); + return callback(null, UrlCache._cacheFilePathForUrl(project_id, url)); + } + }); + }, - _doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) -> - if !lastModified? - return callback null, true - UrlCache._findUrlDetails project_id, url, (error, urlDetails) -> - return callback(error) if error? - if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime() - return callback null, true - else - return callback null, false + _doesUrlNeedDownloading(project_id, url, lastModified, callback) { + if (callback == null) { callback = function(error, needsDownloading) {}; } + if ((lastModified == null)) { + return callback(null, true); + } + return UrlCache._findUrlDetails(project_id, url, function(error, urlDetails) { + if (error != null) { return callback(error); } + if ((urlDetails == null) || (urlDetails.lastModified == null) || (urlDetails.lastModified.getTime() < lastModified.getTime())) { + return callback(null, true); + } else { + return callback(null, false); + } + }); + }, - _cacheFileNameForUrl: (project_id, url) -> - project_id + ":" + crypto.createHash("md5").update(url).digest("hex") + _cacheFileNameForUrl(project_id, url) { + return project_id + ":" + crypto.createHash("md5").update(url).digest("hex"); + }, - _cacheFilePathForUrl: (project_id, url) -> - "#{Settings.path.clsiCacheDir}/#{UrlCache._cacheFileNameForUrl(project_id, url)}" + _cacheFilePathForUrl(project_id, url) { + return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(project_id, url)}`; + }, - _copyFile: (from, to, _callback = (error) ->) -> - callbackOnce = (error) -> - if error? - logger.error err: error, from:from, to:to, "error copying file from cache" - _callback(error) - _callback = () -> - writeStream = fs.createWriteStream(to) - readStream = fs.createReadStream(from) - writeStream.on "error", callbackOnce - readStream.on "error", callbackOnce - writeStream.on "close", callbackOnce - writeStream.on "open", () -> - readStream.pipe(writeStream) + _copyFile(from, to, _callback) { + if (_callback == null) { _callback = function(error) {}; } + const callbackOnce = function(error) { + if (error != null) { + logger.error({err: error, from, to}, "error copying file from cache"); + } + _callback(error); + return _callback = function() {}; + }; + const writeStream = fs.createWriteStream(to); + const readStream = fs.createReadStream(from); + writeStream.on("error", callbackOnce); + readStream.on("error", callbackOnce); + writeStream.on("close", callbackOnce); + return writeStream.on("open", () => readStream.pipe(writeStream)); + }, - _clearUrlFromCache: (project_id, url, callback = (error) ->) -> - UrlCache._clearUrlDetails project_id, url, (error) -> - return callback(error) if error? - UrlCache._deleteUrlCacheFromDisk project_id, url, (error) -> - return callback(error) if error? - callback null + _clearUrlFromCache(project_id, url, callback) { + if (callback == null) { callback = function(error) {}; } + return UrlCache._clearUrlDetails(project_id, url, function(error) { + if (error != null) { return callback(error); } + return UrlCache._deleteUrlCacheFromDisk(project_id, url, function(error) { + if (error != null) { return callback(error); } + return callback(null); + }); + }); + }, - _deleteUrlCacheFromDisk: (project_id, url, callback = (error) ->) -> - fs.unlink UrlCache._cacheFilePathForUrl(project_id, url), (error) -> - if error? and error.code != 'ENOENT' # no error if the file isn't present - return callback(error) - else - return callback() + _deleteUrlCacheFromDisk(project_id, url, callback) { + if (callback == null) { callback = function(error) {}; } + return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function(error) { + if ((error != null) && (error.code !== 'ENOENT')) { // no error if the file isn't present + return callback(error); + } else { + return callback(); + } + }); + }, - _findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) -> - job = (cb)-> - db.UrlCache.find(where: { url: url, project_id: project_id }) - .then((urlDetails) -> cb null, urlDetails) - .error cb - dbQueue.queue.push job, callback + _findUrlDetails(project_id, url, callback) { + if (callback == null) { callback = function(error, urlDetails) {}; } + const job = cb=> + db.UrlCache.find({where: { url, project_id }}) + .then(urlDetails => cb(null, urlDetails)) + .error(cb) + ; + return dbQueue.queue.push(job, callback); + }, - _updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) -> - job = (cb)-> - db.UrlCache.findOrCreate(where: {url: url, project_id: project_id}) + _updateOrCreateUrlDetails(project_id, url, lastModified, callback) { + if (callback == null) { callback = function(error) {}; } + const job = cb=> + db.UrlCache.findOrCreate({where: {url, project_id}}) .spread( - (urlDetails, created) -> - urlDetails.updateAttributes(lastModified: lastModified) - .then(() -> cb()) + (urlDetails, created) => + urlDetails.updateAttributes({lastModified}) + .then(() => cb()) .error(cb) ) - .error cb - dbQueue.queue.push(job, callback) + .error(cb) + ; + return dbQueue.queue.push(job, callback); + }, - _clearUrlDetails: (project_id, url, callback = (error) ->) -> - job = (cb)-> - db.UrlCache.destroy(where: {url: url, project_id: project_id}) - .then(() -> cb null) - .error cb - dbQueue.queue.push(job, callback) + _clearUrlDetails(project_id, url, callback) { + if (callback == null) { callback = function(error) {}; } + const job = cb=> + db.UrlCache.destroy({where: {url, project_id}}) + .then(() => cb(null)) + .error(cb) + ; + return dbQueue.queue.push(job, callback); + }, - _findAllUrlsInProject: (project_id, callback = (error, urls) ->) -> - job = (cb)-> - db.UrlCache.findAll(where: { project_id: project_id }) + _findAllUrlsInProject(project_id, callback) { + if (callback == null) { callback = function(error, urls) {}; } + const job = cb=> + db.UrlCache.findAll({where: { project_id }}) .then( - (urlEntries) -> - cb null, urlEntries.map((entry) -> entry.url) - ) - .error cb - dbQueue.queue.push(job, callback) + urlEntries => cb(null, urlEntries.map(entry => entry.url))) + .error(cb) + ; + return dbQueue.queue.push(job, callback); + } +}); diff --git a/app/coffee/UrlFetcher.js b/app/coffee/UrlFetcher.js index da10859..ea20956 100644 --- a/app/coffee/UrlFetcher.js +++ b/app/coffee/UrlFetcher.js @@ -1,70 +1,88 @@ -request = require("request").defaults(jar: false) -fs = require("fs") -logger = require "logger-sharelatex" -settings = require("settings-sharelatex") -URL = require('url'); +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let UrlFetcher; +const request = require("request").defaults({jar: false}); +const fs = require("fs"); +const logger = require("logger-sharelatex"); +const settings = require("settings-sharelatex"); +const URL = require('url'); -oneMinute = 60 * 1000 +const oneMinute = 60 * 1000; -module.exports = UrlFetcher = - pipeUrlToFile: (url, filePath, _callback = (error) ->) -> - callbackOnce = (error) -> - clearTimeout timeoutHandler if timeoutHandler? - _callback(error) - _callback = () -> +module.exports = (UrlFetcher = { + pipeUrlToFile(url, filePath, _callback) { + if (_callback == null) { _callback = function(error) {}; } + const callbackOnce = function(error) { + if (timeoutHandler != null) { clearTimeout(timeoutHandler); } + _callback(error); + return _callback = function() {}; + }; - if settings.filestoreDomainOveride? - p = URL.parse(url).path - url = "#{settings.filestoreDomainOveride}#{p}" - timeoutHandler = setTimeout () -> - timeoutHandler = null - logger.error url:url, filePath: filePath, "Timed out downloading file to cache" - callbackOnce(new Error("Timed out downloading file to cache #{url}")) - # FIXME: maybe need to close fileStream here - , 3 * oneMinute + if (settings.filestoreDomainOveride != null) { + const p = URL.parse(url).path; + url = `${settings.filestoreDomainOveride}${p}`; + } + var timeoutHandler = setTimeout(function() { + timeoutHandler = null; + logger.error({url, filePath}, "Timed out downloading file to cache"); + return callbackOnce(new Error(`Timed out downloading file to cache ${url}`)); + } + // FIXME: maybe need to close fileStream here + , 3 * oneMinute); - logger.log url:url, filePath: filePath, "started downloading url to cache" - urlStream = request.get({url: url, timeout: oneMinute}) - urlStream.pause() # stop data flowing until we are ready + logger.log({url, filePath}, "started downloading url to cache"); + const urlStream = request.get({url, timeout: oneMinute}); + urlStream.pause(); // stop data flowing until we are ready - # attach handlers before setting up pipes - urlStream.on "error", (error) -> - logger.error err: error, url:url, filePath: filePath, "error downloading url" - callbackOnce(error or new Error("Something went wrong downloading the URL #{url}")) + // attach handlers before setting up pipes + urlStream.on("error", function(error) { + logger.error({err: error, url, filePath}, "error downloading url"); + return callbackOnce(error || new Error(`Something went wrong downloading the URL ${url}`)); + }); - urlStream.on "end", () -> - logger.log url:url, filePath: filePath, "finished downloading file into cache" + urlStream.on("end", () => logger.log({url, filePath}, "finished downloading file into cache")); - urlStream.on "response", (res) -> - if res.statusCode >= 200 and res.statusCode < 300 - fileStream = fs.createWriteStream(filePath) + return urlStream.on("response", function(res) { + if ((res.statusCode >= 200) && (res.statusCode < 300)) { + const fileStream = fs.createWriteStream(filePath); - # attach handlers before setting up pipes - fileStream.on 'error', (error) -> - logger.error err: error, url:url, filePath: filePath, "error writing file into cache" - fs.unlink filePath, (err) -> - if err? - logger.err err: err, filePath: filePath, "error deleting file from cache" - callbackOnce(error) + // attach handlers before setting up pipes + fileStream.on('error', function(error) { + logger.error({err: error, url, filePath}, "error writing file into cache"); + return fs.unlink(filePath, function(err) { + if (err != null) { + logger.err({err, filePath}, "error deleting file from cache"); + } + return callbackOnce(error); + }); + }); - fileStream.on 'finish', () -> - logger.log url:url, filePath: filePath, "finished writing file into cache" - callbackOnce() + fileStream.on('finish', function() { + logger.log({url, filePath}, "finished writing file into cache"); + return callbackOnce(); + }); - fileStream.on 'pipe', () -> - logger.log url:url, filePath: filePath, "piping into filestream" + fileStream.on('pipe', () => logger.log({url, filePath}, "piping into filestream")); - urlStream.pipe(fileStream) - urlStream.resume() # now we are ready to handle the data - else - logger.error statusCode: res.statusCode, url:url, filePath: filePath, "unexpected status code downloading url to cache" - # https://nodejs.org/api/http.html#http_class_http_clientrequest - # If you add a 'response' event handler, then you must consume - # the data from the response object, either by calling - # response.read() whenever there is a 'readable' event, or by - # adding a 'data' handler, or by calling the .resume() - # method. Until the data is consumed, the 'end' event will not - # fire. Also, until the data is read it will consume memory - # that can eventually lead to a 'process out of memory' error. - urlStream.resume() # discard the data - callbackOnce(new Error("URL returned non-success status code: #{res.statusCode} #{url}")) + urlStream.pipe(fileStream); + return urlStream.resume(); // now we are ready to handle the data + } else { + logger.error({statusCode: res.statusCode, url, filePath}, "unexpected status code downloading url to cache"); + // https://nodejs.org/api/http.html#http_class_http_clientrequest + // If you add a 'response' event handler, then you must consume + // the data from the response object, either by calling + // response.read() whenever there is a 'readable' event, or by + // adding a 'data' handler, or by calling the .resume() + // method. Until the data is consumed, the 'end' event will not + // fire. Also, until the data is read it will consume memory + // that can eventually lead to a 'process out of memory' error. + urlStream.resume(); // discard the data + return callbackOnce(new Error(`URL returned non-success status code: ${res.statusCode} ${url}`)); + } + }); + } +}); diff --git a/app/coffee/db.js b/app/coffee/db.js index de48dfd..385ad8d 100644 --- a/app/coffee/db.js +++ b/app/coffee/db.js @@ -1,55 +1,59 @@ -Sequelize = require("sequelize") -Settings = require("settings-sharelatex") -_ = require("underscore") -logger = require "logger-sharelatex" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const Sequelize = require("sequelize"); +const Settings = require("settings-sharelatex"); +const _ = require("underscore"); +const logger = require("logger-sharelatex"); -options = _.extend {logging:false}, Settings.mysql.clsi +const options = _.extend({logging:false}, Settings.mysql.clsi); -logger.log dbPath:Settings.mysql.clsi.storage, "connecting to db" +logger.log({dbPath:Settings.mysql.clsi.storage}, "connecting to db"); -sequelize = new Sequelize( +const sequelize = new Sequelize( Settings.mysql.clsi.database, Settings.mysql.clsi.username, Settings.mysql.clsi.password, options -) +); -if Settings.mysql.clsi.dialect == "sqlite" - logger.log "running PRAGMA journal_mode=WAL;" - sequelize.query("PRAGMA journal_mode=WAL;") - sequelize.query("PRAGMA synchronous=OFF;") - sequelize.query("PRAGMA read_uncommitted = true;") +if (Settings.mysql.clsi.dialect === "sqlite") { + logger.log("running PRAGMA journal_mode=WAL;"); + sequelize.query("PRAGMA journal_mode=WAL;"); + sequelize.query("PRAGMA synchronous=OFF;"); + sequelize.query("PRAGMA read_uncommitted = true;"); +} -module.exports = +module.exports = { UrlCache: sequelize.define("UrlCache", { - url: Sequelize.STRING - project_id: Sequelize.STRING + url: Sequelize.STRING, + project_id: Sequelize.STRING, lastModified: Sequelize.DATE }, { indexes: [ {fields: ['url', 'project_id']}, {fields: ['project_id']} ] - }) + }), Project: sequelize.define("Project", { - project_id: {type: Sequelize.STRING, primaryKey: true} + project_id: {type: Sequelize.STRING, primaryKey: true}, lastAccessed: Sequelize.DATE }, { indexes: [ {fields: ['lastAccessed']} ] - }) + }), - op: Sequelize.Op + op: Sequelize.Op, - sync: () -> - logger.log dbPath:Settings.mysql.clsi.storage, "syncing db schema" - sequelize.sync() - .then(-> - logger.log "db sync complete" - ).catch((err)-> - console.log err, "error syncing" - ) + sync() { + logger.log({dbPath:Settings.mysql.clsi.storage}, "syncing db schema"); + return sequelize.sync() + .then(() => logger.log("db sync complete")).catch(err=> console.log(err, "error syncing")); + } +};