diff --git a/app.coffee b/app.coffee index fb3224e..d13798c 100644 --- a/app.coffee +++ b/app.coffee @@ -56,6 +56,7 @@ app.param 'build_id', (req, res, next, build_id) -> app.post "/project/:project_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile +app.post "/project/:project_id/compile/stop", CompileController.stopCompile app.delete "/project/:project_id", CompileController.clearCache app.get "/project/:project_id/sync/code", CompileController.syncFromCode @@ -65,6 +66,7 @@ app.get "/project/:project_id/status", CompileController.status # Per-user containers app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile +app.post "/project/:project_id/user/:user_id/compile/stop", CompileController.stopCompile app.delete "/project/:project_id/user/:user_id", CompileController.clearCache app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode diff --git a/app/coffee/CommandRunner.coffee b/app/coffee/CommandRunner.coffee index 5ea6765..f47af00 100644 --- a/app/coffee/CommandRunner.coffee +++ b/app/coffee/CommandRunner.coffee @@ -4,14 +4,41 @@ logger = require "logger-sharelatex" logger.info "using standard command runner" module.exports = CommandRunner = - run: (project_id, command, directory, image, timeout, callback = (error) ->) -> + run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) -> command = (arg.replace('$COMPILE_DIR', directory) for arg in command) logger.log project_id: project_id, command: command, directory: directory, "running command" logger.warn "timeouts and sandboxing are not enabled with CommandRunner" - proc = spawn command[0], command.slice(1), stdio: "inherit", cwd: directory + # merge environment settings + env = {} + env[key] = value for key, value of process.env + env[key] = value for key, value of environment + + # run command as detached process so it has its own process group (which can be killed if needed) + proc = spawn command[0], command.slice(1), stdio: "inherit", cwd: directory, detached: true, env: env + proc.on "error", (err)-> logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command" callback(err) - proc.on "close", () -> - callback() \ No newline at end of file + + proc.on "close", (code, signal) -> + logger.info code:code, signal:signal, project_id:project_id, "command exited" + if signal is 'SIGTERM' # signal from kill method below + err = new Error("terminated") + err.terminated = true + return callback(err) + else if code is 1 # exit status from chktex + err = new Error("exited") + err.code = code + return callback(err) + else + callback() + + return proc.pid # return process id to allow job to be killed if necessary + + kill: (pid, callback = (error) ->) -> + try + process.kill -pid # kill all processes in group + catch err + return callback(err) + callback() diff --git a/app/coffee/CompileController.coffee b/app/coffee/CompileController.coffee index d4dddd4..56237bf 100644 --- a/app/coffee/CompileController.coffee +++ b/app/coffee/CompileController.coffee @@ -15,7 +15,11 @@ module.exports = CompileController = ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) -> return next(error) if error? CompileManager.doCompile request, (error, outputFiles = []) -> - if error? + if error?.terminated + status = "terminated" + else if error?.validate + status = "validation-#{error.validate}" + else if error? logger.error err: error, project_id: request.project_id, "error running compile" if error.timedout status = "timedout" @@ -43,7 +47,13 @@ module.exports = CompileController = type: file.type build: file.build } - + + stopCompile: (req, res, next) -> + {project_id, user_id} = req.params + CompileManager.stopCompile project_id, user_id, (error) -> + return next(error) if error? + res.sendStatus(204) + clearCache: (req, res, next = (error) ->) -> ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) -> return next(error) if error? diff --git a/app/coffee/CompileManager.coffee b/app/coffee/CompileManager.coffee index bb93dbd..5d2bebb 100644 --- a/app/coffee/CompileManager.coffee +++ b/app/coffee/CompileManager.coffee @@ -28,7 +28,7 @@ module.exports = CompileManager = compileDir = getCompileDir(request.project_id, request.user_id) timer = new Metrics.Timer("write-to-disk") - logger.log project_id: request.project_id, user_id: request.user_id, "starting compile" + logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk" ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) -> if error? logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk" @@ -41,7 +41,16 @@ module.exports = CompileManager = DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback else callback() - + + # set up environment variables for chktex + env = {} + if request.check? + env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16' + if request.check is 'error' + env['CHKTEX_EXIT_ON_ERROR'] = 1 + if request.check is 'validate' + env['CHKTEX_VALIDATE'] = 1 + injectDraftModeIfRequired (error) -> return callback(error) if error? timer = new Metrics.Timer("run-compile") @@ -57,7 +66,24 @@ module.exports = CompileManager = compiler: request.compiler timeout: request.timeout image: request.imageName + environment: env }, (error, output, stats, timings) -> + # request was for validation only + if request.check is "validate" + result = if error?.code then "fail" else "pass" + error = new Error("validation") + error.validate = result + # request was for compile, and failed on validation + if request.check is "error" and error?.message is 'exited' + error = new Error("compilation") + error.validate = "fail" + # compile was killed by user, was a validation, or a compile which failed validation + if error?.terminated or error?.validate + OutputFileFinder.findOutputFiles request.resources, compileDir, (err, outputFiles) -> + return callback(err) if err? + callback(error, outputFiles) # return output files so user can check logs + return + # compile completed normally return callback(error) if error? Metrics.inc("compiles-succeeded") for metric_key, metric_value of stats or {} @@ -78,6 +104,10 @@ module.exports = CompileManager = OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> callback null, newOutputFiles + stopCompile: (project_id, user_id, callback = (error) ->) -> + compileName = getCompileName(project_id, user_id) + LatexRunner.killLatex compileName, callback + clearProject: (project_id, user_id, _callback = (error) ->) -> callback = (error) -> _callback(error) @@ -163,6 +193,8 @@ module.exports = CompileManager = _runSynctex: (args, callback = (error, stdout) ->) -> bin_path = Path.resolve(__dirname + "/../../bin/synctex") seconds = 1000 + if Settings.clsi?.synctexCommandWrapper? + [bin_path, args] = Settings.clsi?.synctexCommandWrapper bin_path, args child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) -> if error? logger.err err:error, args:args, "error running synctex" @@ -199,19 +231,20 @@ module.exports = CompileManager = wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) -> logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount" file_path = "$COMPILE_DIR/" + file_name - command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"] + command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"] directory = getCompileDir(project_id, user_id) timeout = 10 * 1000 compileName = getCompileName(project_id, user_id) - CommandRunner.run compileName, command, directory, image, timeout, (error) -> + CommandRunner.run compileName, command, directory, image, timeout, {}, (error) -> return callback(error) if error? - try - stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") - catch err - logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output" - return callback(err) - callback null, CompileManager._parseWordcountFromOutput(stdout) + fs.readFile directory + "/" + file_name + ".wc", "utf-8", (err, stdout) -> + if err? + logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output" + return callback(err) + results = CompileManager._parseWordcountFromOutput(stdout) + logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results" + callback null, results _parseWordcountFromOutput: (output) -> results = { @@ -223,6 +256,8 @@ module.exports = CompileManager = elements: 0 mathInline: 0 mathDisplay: 0 + errors: 0 + messages: "" } for line in output.split("\n") [data, info] = line.split(":") @@ -242,4 +277,8 @@ module.exports = CompileManager = results['mathInline'] = parseInt(info, 10) if data.indexOf("Number of math displayed") > -1 results['mathDisplay'] = parseInt(info, 10) + if data is "(errors" # errors reported as (errors:123) + results['errors'] = parseInt(info, 10) + if line.indexOf("!!! ") > -1 # errors logged as !!! message !!! + results['messages'] += line + "\n" return results diff --git a/app/coffee/LatexRunner.coffee b/app/coffee/LatexRunner.coffee index 65a3046..e743cf0 100644 --- a/app/coffee/LatexRunner.coffee +++ b/app/coffee/LatexRunner.coffee @@ -4,13 +4,15 @@ logger = require "logger-sharelatex" Metrics = require "./Metrics" CommandRunner = require(Settings.clsi?.commandRunner or "./CommandRunner") +ProcessTable = {} # table of currently running jobs (pids or docker container names) + module.exports = LatexRunner = runLatex: (project_id, options, callback = (error) ->) -> - {directory, mainFile, compiler, timeout, image} = options + {directory, mainFile, compiler, timeout, image, environment} = options compiler ||= "pdflatex" timeout ||= 60000 # milliseconds - logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, "starting compile" + logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, environment: environment, "starting compile" # We want to run latexmk on the tex file which we will automatically # generate from the Rtex/Rmd/md file. @@ -30,7 +32,10 @@ module.exports = LatexRunner = if Settings.clsi?.strace command = ["strace", "-o", "strace", "-ff"].concat(command) - CommandRunner.run project_id, command, directory, image, timeout, (error, output) -> + id = "#{project_id}" # record running project under this id + + ProcessTable[id] = CommandRunner.run project_id, command, directory, image, timeout, environment, (error, output) -> + delete ProcessTable[id] return callback(error) if error? runs = output?.stderr?.match(/^Run number \d+ of .*latex/mg)?.length or 0 failed = if output?.stdout?.match(/^Latexmk: Errors/m)? then 1 else 0 @@ -49,7 +54,17 @@ module.exports = LatexRunner = timings["sys-time"] = stderr?.match(/System time.*: (\d+.\d+)/m)?[1] or 0 callback error, output, stats, timings - _latexmkBaseCommand: ["/usr/bin/time", "-v", "latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR"] + killLatex: (project_id, callback = (error) ->) -> + id = "#{project_id}" + logger.log {id:id}, "killing running compile" + if not ProcessTable[id]? + return callback new Error("no such project to kill") + else + CommandRunner.kill ProcessTable[id], callback + + _latexmkBaseCommand: (Settings?.clsi?.latexmkCommandPrefix || []).concat( + ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR"] + ) _pdflatexCommand: (mainFile) -> LatexRunner._latexmkBaseCommand.concat [ diff --git a/app/coffee/OutputCacheManager.coffee b/app/coffee/OutputCacheManager.coffee index 7f11bc8..76692b3 100644 --- a/app/coffee/OutputCacheManager.coffee +++ b/app/coffee/OutputCacheManager.coffee @@ -45,9 +45,8 @@ module.exports = OutputCacheManager = cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR) # Put the files into a new cache subdirectory cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId) - - # let file expiry run in the background - OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId} + # Is it a per-user compile? check if compile directory is PROJECTID-USERID + perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/) # Archive logs in background if Settings.clsi?.archive_logs or Settings.clsi?.strace @@ -83,9 +82,15 @@ module.exports = OutputCacheManager = if err? # pass back the original files if we encountered *any* error callback(err, outputFiles) + # clean up the directory we just created + fse.remove cacheDir, (err) -> + if err? + logger.error err: err, dir: dir, "error removing cache dir after failure" else # pass back the list of new files in the cache callback(err, results) + # let file expiry run in the background, expire all previous files if per-user + OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 1 else null} archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) -> archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId) @@ -116,6 +121,8 @@ module.exports = OutputCacheManager = isExpired = (dir, index) -> return false if options?.keep == dir + # remove any directories over the requested (non-null) limit + return true if options?.limit? and index > options.limit # remove any directories over the hard limit return true if index > OutputCacheManager.CACHE_LIMIT # we can get the build time from the first part of the directory name DDDD-RRRR diff --git a/app/coffee/RequestParser.coffee b/app/coffee/RequestParser.coffee index bd081fd..5979c75 100644 --- a/app/coffee/RequestParser.coffee +++ b/app/coffee/RequestParser.coffee @@ -28,6 +28,9 @@ module.exports = RequestParser = compile.options.draft, default: false, type: "boolean" + response.check = @_parseAttribute "check", + compile.options.check, + type: "string" if response.timeout > RequestParser.MAX_TIMEOUT response.timeout = RequestParser.MAX_TIMEOUT diff --git a/config/settings.defaults.coffee b/config/settings.defaults.coffee index ae8e132..f1f7492 100644 --- a/config/settings.defaults.coffee +++ b/config/settings.defaults.coffee @@ -28,6 +28,9 @@ module.exports = # modem: # socketPath: false # user: "tex" + # latexmkCommandPrefix: [] + # # latexmkCommandPrefix: ["/usr/bin/time", "-v"] # on Linux + # # latexmkCommandPrefix: ["/usr/local/bin/gtime", "-v"] # on Mac OSX, installed with `brew install gnu-time` internal: clsi: @@ -41,5 +44,5 @@ module.exports = url: "http://localhost:3013" smokeTest: false - project_cache_length_ms: 60 * 60 * 24 - parallelFileDownloads:1 \ No newline at end of file + project_cache_length_ms: 1000 * 60 * 60 * 24 + parallelFileDownloads:1 diff --git a/test/unit/coffee/CompileManagerTests.coffee b/test/unit/coffee/CompileManagerTests.coffee index 611ed11..d2b6a10 100644 --- a/test/unit/coffee/CompileManagerTests.coffee +++ b/test/unit/coffee/CompileManagerTests.coffee @@ -47,6 +47,7 @@ describe "CompileManager", -> compiler: @compiler = "pdflatex" timeout: @timeout = 42000 imageName: @image = "example.com/image" + @env = {} @Settings.compileDir = "compiles" @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3) @@ -72,6 +73,7 @@ describe "CompileManager", -> compiler: @compiler timeout: @timeout image: @image + environment: @env }) .should.equal true @@ -200,8 +202,8 @@ describe "CompileManager", -> describe "wordcount", -> beforeEach -> - @CommandRunner.run = sinon.stub().callsArg(5) - @fs.readFileSync = sinon.stub().returns @stdout = "Encoding: ascii\nWords in text: 2" + @CommandRunner.run = sinon.stub().callsArg(6) + @fs.readFile = sinon.stub().callsArgWith(2, null, @stdout = "Encoding: ascii\nWords in text: 2") @callback = sinon.stub() @project_id = "project-id-123" @@ -215,10 +217,10 @@ describe "CompileManager", -> it "should run the texcount command", -> @directory = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @file_path = "$COMPILE_DIR/#{@file_name}" - @command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"] + @command =[ "texcount", "-nocol", "-inc", @file_path, "-out=" + @file_path + ".wc"] @CommandRunner.run - .calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout) + .calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout, {}) .should.equal true it "should call the callback with the parsed output", -> @@ -232,5 +234,7 @@ describe "CompileManager", -> elements: 0 mathInline: 0 mathDisplay: 0 + errors: 0 + messages: "" }) .should.equal true diff --git a/test/unit/coffee/LatexRunnerTests.coffee b/test/unit/coffee/LatexRunnerTests.coffee index ace3d18..c26fa64 100644 --- a/test/unit/coffee/LatexRunnerTests.coffee +++ b/test/unit/coffee/LatexRunnerTests.coffee @@ -22,10 +22,11 @@ describe "LatexRunner", -> @image = "example.com/image" @callback = sinon.stub() @project_id = "project-id-123" + @env = {'foo': '123'} describe "runLatex", -> beforeEach -> - @CommandRunner.run = sinon.stub().callsArg(5) + @CommandRunner.run = sinon.stub().callsArg(6) describe "normally", -> beforeEach -> @@ -35,11 +36,12 @@ describe "LatexRunner", -> compiler: @compiler timeout: @timeout = 42000 image: @image + environment: @env @callback it "should run the latex command", -> @CommandRunner.run - .calledWith(@project_id, sinon.match.any, @directory, @image, @timeout) + .calledWith(@project_id, sinon.match.any, @directory, @image, @timeout, @env) .should.equal true describe "with an .Rtex main file", ->