diff --git a/app/coffee/LatexRunner.coffee b/app/coffee/LatexRunner.coffee index 1cd851b..938d4a5 100644 --- a/app/coffee/LatexRunner.coffee +++ b/app/coffee/LatexRunner.coffee @@ -26,6 +26,9 @@ module.exports = LatexRunner = command = LatexRunner._lualatexCommand mainFile else return callback new Error("unknown compiler: #{compiler}") + + if Settings.clsi?.strace + command = ["strace", "-o", "strace-#{Date.now()}", "-ff"].concat(command) CommandRunner.run project_id, command, directory, image, timeout, (error, output) -> return callback(error) if error? diff --git a/app/coffee/OutputCacheManager.coffee b/app/coffee/OutputCacheManager.coffee index 5aca4a5..1517ccc 100644 --- a/app/coffee/OutputCacheManager.coffee +++ b/app/coffee/OutputCacheManager.coffee @@ -4,11 +4,13 @@ fse = require "fs-extra" Path = require "path" logger = require "logger-sharelatex" _ = require "underscore" +Settings = require "settings-sharelatex" OutputFileOptimiser = require "./OutputFileOptimiser" module.exports = OutputCacheManager = CACHE_SUBDIR: '.cache/clsi' + ARCHIVE_SUBDIR: '.archive/clsi' BUILD_REGEX: /^[0-9a-f]+$/ # build id is Date.now() converted to hex CACHE_LIMIT: 2 # maximum number of cache directories CACHE_AGE: 60*60*1000 # up to one hour old @@ -28,33 +30,15 @@ module.exports = OutputCacheManager = # Put the files into a new cache subdirectory buildId = Date.now().toString(16) cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId) + # let file expiry run in the background OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId} - checkFile = (src, callback) -> - # check if we have a valid file to copy into the cache - fs.stat src, (err, stats) -> + # Archive logs in background + if Settings.clsi?.archive_logs + OutputCacheManager.archiveLogs outputFiles, compileDir, (err) -> if err? - # some problem reading the file - logger.error err: err, file: src, "stat error for file in cache" - callback(err) - else if not stats.isFile() - # other filetype - reject it - logger.error err: err, src: src, stat: stats, "nonfile output - refusing to copy to cache" - callback(new Error("output file is not a file"), file) - else - # it's a plain file, ok to copy - callback(null) - - copyFile = (src, dst, callback) -> - # copy output file into the cache - fse.copy src, dst, (err) -> - if err? - logger.error err: err, src: src, dst: dst, "copy error for file in cache" - callback(err) - else - # call the optimiser for the file too - OutputFileOptimiser.optimiseFile src, dst, callback + logger.warn err:err, "erroring archiving log files" # make the new cache directory fse.ensureDir cacheDir, (err) -> @@ -63,21 +47,47 @@ module.exports = OutputCacheManager = callback(err, outputFiles) else # copy all the output files into the new cache directory + results = [] async.mapSeries outputFiles, (file, cb) -> newFile = _.clone(file) [src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)] - checkFile src, (err) -> - copyFile src, dst, (err) -> - if not err? + OutputCacheManager._checkFileIsSafe src, (err, isSafe) -> + return cb(err) if err? + if !isSafe + return cb() + OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) -> + return cb(err) if err? + if !shouldCopy + return cb() + OutputCacheManager._copyFile src, dst, (err) -> + return cb(err) if err? newFile.build = buildId # attach a build id if we cached the file - cb(err, newFile) - , (err, results) -> + results.push newFile + cb() + , (err) -> if err? # pass back the original files if we encountered *any* error callback(err, outputFiles) else # pass back the list of new files in the cache callback(err, results) + + archiveLogs: (outputFiles, compileDir, callback = (error) ->) -> + buildId = Date.now().toString(16) + archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId) + logger.log {dir: archiveDir}, "archiving log files for project" + fse.ensureDir archiveDir, (err) -> + return callback(err) if err? + async.mapSeries outputFiles, (file, cb) -> + [src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)] + OutputCacheManager._checkFileIsSafe src, (err, isSafe) -> + return cb(err) if err? + return cb() if !isSafe + OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) -> + return cb(err) if err? + return cb() if !shouldArchive + OutputCacheManager._copyFile src, dst, cb + , callback expireOutputFiles: (cacheRoot, options, callback = (error) ->) -> # look in compileDir for build dirs and delete if > N or age of mod time > T @@ -111,3 +121,38 @@ module.exports = OutputCacheManager = async.eachSeries toRemove, (dir, cb) -> removeDir dir, cb , callback + + _checkFileIsSafe: (src, callback = (error, isSafe) ->) -> + # check if we have a valid file to copy into the cache + fs.stat src, (err, stats) -> + if err? + # some problem reading the file + logger.error err: err, file: src, "stat error for file in cache" + callback(err, false) + else if not stats.isFile() + # other filetype - reject it + logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache" + callback(null, false) + else + # it's a plain file, ok to copy + callback(null, true) + + _copyFile: (src, dst, callback) -> + # copy output file into the cache + fse.copy src, dst, (err) -> + if err? + logger.error err: err, src: src, dst: dst, "copy error for file in cache" + callback(err) + else + # call the optimiser for the file too + OutputFileOptimiser.optimiseFile src, dst, callback + + _checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) -> + return callback(null, !Path.basename(src).match(/^strace/)) + + _checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) -> + if Path.basename(src).match(/^strace/) + return callback(null, true) + if Path.basename(src).match(/^output\.(?!pdf)/) + return callback(null, true) + return callback(null, false) \ No newline at end of file