Merge branch 'master' into bg-support-other-runtimes
port coffee changes to js
This commit is contained in:
Henry Oswald
20
app/js/CommandRunner.js
Normal file
20
app/js/CommandRunner.js
Normal file
@@ -0,0 +1,20 @@
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let commandRunnerPath
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) {
|
||||
commandRunnerPath = './DockerRunner'
|
||||
} else {
|
||||
commandRunnerPath = './LocalCommandRunner'
|
||||
}
|
||||
logger.info({ commandRunnerPath }, 'selecting command runner for clsi')
|
||||
const CommandRunner = require(commandRunnerPath)
|
||||
|
||||
module.exports = CommandRunner
|
||||
238
app/js/CompileController.js
Normal file
238
app/js/CompileController.js
Normal file
@@ -0,0 +1,238 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CompileController
|
||||
const RequestParser = require('./RequestParser')
|
||||
const CompileManager = require('./CompileManager')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const Metrics = require('./Metrics')
|
||||
const ProjectPersistenceManager = require('./ProjectPersistenceManager')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Errors = require('./Errors')
|
||||
|
||||
module.exports = CompileController = {
|
||||
compile(req, res, next) {
|
||||
if (next == null) {
|
||||
next = function(error) {}
|
||||
}
|
||||
const timer = new Metrics.Timer('compile-request')
|
||||
return RequestParser.parse(req.body, function(error, request) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
request.project_id = req.params.project_id
|
||||
if (req.params.user_id != null) {
|
||||
request.user_id = req.params.user_id
|
||||
}
|
||||
return ProjectPersistenceManager.markProjectAsJustAccessed(
|
||||
request.project_id,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return CompileManager.doCompileWithLock(request, function(
|
||||
error,
|
||||
outputFiles
|
||||
) {
|
||||
let code, status
|
||||
if (outputFiles == null) {
|
||||
outputFiles = []
|
||||
}
|
||||
if (error instanceof Errors.AlreadyCompilingError) {
|
||||
code = 423 // Http 423 Locked
|
||||
status = 'compile-in-progress'
|
||||
} else if (error instanceof Errors.FilesOutOfSyncError) {
|
||||
code = 409 // Http 409 Conflict
|
||||
status = 'retry'
|
||||
} else if (error != null ? error.terminated : undefined) {
|
||||
status = 'terminated'
|
||||
} else if (error != null ? error.validate : undefined) {
|
||||
status = `validation-${error.validate}`
|
||||
} else if (error != null ? error.timedout : undefined) {
|
||||
status = 'timedout'
|
||||
logger.log(
|
||||
{ err: error, project_id: request.project_id },
|
||||
'timeout running compile'
|
||||
)
|
||||
} else if (error != null) {
|
||||
status = 'error'
|
||||
code = 500
|
||||
logger.warn(
|
||||
{ err: error, project_id: request.project_id },
|
||||
'error running compile'
|
||||
)
|
||||
} else {
|
||||
let file
|
||||
status = 'failure'
|
||||
for (file of Array.from(outputFiles)) {
|
||||
if (
|
||||
file.path != null
|
||||
? file.path.match(/output\.pdf$/)
|
||||
: undefined
|
||||
) {
|
||||
status = 'success'
|
||||
}
|
||||
}
|
||||
|
||||
if (status === 'failure') {
|
||||
logger.warn(
|
||||
{ project_id: request.project_id, outputFiles },
|
||||
'project failed to compile successfully, no output.pdf generated'
|
||||
)
|
||||
}
|
||||
|
||||
// log an error if any core files are found
|
||||
for (file of Array.from(outputFiles)) {
|
||||
if (file.path === 'core') {
|
||||
logger.error(
|
||||
{ project_id: request.project_id, req, outputFiles },
|
||||
'core file found in output'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (error != null) {
|
||||
outputFiles = error.outputFiles || []
|
||||
}
|
||||
|
||||
timer.done()
|
||||
return res.status(code || 200).send({
|
||||
compile: {
|
||||
status,
|
||||
error: (error != null ? error.message : undefined) || error,
|
||||
outputFiles: outputFiles.map(file => ({
|
||||
url:
|
||||
`${Settings.apis.clsi.url}/project/${request.project_id}` +
|
||||
(request.user_id != null
|
||||
? `/user/${request.user_id}`
|
||||
: '') +
|
||||
(file.build != null ? `/build/${file.build}` : '') +
|
||||
`/output/${file.path}`,
|
||||
path: file.path,
|
||||
type: file.type,
|
||||
build: file.build
|
||||
}))
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
stopCompile(req, res, next) {
|
||||
const { project_id, user_id } = req.params
|
||||
return CompileManager.stopCompile(project_id, user_id, function(error) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.sendStatus(204)
|
||||
})
|
||||
},
|
||||
|
||||
clearCache(req, res, next) {
|
||||
if (next == null) {
|
||||
next = function(error) {}
|
||||
}
|
||||
return ProjectPersistenceManager.clearProject(
|
||||
req.params.project_id,
|
||||
req.params.user_id,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.sendStatus(204)
|
||||
}
|
||||
)
|
||||
}, // No content
|
||||
|
||||
syncFromCode(req, res, next) {
|
||||
if (next == null) {
|
||||
next = function(error) {}
|
||||
}
|
||||
const { file } = req.query
|
||||
const line = parseInt(req.query.line, 10)
|
||||
const column = parseInt(req.query.column, 10)
|
||||
const { project_id } = req.params
|
||||
const { user_id } = req.params
|
||||
return CompileManager.syncFromCode(
|
||||
project_id,
|
||||
user_id,
|
||||
file,
|
||||
line,
|
||||
column,
|
||||
function(error, pdfPositions) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.json({
|
||||
pdf: pdfPositions
|
||||
})
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
syncFromPdf(req, res, next) {
|
||||
if (next == null) {
|
||||
next = function(error) {}
|
||||
}
|
||||
const page = parseInt(req.query.page, 10)
|
||||
const h = parseFloat(req.query.h)
|
||||
const v = parseFloat(req.query.v)
|
||||
const { project_id } = req.params
|
||||
const { user_id } = req.params
|
||||
return CompileManager.syncFromPdf(project_id, user_id, page, h, v, function(
|
||||
error,
|
||||
codePositions
|
||||
) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.json({
|
||||
code: codePositions
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
wordcount(req, res, next) {
|
||||
if (next == null) {
|
||||
next = function(error) {}
|
||||
}
|
||||
const file = req.query.file || 'main.tex'
|
||||
const { project_id } = req.params
|
||||
const { user_id } = req.params
|
||||
const { image } = req.query
|
||||
logger.log({ image, file, project_id }, 'word count request')
|
||||
|
||||
return CompileManager.wordcount(project_id, user_id, file, image, function(
|
||||
error,
|
||||
result
|
||||
) {
|
||||
if (error != null) {
|
||||
return next(error)
|
||||
}
|
||||
return res.json({
|
||||
texcount: result
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
status(req, res, next) {
|
||||
if (next == null) {
|
||||
next = function(error) {}
|
||||
}
|
||||
return res.send('OK')
|
||||
}
|
||||
}
|
||||
705
app/js/CompileManager.js
Normal file
705
app/js/CompileManager.js
Normal file
@@ -0,0 +1,705 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-undef,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CompileManager
|
||||
const ResourceWriter = require('./ResourceWriter')
|
||||
const LatexRunner = require('./LatexRunner')
|
||||
const OutputFileFinder = require('./OutputFileFinder')
|
||||
const OutputCacheManager = require('./OutputCacheManager')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const Path = require('path')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Metrics = require('./Metrics')
|
||||
const child_process = require('child_process')
|
||||
const DraftModeManager = require('./DraftModeManager')
|
||||
const TikzManager = require('./TikzManager')
|
||||
const LockManager = require('./LockManager')
|
||||
const fs = require('fs')
|
||||
const fse = require('fs-extra')
|
||||
const os = require('os')
|
||||
const async = require('async')
|
||||
const Errors = require('./Errors')
|
||||
const CommandRunner = require('./CommandRunner')
|
||||
|
||||
const getCompileName = function(project_id, user_id) {
|
||||
if (user_id != null) {
|
||||
return `${project_id}-${user_id}`
|
||||
} else {
|
||||
return project_id
|
||||
}
|
||||
}
|
||||
|
||||
const getCompileDir = (project_id, user_id) =>
|
||||
Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
|
||||
|
||||
module.exports = CompileManager = {
|
||||
doCompileWithLock(request, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, outputFiles) {}
|
||||
}
|
||||
const compileDir = getCompileDir(request.project_id, request.user_id)
|
||||
const lockFile = Path.join(compileDir, '.project-lock')
|
||||
// use a .project-lock file in the compile directory to prevent
|
||||
// simultaneous compiles
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return LockManager.runWithLock(
|
||||
lockFile,
|
||||
releaseLock => CompileManager.doCompile(request, releaseLock),
|
||||
callback
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
doCompile(request, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, outputFiles) {}
|
||||
}
|
||||
const compileDir = getCompileDir(request.project_id, request.user_id)
|
||||
let timer = new Metrics.Timer('write-to-disk')
|
||||
logger.log(
|
||||
{ project_id: request.project_id, user_id: request.user_id },
|
||||
'syncing resources to disk'
|
||||
)
|
||||
return ResourceWriter.syncResourcesToDisk(request, compileDir, function(
|
||||
error,
|
||||
resourceList
|
||||
) {
|
||||
// NOTE: resourceList is insecure, it should only be used to exclude files from the output list
|
||||
if (error != null && error instanceof Errors.FilesOutOfSyncError) {
|
||||
logger.warn(
|
||||
{ project_id: request.project_id, user_id: request.user_id },
|
||||
'files out of sync, please retry'
|
||||
)
|
||||
return callback(error)
|
||||
} else if (error != null) {
|
||||
logger.err(
|
||||
{
|
||||
err: error,
|
||||
project_id: request.project_id,
|
||||
user_id: request.user_id
|
||||
},
|
||||
'error writing resources to disk'
|
||||
)
|
||||
return callback(error)
|
||||
}
|
||||
logger.log(
|
||||
{
|
||||
project_id: request.project_id,
|
||||
user_id: request.user_id,
|
||||
time_taken: Date.now() - timer.start
|
||||
},
|
||||
'written files to disk'
|
||||
)
|
||||
timer.done()
|
||||
|
||||
const injectDraftModeIfRequired = function(callback) {
|
||||
if (request.draft) {
|
||||
return DraftModeManager.injectDraftMode(
|
||||
Path.join(compileDir, request.rootResourcePath),
|
||||
callback
|
||||
)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
}
|
||||
|
||||
const createTikzFileIfRequired = callback =>
|
||||
TikzManager.checkMainFile(
|
||||
compileDir,
|
||||
request.rootResourcePath,
|
||||
resourceList,
|
||||
function(error, needsMainFile) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
if (needsMainFile) {
|
||||
return TikzManager.injectOutputFile(
|
||||
compileDir,
|
||||
request.rootResourcePath,
|
||||
callback
|
||||
)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
}
|
||||
)
|
||||
// set up environment variables for chktex
|
||||
const env = {}
|
||||
// only run chktex on LaTeX files (not knitr .Rtex files or any others)
|
||||
const isLaTeXFile =
|
||||
request.rootResourcePath != null
|
||||
? request.rootResourcePath.match(/\.tex$/i)
|
||||
: undefined
|
||||
if (request.check != null && isLaTeXFile) {
|
||||
env.CHKTEX_OPTIONS = '-nall -e9 -e10 -w15 -w16'
|
||||
env.CHKTEX_ULIMIT_OPTIONS = '-t 5 -v 64000'
|
||||
if (request.check === 'error') {
|
||||
env.CHKTEX_EXIT_ON_ERROR = 1
|
||||
}
|
||||
if (request.check === 'validate') {
|
||||
env.CHKTEX_VALIDATE = 1
|
||||
}
|
||||
}
|
||||
|
||||
// apply a series of file modifications/creations for draft mode and tikz
|
||||
return async.series(
|
||||
[injectDraftModeIfRequired, createTikzFileIfRequired],
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
timer = new Metrics.Timer('run-compile')
|
||||
// find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
|
||||
let tag =
|
||||
__guard__(
|
||||
__guard__(
|
||||
request.imageName != null
|
||||
? request.imageName.match(/:(.*)/)
|
||||
: undefined,
|
||||
x1 => x1[1]
|
||||
),
|
||||
x => x.replace(/\./g, '-')
|
||||
) || 'default'
|
||||
if (!request.project_id.match(/^[0-9a-f]{24}$/)) {
|
||||
tag = 'other'
|
||||
} // exclude smoke test
|
||||
Metrics.inc('compiles')
|
||||
Metrics.inc(`compiles-with-image.${tag}`)
|
||||
const compileName = getCompileName(
|
||||
request.project_id,
|
||||
request.user_id
|
||||
)
|
||||
return LatexRunner.runLatex(
|
||||
compileName,
|
||||
{
|
||||
directory: compileDir,
|
||||
mainFile: request.rootResourcePath,
|
||||
compiler: request.compiler,
|
||||
timeout: request.timeout,
|
||||
image: request.imageName,
|
||||
flags: request.flags,
|
||||
environment: env
|
||||
},
|
||||
function(error, output, stats, timings) {
|
||||
// request was for validation only
|
||||
let metric_key, metric_value
|
||||
if (request.check === 'validate') {
|
||||
const result = (error != null
|
||||
? error.code
|
||||
: undefined)
|
||||
? 'fail'
|
||||
: 'pass'
|
||||
error = new Error('validation')
|
||||
error.validate = result
|
||||
}
|
||||
// request was for compile, and failed on validation
|
||||
if (
|
||||
request.check === 'error' &&
|
||||
(error != null ? error.message : undefined) === 'exited'
|
||||
) {
|
||||
error = new Error('compilation')
|
||||
error.validate = 'fail'
|
||||
}
|
||||
// compile was killed by user, was a validation, or a compile which failed validation
|
||||
if (
|
||||
(error != null ? error.terminated : undefined) ||
|
||||
(error != null ? error.validate : undefined) ||
|
||||
(error != null ? error.timedout : undefined)
|
||||
) {
|
||||
OutputFileFinder.findOutputFiles(
|
||||
resourceList,
|
||||
compileDir,
|
||||
function(err, outputFiles) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
error.outputFiles = outputFiles // return output files so user can check logs
|
||||
return callback(error)
|
||||
}
|
||||
)
|
||||
return
|
||||
}
|
||||
// compile completed normally
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
Metrics.inc('compiles-succeeded')
|
||||
const object = stats || {}
|
||||
for (metric_key in object) {
|
||||
metric_value = object[metric_key]
|
||||
Metrics.count(metric_key, metric_value)
|
||||
}
|
||||
const object1 = timings || {}
|
||||
for (metric_key in object1) {
|
||||
metric_value = object1[metric_key]
|
||||
Metrics.timing(metric_key, metric_value)
|
||||
}
|
||||
const loadavg =
|
||||
typeof os.loadavg === 'function' ? os.loadavg() : undefined
|
||||
if (loadavg != null) {
|
||||
Metrics.gauge('load-avg', loadavg[0])
|
||||
}
|
||||
const ts = timer.done()
|
||||
logger.log(
|
||||
{
|
||||
project_id: request.project_id,
|
||||
user_id: request.user_id,
|
||||
time_taken: ts,
|
||||
stats,
|
||||
timings,
|
||||
loadavg
|
||||
},
|
||||
'done compile'
|
||||
)
|
||||
if ((stats != null ? stats['latex-runs'] : undefined) > 0) {
|
||||
Metrics.timing('run-compile-per-pass', ts / stats['latex-runs'])
|
||||
}
|
||||
if (
|
||||
(stats != null ? stats['latex-runs'] : undefined) > 0 &&
|
||||
(timings != null ? timings['cpu-time'] : undefined) > 0
|
||||
) {
|
||||
Metrics.timing(
|
||||
'run-compile-cpu-time-per-pass',
|
||||
timings['cpu-time'] / stats['latex-runs']
|
||||
)
|
||||
}
|
||||
|
||||
return OutputFileFinder.findOutputFiles(
|
||||
resourceList,
|
||||
compileDir,
|
||||
function(error, outputFiles) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return OutputCacheManager.saveOutputFiles(
|
||||
outputFiles,
|
||||
compileDir,
|
||||
(error, newOutputFiles) => callback(null, newOutputFiles)
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
stopCompile(project_id, user_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const compileName = getCompileName(project_id, user_id)
|
||||
return LatexRunner.killLatex(compileName, callback)
|
||||
},
|
||||
|
||||
clearProject(project_id, user_id, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error) {}
|
||||
}
|
||||
const callback = function(error) {
|
||||
_callback(error)
|
||||
return (_callback = function() {})
|
||||
}
|
||||
|
||||
const compileDir = getCompileDir(project_id, user_id)
|
||||
|
||||
return CompileManager._checkDirectory(compileDir, function(err, exists) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
if (!exists) {
|
||||
return callback()
|
||||
} // skip removal if no directory present
|
||||
|
||||
const proc = child_process.spawn('rm', ['-r', compileDir])
|
||||
|
||||
proc.on('error', callback)
|
||||
|
||||
let stderr = ''
|
||||
proc.stderr.on('data', chunk => (stderr += chunk.toString()))
|
||||
|
||||
return proc.on('close', function(code) {
|
||||
if (code === 0) {
|
||||
return callback(null)
|
||||
} else {
|
||||
return callback(new Error(`rm -r ${compileDir} failed: ${stderr}`))
|
||||
}
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
_findAllDirs(callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, allDirs) {}
|
||||
}
|
||||
const root = Settings.path.compilesDir
|
||||
return fs.readdir(root, function(err, files) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
const allDirs = Array.from(files).map(file => Path.join(root, file))
|
||||
return callback(null, allDirs)
|
||||
})
|
||||
},
|
||||
|
||||
clearExpiredProjects(max_cache_age_ms, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const now = Date.now()
|
||||
// action for each directory
|
||||
const expireIfNeeded = (checkDir, cb) =>
|
||||
fs.stat(checkDir, function(err, stats) {
|
||||
if (err != null) {
|
||||
return cb()
|
||||
} // ignore errors checking directory
|
||||
const age = now - stats.mtime
|
||||
const hasExpired = age > max_cache_age_ms
|
||||
if (hasExpired) {
|
||||
return fse.remove(checkDir, cb)
|
||||
} else {
|
||||
return cb()
|
||||
}
|
||||
})
|
||||
// iterate over all project directories
|
||||
return CompileManager._findAllDirs(function(error, allDirs) {
|
||||
if (error != null) {
|
||||
return callback()
|
||||
}
|
||||
return async.eachSeries(allDirs, expireIfNeeded, callback)
|
||||
})
|
||||
},
|
||||
|
||||
_checkDirectory(compileDir, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, exists) {}
|
||||
}
|
||||
return fs.lstat(compileDir, function(err, stats) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
return callback(null, false) // directory does not exist
|
||||
} else if (err != null) {
|
||||
logger.err(
|
||||
{ dir: compileDir, err },
|
||||
'error on stat of project directory for removal'
|
||||
)
|
||||
return callback(err)
|
||||
} else if (!(stats != null ? stats.isDirectory() : undefined)) {
|
||||
logger.err(
|
||||
{ dir: compileDir, stats },
|
||||
'bad project directory for removal'
|
||||
)
|
||||
return callback(new Error('project directory is not directory'))
|
||||
} else {
|
||||
return callback(null, true)
|
||||
}
|
||||
})
|
||||
}, // directory exists
|
||||
|
||||
syncFromCode(project_id, user_id, file_name, line, column, callback) {
|
||||
// If LaTeX was run in a virtual environment, the file path that synctex expects
|
||||
// might not match the file path on the host. The .synctex.gz file however, will be accessed
|
||||
// wherever it is on the host.
|
||||
if (callback == null) {
|
||||
callback = function(error, pdfPositions) {}
|
||||
}
|
||||
const compileName = getCompileName(project_id, user_id)
|
||||
const base_dir = Settings.path.synctexBaseDir(compileName)
|
||||
const file_path = base_dir + '/' + file_name
|
||||
const compileDir = getCompileDir(project_id, user_id)
|
||||
const synctex_path = `${base_dir}/output.pdf`
|
||||
const command = ['code', synctex_path, file_path, line, column]
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{ error, project_id, user_id, file_name },
|
||||
'error ensuring dir for sync from code'
|
||||
)
|
||||
return callback(error)
|
||||
}
|
||||
return CompileManager._runSynctex(project_id, user_id, command, function(
|
||||
error,
|
||||
stdout
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
logger.log(
|
||||
{ project_id, user_id, file_name, line, column, command, stdout },
|
||||
'synctex code output'
|
||||
)
|
||||
return callback(
|
||||
null,
|
||||
CompileManager._parseSynctexFromCodeOutput(stdout)
|
||||
)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
syncFromPdf(project_id, user_id, page, h, v, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, filePositions) {}
|
||||
}
|
||||
const compileName = getCompileName(project_id, user_id)
|
||||
const compileDir = getCompileDir(project_id, user_id)
|
||||
const base_dir = Settings.path.synctexBaseDir(compileName)
|
||||
const synctex_path = `${base_dir}/output.pdf`
|
||||
const command = ['pdf', synctex_path, page, h, v]
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{ error, project_id, user_id, file_name },
|
||||
'error ensuring dir for sync to code'
|
||||
)
|
||||
return callback(error)
|
||||
}
|
||||
return CompileManager._runSynctex(project_id, user_id, command, function(
|
||||
error,
|
||||
stdout
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
logger.log(
|
||||
{ project_id, user_id, page, h, v, stdout },
|
||||
'synctex pdf output'
|
||||
)
|
||||
return callback(
|
||||
null,
|
||||
CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
|
||||
)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
_checkFileExists(path, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const synctexDir = Path.dirname(path)
|
||||
const synctexFile = Path.join(synctexDir, 'output.synctex.gz')
|
||||
return fs.stat(synctexDir, function(error, stats) {
|
||||
if ((error != null ? error.code : undefined) === 'ENOENT') {
|
||||
return callback(
|
||||
new Errors.NotFoundError('called synctex with no output directory')
|
||||
)
|
||||
}
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return fs.stat(synctexFile, function(error, stats) {
|
||||
if ((error != null ? error.code : undefined) === 'ENOENT') {
|
||||
return callback(
|
||||
new Errors.NotFoundError('called synctex with no output file')
|
||||
)
|
||||
}
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
if (!(stats != null ? stats.isFile() : undefined)) {
|
||||
return callback(new Error('not a file'))
|
||||
}
|
||||
return callback()
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
_runSynctex(project_id, user_id, command, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, stdout) {}
|
||||
}
|
||||
const seconds = 1000
|
||||
|
||||
command.unshift('/opt/synctex')
|
||||
|
||||
const directory = getCompileDir(project_id, user_id)
|
||||
const timeout = 60 * 1000 // increased to allow for large projects
|
||||
const compileName = getCompileName(project_id, user_id)
|
||||
return CommandRunner.run(
|
||||
compileName,
|
||||
command,
|
||||
directory,
|
||||
Settings.clsi != null ? Settings.clsi.docker.image : undefined,
|
||||
timeout,
|
||||
{},
|
||||
function(error, output) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{ err: error, command, project_id, user_id },
|
||||
'error running synctex'
|
||||
)
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null, output.stdout)
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
_parseSynctexFromCodeOutput(output) {
|
||||
const results = []
|
||||
for (const line of Array.from(output.split('\n'))) {
|
||||
const [node, page, h, v, width, height] = Array.from(line.split('\t'))
|
||||
if (node === 'NODE') {
|
||||
results.push({
|
||||
page: parseInt(page, 10),
|
||||
h: parseFloat(h),
|
||||
v: parseFloat(v),
|
||||
height: parseFloat(height),
|
||||
width: parseFloat(width)
|
||||
})
|
||||
}
|
||||
}
|
||||
return results
|
||||
},
|
||||
|
||||
_parseSynctexFromPdfOutput(output, base_dir) {
|
||||
const results = []
|
||||
for (let line of Array.from(output.split('\n'))) {
|
||||
let column, file_path, node
|
||||
;[node, file_path, line, column] = Array.from(line.split('\t'))
|
||||
if (node === 'NODE') {
|
||||
const file = file_path.slice(base_dir.length + 1)
|
||||
results.push({
|
||||
file,
|
||||
line: parseInt(line, 10),
|
||||
column: parseInt(column, 10)
|
||||
})
|
||||
}
|
||||
}
|
||||
return results
|
||||
},
|
||||
|
||||
wordcount(project_id, user_id, file_name, image, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, pdfPositions) {}
|
||||
}
|
||||
logger.log({ project_id, user_id, file_name, image }, 'running wordcount')
|
||||
const file_path = `$COMPILE_DIR/${file_name}`
|
||||
const command = [
|
||||
'texcount',
|
||||
'-nocol',
|
||||
'-inc',
|
||||
file_path,
|
||||
`-out=${file_path}.wc`
|
||||
]
|
||||
const compileDir = getCompileDir(project_id, user_id)
|
||||
const timeout = 60 * 1000
|
||||
const compileName = getCompileName(project_id, user_id)
|
||||
return fse.ensureDir(compileDir, function(error) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{ error, project_id, user_id, file_name },
|
||||
'error ensuring dir for sync from code'
|
||||
)
|
||||
return callback(error)
|
||||
}
|
||||
return CommandRunner.run(
|
||||
compileName,
|
||||
command,
|
||||
compileDir,
|
||||
image,
|
||||
timeout,
|
||||
{},
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return fs.readFile(
|
||||
compileDir + '/' + file_name + '.wc',
|
||||
'utf-8',
|
||||
function(err, stdout) {
|
||||
if (err != null) {
|
||||
// call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
|
||||
logger.err(
|
||||
{ node_err: err, command, compileDir, project_id, user_id },
|
||||
'error reading word count output'
|
||||
)
|
||||
return callback(err)
|
||||
}
|
||||
const results = CompileManager._parseWordcountFromOutput(stdout)
|
||||
logger.log(
|
||||
{ project_id, user_id, wordcount: results },
|
||||
'word count results'
|
||||
)
|
||||
return callback(null, results)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
_parseWordcountFromOutput(output) {
|
||||
const results = {
|
||||
encode: '',
|
||||
textWords: 0,
|
||||
headWords: 0,
|
||||
outside: 0,
|
||||
headers: 0,
|
||||
elements: 0,
|
||||
mathInline: 0,
|
||||
mathDisplay: 0,
|
||||
errors: 0,
|
||||
messages: ''
|
||||
}
|
||||
for (const line of Array.from(output.split('\n'))) {
|
||||
const [data, info] = Array.from(line.split(':'))
|
||||
if (data.indexOf('Encoding') > -1) {
|
||||
results.encode = info.trim()
|
||||
}
|
||||
if (data.indexOf('in text') > -1) {
|
||||
results.textWords = parseInt(info, 10)
|
||||
}
|
||||
if (data.indexOf('in head') > -1) {
|
||||
results.headWords = parseInt(info, 10)
|
||||
}
|
||||
if (data.indexOf('outside') > -1) {
|
||||
results.outside = parseInt(info, 10)
|
||||
}
|
||||
if (data.indexOf('of head') > -1) {
|
||||
results.headers = parseInt(info, 10)
|
||||
}
|
||||
if (data.indexOf('Number of floats/tables/figures') > -1) {
|
||||
results.elements = parseInt(info, 10)
|
||||
}
|
||||
if (data.indexOf('Number of math inlines') > -1) {
|
||||
results.mathInline = parseInt(info, 10)
|
||||
}
|
||||
if (data.indexOf('Number of math displayed') > -1) {
|
||||
results.mathDisplay = parseInt(info, 10)
|
||||
}
|
||||
if (data === '(errors') {
|
||||
// errors reported as (errors:123)
|
||||
results.errors = parseInt(info, 10)
|
||||
}
|
||||
if (line.indexOf('!!! ') > -1) {
|
||||
// errors logged as !!! message !!!
|
||||
results.messages += line + '\n'
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
38
app/js/ContentTypeMapper.js
Normal file
38
app/js/ContentTypeMapper.js
Normal file
@@ -0,0 +1,38 @@
|
||||
/* eslint-disable
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
let ContentTypeMapper
|
||||
const Path = require('path')
|
||||
|
||||
// here we coerce html, css and js to text/plain,
|
||||
// otherwise choose correct mime type based on file extension,
|
||||
// falling back to octet-stream
|
||||
module.exports = ContentTypeMapper = {
|
||||
map(path) {
|
||||
switch (Path.extname(path)) {
|
||||
case '.txt':
|
||||
case '.html':
|
||||
case '.js':
|
||||
case '.css':
|
||||
case '.svg':
|
||||
return 'text/plain'
|
||||
case '.csv':
|
||||
return 'text/csv'
|
||||
case '.pdf':
|
||||
return 'application/pdf'
|
||||
case '.png':
|
||||
return 'image/png'
|
||||
case '.jpg':
|
||||
case '.jpeg':
|
||||
return 'image/jpeg'
|
||||
case '.tiff':
|
||||
return 'image/tiff'
|
||||
case '.gif':
|
||||
return 'image/gif'
|
||||
default:
|
||||
return 'application/octet-stream'
|
||||
}
|
||||
}
|
||||
}
|
||||
18
app/js/DbQueue.js
Normal file
18
app/js/DbQueue.js
Normal file
@@ -0,0 +1,18 @@
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const async = require('async')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
const queue = async.queue(
|
||||
(task, cb) => task(cb),
|
||||
Settings.parallelSqlQueryLimit
|
||||
)
|
||||
|
||||
queue.drain = () => logger.debug('all items have been processed')
|
||||
|
||||
module.exports = { queue }
|
||||
113
app/js/DockerLockManager.js
Normal file
113
app/js/DockerLockManager.js
Normal file
@@ -0,0 +1,113 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LockManager
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
const LockState = {} // locks for docker container operations, by container name
|
||||
|
||||
module.exports = LockManager = {
|
||||
MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock
|
||||
MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock
|
||||
LOCK_TEST_INTERVAL: 1000, // retry time
|
||||
|
||||
tryLock(key, callback) {
|
||||
let lockValue
|
||||
if (callback == null) {
|
||||
callback = function(err, gotLock) {}
|
||||
}
|
||||
const existingLock = LockState[key]
|
||||
if (existingLock != null) {
|
||||
// the lock is already taken, check how old it is
|
||||
const lockAge = Date.now() - existingLock.created
|
||||
if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) {
|
||||
return callback(null, false) // we didn't get the lock, bail out
|
||||
} else {
|
||||
logger.error(
|
||||
{ key, lock: existingLock, age: lockAge },
|
||||
'taking old lock by force'
|
||||
)
|
||||
}
|
||||
}
|
||||
// take the lock
|
||||
LockState[key] = lockValue = { created: Date.now() }
|
||||
return callback(null, true, lockValue)
|
||||
},
|
||||
|
||||
getLock(key, callback) {
|
||||
let attempt
|
||||
if (callback == null) {
|
||||
callback = function(error, lockValue) {}
|
||||
}
|
||||
const startTime = Date.now()
|
||||
return (attempt = () =>
|
||||
LockManager.tryLock(key, function(error, gotLock, lockValue) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
if (gotLock) {
|
||||
return callback(null, lockValue)
|
||||
} else if (Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME) {
|
||||
const e = new Error('Lock timeout')
|
||||
e.key = key
|
||||
return callback(e)
|
||||
} else {
|
||||
return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL)
|
||||
}
|
||||
}))()
|
||||
},
|
||||
|
||||
releaseLock(key, lockValue, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const existingLock = LockState[key]
|
||||
if (existingLock === lockValue) {
|
||||
// lockValue is an object, so we can test by reference
|
||||
delete LockState[key] // our lock, so we can free it
|
||||
return callback()
|
||||
} else if (existingLock != null) {
|
||||
// lock exists but doesn't match ours
|
||||
logger.error(
|
||||
{ key, lock: existingLock },
|
||||
'tried to release lock taken by force'
|
||||
)
|
||||
return callback()
|
||||
} else {
|
||||
logger.error(
|
||||
{ key, lock: existingLock },
|
||||
'tried to release lock that has gone'
|
||||
)
|
||||
return callback()
|
||||
}
|
||||
},
|
||||
|
||||
runWithLock(key, runner, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return LockManager.getLock(key, function(error, lockValue) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return runner((error1, ...args) =>
|
||||
LockManager.releaseLock(key, lockValue, function(error2) {
|
||||
error = error1 || error2
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null, ...Array.from(args))
|
||||
})
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
695
app/js/DockerRunner.js
Normal file
695
app/js/DockerRunner.js
Normal file
@@ -0,0 +1,695 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let DockerRunner, oneHour
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Docker = require('dockerode')
|
||||
const dockerode = new Docker()
|
||||
const crypto = require('crypto')
|
||||
const async = require('async')
|
||||
const LockManager = require('./DockerLockManager')
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
const _ = require('underscore')
|
||||
|
||||
logger.info('using docker runner')
|
||||
|
||||
const usingSiblingContainers = () =>
|
||||
__guard__(
|
||||
Settings != null ? Settings.path : undefined,
|
||||
x => x.sandboxedCompilesHostDir
|
||||
) != null
|
||||
|
||||
let containerMonitorTimeout
|
||||
let containerMonitorInterval
|
||||
|
||||
module.exports = DockerRunner = {
|
||||
ERR_NOT_DIRECTORY: new Error('not a directory'),
|
||||
ERR_TERMINATED: new Error('terminated'),
|
||||
ERR_EXITED: new Error('exited'),
|
||||
ERR_TIMED_OUT: new Error('container timed out'),
|
||||
|
||||
run(project_id, command, directory, image, timeout, environment, callback) {
|
||||
let name
|
||||
if (callback == null) {
|
||||
callback = function(error, output) {}
|
||||
}
|
||||
if (usingSiblingContainers()) {
|
||||
const _newPath = Settings.path.sandboxedCompilesHostDir
|
||||
logger.log(
|
||||
{ path: _newPath },
|
||||
'altering bind path for sibling containers'
|
||||
)
|
||||
// Server Pro, example:
|
||||
// '/var/lib/sharelatex/data/compiles/<project-id>'
|
||||
// ... becomes ...
|
||||
// '/opt/sharelatex_data/data/compiles/<project-id>'
|
||||
directory = Path.join(
|
||||
Settings.path.sandboxedCompilesHostDir,
|
||||
Path.basename(directory)
|
||||
)
|
||||
}
|
||||
|
||||
const volumes = {}
|
||||
volumes[directory] = '/compile'
|
||||
|
||||
command = Array.from(command).map(arg =>
|
||||
__guardMethod__(arg.toString(), 'replace', o =>
|
||||
o.replace('$COMPILE_DIR', '/compile')
|
||||
)
|
||||
)
|
||||
if (image == null) {
|
||||
;({ image } = Settings.clsi.docker)
|
||||
}
|
||||
|
||||
if (Settings.texliveImageNameOveride != null) {
|
||||
const img = image.split('/')
|
||||
image = `${Settings.texliveImageNameOveride}/${img[2]}`
|
||||
}
|
||||
|
||||
const options = DockerRunner._getContainerOptions(
|
||||
command,
|
||||
image,
|
||||
volumes,
|
||||
timeout,
|
||||
environment
|
||||
)
|
||||
const fingerprint = DockerRunner._fingerprintContainer(options)
|
||||
options.name = name = `project-${project_id}-${fingerprint}`
|
||||
|
||||
// logOptions = _.clone(options)
|
||||
// logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
|
||||
logger.log({ project_id }, 'running docker container')
|
||||
DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(
|
||||
error,
|
||||
output
|
||||
) {
|
||||
if (error && error.statusCode === 500) {
|
||||
logger.log(
|
||||
{ err: error, project_id },
|
||||
'error running container so destroying and retrying'
|
||||
)
|
||||
return DockerRunner.destroyContainer(name, null, true, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return DockerRunner._runAndWaitForContainer(
|
||||
options,
|
||||
volumes,
|
||||
timeout,
|
||||
callback
|
||||
)
|
||||
})
|
||||
} else {
|
||||
return callback(error, output)
|
||||
}
|
||||
})
|
||||
|
||||
return name
|
||||
}, // pass back the container name to allow it to be killed
|
||||
|
||||
kill(container_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
logger.log({ container_id }, 'sending kill signal to container')
|
||||
const container = dockerode.getContainer(container_id)
|
||||
return container.kill(function(error) {
|
||||
if (
|
||||
error != null &&
|
||||
__guardMethod__(error != null ? error.message : undefined, 'match', o =>
|
||||
o.match(/Cannot kill container .* is not running/)
|
||||
)
|
||||
) {
|
||||
logger.warn(
|
||||
{ err: error, container_id },
|
||||
'container not running, continuing'
|
||||
)
|
||||
error = null
|
||||
}
|
||||
if (error != null) {
|
||||
logger.error({ err: error, container_id }, 'error killing container')
|
||||
return callback(error)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_runAndWaitForContainer(options, volumes, timeout, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error, output) {}
|
||||
}
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []))
|
||||
// Only call the callback once
|
||||
return (_callback = function() {})
|
||||
}
|
||||
|
||||
const { name } = options
|
||||
|
||||
let streamEnded = false
|
||||
let containerReturned = false
|
||||
let output = {}
|
||||
|
||||
const callbackIfFinished = function() {
|
||||
if (streamEnded && containerReturned) {
|
||||
return callback(null, output)
|
||||
}
|
||||
}
|
||||
|
||||
const attachStreamHandler = function(error, _output) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
output = _output
|
||||
streamEnded = true
|
||||
return callbackIfFinished()
|
||||
}
|
||||
|
||||
return DockerRunner.startContainer(
|
||||
options,
|
||||
volumes,
|
||||
attachStreamHandler,
|
||||
function(error, containerId) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
|
||||
return DockerRunner.waitForContainer(name, timeout, function(
|
||||
error,
|
||||
exitCode
|
||||
) {
|
||||
let err
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
if (exitCode === 137) {
|
||||
// exit status from kill -9
|
||||
err = DockerRunner.ERR_TERMINATED
|
||||
err.terminated = true
|
||||
return callback(err)
|
||||
}
|
||||
if (exitCode === 1) {
|
||||
// exit status from chktex
|
||||
err = DockerRunner.ERR_EXITED
|
||||
err.code = exitCode
|
||||
return callback(err)
|
||||
}
|
||||
containerReturned = true
|
||||
__guard__(
|
||||
options != null ? options.HostConfig : undefined,
|
||||
x => (x.SecurityOpt = null)
|
||||
) // small log line
|
||||
logger.log({ err, exitCode, options }, 'docker container has exited')
|
||||
return callbackIfFinished()
|
||||
})
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
_getContainerOptions(command, image, volumes, timeout, environment) {
|
||||
let m, year
|
||||
let key, value, hostVol, dockerVol
|
||||
const timeoutInSeconds = timeout / 1000
|
||||
|
||||
const dockerVolumes = {}
|
||||
for (hostVol in volumes) {
|
||||
dockerVol = volumes[hostVol]
|
||||
dockerVolumes[dockerVol] = {}
|
||||
|
||||
if (volumes[hostVol].slice(-3).indexOf(':r') === -1) {
|
||||
volumes[hostVol] = `${dockerVol}:rw`
|
||||
}
|
||||
}
|
||||
|
||||
// merge settings and environment parameter
|
||||
const env = {}
|
||||
for (const src of [Settings.clsi.docker.env, environment || {}]) {
|
||||
for (key in src) {
|
||||
value = src[key]
|
||||
env[key] = value
|
||||
}
|
||||
}
|
||||
// set the path based on the image year
|
||||
if ((m = image.match(/:([0-9]+)\.[0-9]+/))) {
|
||||
year = m[1]
|
||||
} else {
|
||||
year = '2014'
|
||||
}
|
||||
env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`
|
||||
const options = {
|
||||
Cmd: command,
|
||||
Image: image,
|
||||
Volumes: dockerVolumes,
|
||||
WorkingDir: '/compile',
|
||||
NetworkDisabled: true,
|
||||
Memory: 1024 * 1024 * 1024 * 1024, // 1 Gb
|
||||
User: Settings.clsi.docker.user,
|
||||
Env: (() => {
|
||||
const result = []
|
||||
for (key in env) {
|
||||
value = env[key]
|
||||
result.push(`${key}=${value}`)
|
||||
}
|
||||
return result
|
||||
})(), // convert the environment hash to an array
|
||||
HostConfig: {
|
||||
Binds: (() => {
|
||||
const result1 = []
|
||||
for (hostVol in volumes) {
|
||||
dockerVol = volumes[hostVol]
|
||||
result1.push(`${hostVol}:${dockerVol}`)
|
||||
}
|
||||
return result1
|
||||
})(),
|
||||
LogConfig: { Type: 'none', Config: {} },
|
||||
Ulimits: [
|
||||
{
|
||||
Name: 'cpu',
|
||||
Soft: timeoutInSeconds + 5,
|
||||
Hard: timeoutInSeconds + 10
|
||||
}
|
||||
],
|
||||
CapDrop: 'ALL',
|
||||
SecurityOpt: ['no-new-privileges']
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
(Settings.path != null ? Settings.path.synctexBinHostPath : undefined) !=
|
||||
null
|
||||
) {
|
||||
options.HostConfig.Binds.push(
|
||||
`${Settings.path.synctexBinHostPath}:/opt/synctex:ro`
|
||||
)
|
||||
}
|
||||
|
||||
if (Settings.clsi.docker.seccomp_profile != null) {
|
||||
options.HostConfig.SecurityOpt.push(
|
||||
`seccomp=${Settings.clsi.docker.seccomp_profile}`
|
||||
)
|
||||
}
|
||||
|
||||
if (Settings.clsi.docker.runtime) {
|
||||
options.HostConfig.Runtime = Settings.clsi.docker.runtime
|
||||
}
|
||||
|
||||
return options
|
||||
},
|
||||
|
||||
_fingerprintContainer(containerOptions) {
|
||||
// Yay, Hashing!
|
||||
const json = JSON.stringify(containerOptions)
|
||||
return crypto
|
||||
.createHash('md5')
|
||||
.update(json)
|
||||
.digest('hex')
|
||||
},
|
||||
|
||||
startContainer(options, volumes, attachStreamHandler, callback) {
|
||||
return LockManager.runWithLock(
|
||||
options.name,
|
||||
releaseLock =>
|
||||
// Check that volumes exist before starting the container.
|
||||
// When a container is started with volume pointing to a
|
||||
// non-existent directory then docker creates the directory but
|
||||
// with root ownership.
|
||||
DockerRunner._checkVolumes(options, volumes, function(err) {
|
||||
if (err != null) {
|
||||
return releaseLock(err)
|
||||
}
|
||||
return DockerRunner._startContainer(
|
||||
options,
|
||||
volumes,
|
||||
attachStreamHandler,
|
||||
releaseLock
|
||||
)
|
||||
}),
|
||||
|
||||
callback
|
||||
)
|
||||
},
|
||||
|
||||
// Check that volumes exist and are directories
|
||||
_checkVolumes(options, volumes, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, containerName) {}
|
||||
}
|
||||
if (usingSiblingContainers()) {
|
||||
// Server Pro, with sibling-containers active, skip checks
|
||||
return callback(null)
|
||||
}
|
||||
|
||||
const checkVolume = (path, cb) =>
|
||||
fs.stat(path, function(err, stats) {
|
||||
if (err != null) {
|
||||
return cb(err)
|
||||
}
|
||||
if (!(stats != null ? stats.isDirectory() : undefined)) {
|
||||
return cb(DockerRunner.ERR_NOT_DIRECTORY)
|
||||
}
|
||||
return cb()
|
||||
})
|
||||
const jobs = []
|
||||
for (const vol in volumes) {
|
||||
;(vol => jobs.push(cb => checkVolume(vol, cb)))(vol)
|
||||
}
|
||||
return async.series(jobs, callback)
|
||||
},
|
||||
|
||||
_startContainer(options, volumes, attachStreamHandler, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, output) {}
|
||||
}
|
||||
callback = _.once(callback)
|
||||
const { name } = options
|
||||
|
||||
logger.log({ container_name: name }, 'starting container')
|
||||
const container = dockerode.getContainer(name)
|
||||
|
||||
const createAndStartContainer = () =>
|
||||
dockerode.createContainer(options, function(error, container) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return startExistingContainer()
|
||||
})
|
||||
var startExistingContainer = () =>
|
||||
DockerRunner.attachToContainer(
|
||||
options.name,
|
||||
attachStreamHandler,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return container.start(function(error) {
|
||||
if (
|
||||
error != null &&
|
||||
(error != null ? error.statusCode : undefined) !== 304
|
||||
) {
|
||||
// already running
|
||||
return callback(error)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
}
|
||||
)
|
||||
return container.inspect(function(error, stats) {
|
||||
if ((error != null ? error.statusCode : undefined) === 404) {
|
||||
return createAndStartContainer()
|
||||
} else if (error != null) {
|
||||
logger.err(
|
||||
{ container_name: name, error },
|
||||
'unable to inspect container to start'
|
||||
)
|
||||
return callback(error)
|
||||
} else {
|
||||
return startExistingContainer()
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
|
||||
const container = dockerode.getContainer(containerId)
|
||||
return container.attach({ stdout: 1, stderr: 1, stream: 1 }, function(
|
||||
error,
|
||||
stream
|
||||
) {
|
||||
if (error != null) {
|
||||
logger.error(
|
||||
{ err: error, container_id: containerId },
|
||||
'error attaching to container'
|
||||
)
|
||||
return attachStartCallback(error)
|
||||
} else {
|
||||
attachStartCallback()
|
||||
}
|
||||
|
||||
logger.log({ container_id: containerId }, 'attached to container')
|
||||
|
||||
const MAX_OUTPUT = 1024 * 1024 // limit output to 1MB
|
||||
const createStringOutputStream = function(name) {
|
||||
return {
|
||||
data: '',
|
||||
overflowed: false,
|
||||
write(data) {
|
||||
if (this.overflowed) {
|
||||
return
|
||||
}
|
||||
if (this.data.length < MAX_OUTPUT) {
|
||||
return (this.data += data)
|
||||
} else {
|
||||
logger.error(
|
||||
{
|
||||
container_id: containerId,
|
||||
length: this.data.length,
|
||||
maxLen: MAX_OUTPUT
|
||||
},
|
||||
`${name} exceeds max size`
|
||||
)
|
||||
this.data += `(...truncated at ${MAX_OUTPUT} chars...)`
|
||||
return (this.overflowed = true)
|
||||
}
|
||||
}
|
||||
// kill container if too much output
|
||||
// docker.containers.kill(containerId, () ->)
|
||||
}
|
||||
}
|
||||
|
||||
const stdout = createStringOutputStream('stdout')
|
||||
const stderr = createStringOutputStream('stderr')
|
||||
|
||||
container.modem.demuxStream(stream, stdout, stderr)
|
||||
|
||||
stream.on('error', err =>
|
||||
logger.error(
|
||||
{ err, container_id: containerId },
|
||||
'error reading from container stream'
|
||||
)
|
||||
)
|
||||
|
||||
return stream.on('end', () =>
|
||||
attachStreamHandler(null, { stdout: stdout.data, stderr: stderr.data })
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
waitForContainer(containerId, timeout, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error, exitCode) {}
|
||||
}
|
||||
const callback = function(...args) {
|
||||
_callback(...Array.from(args || []))
|
||||
// Only call the callback once
|
||||
return (_callback = function() {})
|
||||
}
|
||||
|
||||
const container = dockerode.getContainer(containerId)
|
||||
|
||||
let timedOut = false
|
||||
const timeoutId = setTimeout(function() {
|
||||
timedOut = true
|
||||
logger.log(
|
||||
{ container_id: containerId },
|
||||
'timeout reached, killing container'
|
||||
)
|
||||
return container.kill(function() {})
|
||||
}, timeout)
|
||||
|
||||
logger.log({ container_id: containerId }, 'waiting for docker container')
|
||||
return container.wait(function(error, res) {
|
||||
if (error != null) {
|
||||
clearTimeout(timeoutId)
|
||||
logger.error(
|
||||
{ err: error, container_id: containerId },
|
||||
'error waiting for container'
|
||||
)
|
||||
return callback(error)
|
||||
}
|
||||
if (timedOut) {
|
||||
logger.log({ containerId }, 'docker container timed out')
|
||||
error = DockerRunner.ERR_TIMED_OUT
|
||||
error.timedout = true
|
||||
return callback(error)
|
||||
} else {
|
||||
clearTimeout(timeoutId)
|
||||
logger.log(
|
||||
{ container_id: containerId, exitCode: res.StatusCode },
|
||||
'docker container returned'
|
||||
)
|
||||
return callback(null, res.StatusCode)
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
destroyContainer(containerName, containerId, shouldForce, callback) {
|
||||
// We want the containerName for the lock and, ideally, the
|
||||
// containerId to delete. There is a bug in the docker.io module
|
||||
// where if you delete by name and there is an error, it throws an
|
||||
// async exception, but if you delete by id it just does a normal
|
||||
// error callback. We fall back to deleting by name if no id is
|
||||
// supplied.
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return LockManager.runWithLock(
|
||||
containerName,
|
||||
releaseLock =>
|
||||
DockerRunner._destroyContainer(
|
||||
containerId || containerName,
|
||||
shouldForce,
|
||||
releaseLock
|
||||
),
|
||||
callback
|
||||
)
|
||||
},
|
||||
|
||||
_destroyContainer(containerId, shouldForce, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
logger.log({ container_id: containerId }, 'destroying docker container')
|
||||
const container = dockerode.getContainer(containerId)
|
||||
return container.remove({ force: shouldForce === true }, function(error) {
|
||||
if (
|
||||
error != null &&
|
||||
(error != null ? error.statusCode : undefined) === 404
|
||||
) {
|
||||
logger.warn(
|
||||
{ err: error, container_id: containerId },
|
||||
'container not found, continuing'
|
||||
)
|
||||
error = null
|
||||
}
|
||||
if (error != null) {
|
||||
logger.error(
|
||||
{ err: error, container_id: containerId },
|
||||
'error destroying container'
|
||||
)
|
||||
} else {
|
||||
logger.log({ container_id: containerId }, 'destroyed container')
|
||||
}
|
||||
return callback(error)
|
||||
})
|
||||
},
|
||||
|
||||
// handle expiry of docker containers
|
||||
|
||||
MAX_CONTAINER_AGE:
|
||||
Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000),
|
||||
|
||||
examineOldContainer(container, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, name, id, ttl) {}
|
||||
}
|
||||
const name =
|
||||
container.Name ||
|
||||
(container.Names != null ? container.Names[0] : undefined)
|
||||
const created = container.Created * 1000 // creation time is returned in seconds
|
||||
const now = Date.now()
|
||||
const age = now - created
|
||||
const maxAge = DockerRunner.MAX_CONTAINER_AGE
|
||||
const ttl = maxAge - age
|
||||
logger.log(
|
||||
{ containerName: name, created, now, age, maxAge, ttl },
|
||||
'checking whether to destroy container'
|
||||
)
|
||||
return callback(null, name, container.Id, ttl)
|
||||
},
|
||||
|
||||
destroyOldContainers(callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return dockerode.listContainers({ all: true }, function(error, containers) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const jobs = []
|
||||
for (const container of Array.from(containers || [])) {
|
||||
;(container =>
|
||||
DockerRunner.examineOldContainer(container, function(
|
||||
err,
|
||||
name,
|
||||
id,
|
||||
ttl
|
||||
) {
|
||||
if (name.slice(0, 9) === '/project-' && ttl <= 0) {
|
||||
return jobs.push(cb =>
|
||||
DockerRunner.destroyContainer(name, id, false, () => cb())
|
||||
)
|
||||
}
|
||||
}))(container)
|
||||
}
|
||||
// Ignore errors because some containers get stuck but
|
||||
// will be destroyed next time
|
||||
return async.series(jobs, callback)
|
||||
})
|
||||
},
|
||||
|
||||
startContainerMonitor() {
|
||||
logger.log(
|
||||
{ maxAge: DockerRunner.MAX_CONTAINER_AGE },
|
||||
'starting container expiry'
|
||||
)
|
||||
|
||||
// guarantee only one monitor is running
|
||||
DockerRunner.stopContainerMonitor()
|
||||
|
||||
// randomise the start time
|
||||
const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
|
||||
containerMonitorTimeout = setTimeout(() => {
|
||||
containerMonitorInterval = setInterval(
|
||||
() => DockerRunner.destroyOldContainers(),
|
||||
(oneHour = 60 * 60 * 1000)
|
||||
)
|
||||
}, randomDelay)
|
||||
},
|
||||
|
||||
stopContainerMonitor() {
|
||||
if (containerMonitorTimeout) {
|
||||
clearTimeout(containerMonitorTimeout)
|
||||
containerMonitorTimeout = undefined
|
||||
}
|
||||
if (containerMonitorInterval) {
|
||||
clearInterval(containerMonitorTimeout)
|
||||
containerMonitorTimeout = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
DockerRunner.startContainerMonitor()
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
function __guardMethod__(obj, methodName, transform) {
|
||||
if (
|
||||
typeof obj !== 'undefined' &&
|
||||
obj !== null &&
|
||||
typeof obj[methodName] === 'function'
|
||||
) {
|
||||
return transform(obj, methodName)
|
||||
} else {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
57
app/js/DraftModeManager.js
Normal file
57
app/js/DraftModeManager.js
Normal file
@@ -0,0 +1,57 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-useless-escape,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let DraftModeManager
|
||||
const fs = require('fs')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
module.exports = DraftModeManager = {
|
||||
injectDraftMode(filename, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return fs.readFile(filename, 'utf8', function(error, content) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
// avoid adding draft mode more than once
|
||||
if (
|
||||
(content != null
|
||||
? content.indexOf('\\documentclass[draft')
|
||||
: undefined) >= 0
|
||||
) {
|
||||
return callback()
|
||||
}
|
||||
const modified_content = DraftModeManager._injectDraftOption(content)
|
||||
logger.log(
|
||||
{
|
||||
content: content.slice(0, 1024), // \documentclass is normally v near the top
|
||||
modified_content: modified_content.slice(0, 1024),
|
||||
filename
|
||||
},
|
||||
'injected draft class'
|
||||
)
|
||||
return fs.writeFile(filename, modified_content, callback)
|
||||
})
|
||||
},
|
||||
|
||||
_injectDraftOption(content) {
|
||||
return (
|
||||
content
|
||||
// With existing options (must be first, otherwise both are applied)
|
||||
.replace(/\\documentclass\[/g, '\\documentclass[draft,')
|
||||
// Without existing options
|
||||
.replace(/\\documentclass\{/g, '\\documentclass[draft]{')
|
||||
)
|
||||
}
|
||||
}
|
||||
36
app/js/Errors.js
Normal file
36
app/js/Errors.js
Normal file
@@ -0,0 +1,36 @@
|
||||
/* eslint-disable
|
||||
no-proto,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
let Errors
|
||||
var NotFoundError = function(message) {
|
||||
const error = new Error(message)
|
||||
error.name = 'NotFoundError'
|
||||
error.__proto__ = NotFoundError.prototype
|
||||
return error
|
||||
}
|
||||
NotFoundError.prototype.__proto__ = Error.prototype
|
||||
|
||||
var FilesOutOfSyncError = function(message) {
|
||||
const error = new Error(message)
|
||||
error.name = 'FilesOutOfSyncError'
|
||||
error.__proto__ = FilesOutOfSyncError.prototype
|
||||
return error
|
||||
}
|
||||
FilesOutOfSyncError.prototype.__proto__ = Error.prototype
|
||||
|
||||
var AlreadyCompilingError = function(message) {
|
||||
const error = new Error(message)
|
||||
error.name = 'AlreadyCompilingError'
|
||||
error.__proto__ = AlreadyCompilingError.prototype
|
||||
return error
|
||||
}
|
||||
AlreadyCompilingError.prototype.__proto__ = Error.prototype
|
||||
|
||||
module.exports = Errors = {
|
||||
NotFoundError,
|
||||
FilesOutOfSyncError,
|
||||
AlreadyCompilingError
|
||||
}
|
||||
204
app/js/LatexRunner.js
Normal file
204
app/js/LatexRunner.js
Normal file
@@ -0,0 +1,204 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LatexRunner
|
||||
const Path = require('path')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Metrics = require('./Metrics')
|
||||
const CommandRunner = require('./CommandRunner')
|
||||
|
||||
const ProcessTable = {} // table of currently running jobs (pids or docker container names)
|
||||
|
||||
module.exports = LatexRunner = {
|
||||
runLatex(project_id, options, callback) {
|
||||
let command
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
let {
|
||||
directory,
|
||||
mainFile,
|
||||
compiler,
|
||||
timeout,
|
||||
image,
|
||||
environment,
|
||||
flags
|
||||
} = options
|
||||
if (!compiler) {
|
||||
compiler = 'pdflatex'
|
||||
}
|
||||
if (!timeout) {
|
||||
timeout = 60000
|
||||
} // milliseconds
|
||||
|
||||
logger.log(
|
||||
{ directory, compiler, timeout, mainFile, environment, flags },
|
||||
'starting compile'
|
||||
)
|
||||
|
||||
// We want to run latexmk on the tex file which we will automatically
|
||||
// generate from the Rtex/Rmd/md file.
|
||||
mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, '.tex')
|
||||
|
||||
if (compiler === 'pdflatex') {
|
||||
command = LatexRunner._pdflatexCommand(mainFile, flags)
|
||||
} else if (compiler === 'latex') {
|
||||
command = LatexRunner._latexCommand(mainFile, flags)
|
||||
} else if (compiler === 'xelatex') {
|
||||
command = LatexRunner._xelatexCommand(mainFile, flags)
|
||||
} else if (compiler === 'lualatex') {
|
||||
command = LatexRunner._lualatexCommand(mainFile, flags)
|
||||
} else {
|
||||
return callback(new Error(`unknown compiler: ${compiler}`))
|
||||
}
|
||||
|
||||
if (Settings.clsi != null ? Settings.clsi.strace : undefined) {
|
||||
command = ['strace', '-o', 'strace', '-ff'].concat(command)
|
||||
}
|
||||
|
||||
const id = `${project_id}` // record running project under this id
|
||||
|
||||
return (ProcessTable[id] = CommandRunner.run(
|
||||
project_id,
|
||||
command,
|
||||
directory,
|
||||
image,
|
||||
timeout,
|
||||
environment,
|
||||
function(error, output) {
|
||||
delete ProcessTable[id]
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const runs =
|
||||
__guard__(
|
||||
__guard__(output != null ? output.stderr : undefined, x1 =>
|
||||
x1.match(/^Run number \d+ of .*latex/gm)
|
||||
),
|
||||
x => x.length
|
||||
) || 0
|
||||
const failed =
|
||||
__guard__(output != null ? output.stdout : undefined, x2 =>
|
||||
x2.match(/^Latexmk: Errors/m)
|
||||
) != null
|
||||
? 1
|
||||
: 0
|
||||
// counters from latexmk output
|
||||
const stats = {}
|
||||
stats['latexmk-errors'] = failed
|
||||
stats['latex-runs'] = runs
|
||||
stats['latex-runs-with-errors'] = failed ? runs : 0
|
||||
stats[`latex-runs-${runs}`] = 1
|
||||
stats[`latex-runs-with-errors-${runs}`] = failed ? 1 : 0
|
||||
// timing information from /usr/bin/time
|
||||
const timings = {}
|
||||
const stderr = output != null ? output.stderr : undefined
|
||||
timings['cpu-percent'] =
|
||||
__guard__(
|
||||
stderr != null
|
||||
? stderr.match(/Percent of CPU this job got: (\d+)/m)
|
||||
: undefined,
|
||||
x3 => x3[1]
|
||||
) || 0
|
||||
timings['cpu-time'] =
|
||||
__guard__(
|
||||
stderr != null
|
||||
? stderr.match(/User time.*: (\d+.\d+)/m)
|
||||
: undefined,
|
||||
x4 => x4[1]
|
||||
) || 0
|
||||
timings['sys-time'] =
|
||||
__guard__(
|
||||
stderr != null
|
||||
? stderr.match(/System time.*: (\d+.\d+)/m)
|
||||
: undefined,
|
||||
x5 => x5[1]
|
||||
) || 0
|
||||
return callback(error, output, stats, timings)
|
||||
}
|
||||
))
|
||||
},
|
||||
|
||||
killLatex(project_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const id = `${project_id}`
|
||||
logger.log({ id }, 'killing running compile')
|
||||
if (ProcessTable[id] == null) {
|
||||
logger.warn({ id }, 'no such project to kill')
|
||||
return callback(null)
|
||||
} else {
|
||||
return CommandRunner.kill(ProcessTable[id], callback)
|
||||
}
|
||||
},
|
||||
|
||||
_latexmkBaseCommand(flags) {
|
||||
let args = [
|
||||
'latexmk',
|
||||
'-cd',
|
||||
'-f',
|
||||
'-jobname=output',
|
||||
'-auxdir=$COMPILE_DIR',
|
||||
'-outdir=$COMPILE_DIR',
|
||||
'-synctex=1',
|
||||
'-interaction=batchmode'
|
||||
]
|
||||
if (flags) {
|
||||
args = args.concat(flags)
|
||||
}
|
||||
return (
|
||||
__guard__(
|
||||
Settings != null ? Settings.clsi : undefined,
|
||||
x => x.latexmkCommandPrefix
|
||||
) || []
|
||||
).concat(args)
|
||||
},
|
||||
|
||||
_pdflatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
'-pdf',
|
||||
Path.join('$COMPILE_DIR', mainFile)
|
||||
])
|
||||
},
|
||||
|
||||
_latexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
'-pdfdvi',
|
||||
Path.join('$COMPILE_DIR', mainFile)
|
||||
])
|
||||
},
|
||||
|
||||
_xelatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
'-xelatex',
|
||||
Path.join('$COMPILE_DIR', mainFile)
|
||||
])
|
||||
},
|
||||
|
||||
_lualatexCommand(mainFile, flags) {
|
||||
return LatexRunner._latexmkBaseCommand(flags).concat([
|
||||
'-lualatex',
|
||||
Path.join('$COMPILE_DIR', mainFile)
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
91
app/js/LocalCommandRunner.js
Normal file
91
app/js/LocalCommandRunner.js
Normal file
@@ -0,0 +1,91 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let CommandRunner
|
||||
const { spawn } = require('child_process')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
logger.info('using standard command runner')
|
||||
|
||||
module.exports = CommandRunner = {
|
||||
run(project_id, command, directory, image, timeout, environment, callback) {
|
||||
let key, value
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
command = Array.from(command).map(arg =>
|
||||
arg.toString().replace('$COMPILE_DIR', directory)
|
||||
)
|
||||
logger.log({ project_id, command, directory }, 'running command')
|
||||
logger.warn('timeouts and sandboxing are not enabled with CommandRunner')
|
||||
|
||||
// merge environment settings
|
||||
const env = {}
|
||||
for (key in process.env) {
|
||||
value = process.env[key]
|
||||
env[key] = value
|
||||
}
|
||||
for (key in environment) {
|
||||
value = environment[key]
|
||||
env[key] = value
|
||||
}
|
||||
|
||||
// run command as detached process so it has its own process group (which can be killed if needed)
|
||||
const proc = spawn(command[0], command.slice(1), { cwd: directory, env })
|
||||
|
||||
let stdout = ''
|
||||
proc.stdout.on('data', data => (stdout += data))
|
||||
|
||||
proc.on('error', function(err) {
|
||||
logger.err(
|
||||
{ err, project_id, command, directory },
|
||||
'error running command'
|
||||
)
|
||||
return callback(err)
|
||||
})
|
||||
|
||||
proc.on('close', function(code, signal) {
|
||||
let err
|
||||
logger.info({ code, signal, project_id }, 'command exited')
|
||||
if (signal === 'SIGTERM') {
|
||||
// signal from kill method below
|
||||
err = new Error('terminated')
|
||||
err.terminated = true
|
||||
return callback(err)
|
||||
} else if (code === 1) {
|
||||
// exit status from chktex
|
||||
err = new Error('exited')
|
||||
err.code = code
|
||||
return callback(err)
|
||||
} else {
|
||||
return callback(null, { stdout: stdout })
|
||||
}
|
||||
})
|
||||
|
||||
return proc.pid
|
||||
}, // return process id to allow job to be killed if necessary
|
||||
|
||||
kill(pid, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
try {
|
||||
process.kill(-pid) // kill all processes in group
|
||||
} catch (err) {
|
||||
return callback(err)
|
||||
}
|
||||
return callback()
|
||||
}
|
||||
}
|
||||
72
app/js/LockManager.js
Normal file
72
app/js/LockManager.js
Normal file
@@ -0,0 +1,72 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let LockManager
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Lockfile = require('lockfile') // from https://github.com/npm/lockfile
|
||||
const Errors = require('./Errors')
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
module.exports = LockManager = {
|
||||
LOCK_TEST_INTERVAL: 1000, // 50ms between each test of the lock
|
||||
MAX_LOCK_WAIT_TIME: 15000, // 10s maximum time to spend trying to get the lock
|
||||
LOCK_STALE: 5 * 60 * 1000, // 5 mins time until lock auto expires
|
||||
|
||||
runWithLock(path, runner, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const lockOpts = {
|
||||
wait: this.MAX_LOCK_WAIT_TIME,
|
||||
pollPeriod: this.LOCK_TEST_INTERVAL,
|
||||
stale: this.LOCK_STALE
|
||||
}
|
||||
return Lockfile.lock(path, lockOpts, function(error) {
|
||||
if ((error != null ? error.code : undefined) === 'EEXIST') {
|
||||
return callback(new Errors.AlreadyCompilingError('compile in progress'))
|
||||
} else if (error != null) {
|
||||
return fs.lstat(path, (statLockErr, statLock) =>
|
||||
fs.lstat(Path.dirname(path), (statDirErr, statDir) =>
|
||||
fs.readdir(Path.dirname(path), function(readdirErr, readdirDir) {
|
||||
logger.err(
|
||||
{
|
||||
error,
|
||||
path,
|
||||
statLock,
|
||||
statLockErr,
|
||||
statDir,
|
||||
statDirErr,
|
||||
readdirErr,
|
||||
readdirDir
|
||||
},
|
||||
'unable to get lock'
|
||||
)
|
||||
return callback(error)
|
||||
})
|
||||
)
|
||||
)
|
||||
} else {
|
||||
return runner((error1, ...args) =>
|
||||
Lockfile.unlock(path, function(error2) {
|
||||
error = error1 || error2
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null, ...Array.from(args))
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
3
app/js/Metrics.js
Normal file
3
app/js/Metrics.js
Normal file
@@ -0,0 +1,3 @@
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Sanity-check the conversion and remove this comment.
|
||||
module.exports = require('metrics-sharelatex')
|
||||
399
app/js/OutputCacheManager.js
Normal file
399
app/js/OutputCacheManager.js
Normal file
@@ -0,0 +1,399 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS104: Avoid inline assignments
|
||||
* DS204: Change includes calls to have a more natural evaluation order
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputCacheManager
|
||||
const async = require('async')
|
||||
const fs = require('fs')
|
||||
const fse = require('fs-extra')
|
||||
const Path = require('path')
|
||||
const logger = require('logger-sharelatex')
|
||||
const _ = require('underscore')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const crypto = require('crypto')
|
||||
|
||||
const OutputFileOptimiser = require('./OutputFileOptimiser')
|
||||
|
||||
module.exports = OutputCacheManager = {
|
||||
CACHE_SUBDIR: '.cache/clsi',
|
||||
ARCHIVE_SUBDIR: '.archive/clsi',
|
||||
// build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
|
||||
// for backwards compatibility, make the randombytes part optional
|
||||
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
|
||||
CACHE_LIMIT: 2, // maximum number of cache directories
|
||||
CACHE_AGE: 60 * 60 * 1000, // up to one hour old
|
||||
|
||||
path(buildId, file) {
|
||||
// used by static server, given build id return '.cache/clsi/buildId'
|
||||
if (buildId.match(OutputCacheManager.BUILD_REGEX)) {
|
||||
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
|
||||
} else {
|
||||
// for invalid build id, return top level
|
||||
return file
|
||||
}
|
||||
},
|
||||
|
||||
generateBuildId(callback) {
|
||||
// generate a secure build id from Date.now() and 8 random bytes in hex
|
||||
if (callback == null) {
|
||||
callback = function(error, buildId) {}
|
||||
}
|
||||
return crypto.randomBytes(8, function(err, buf) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
const random = buf.toString('hex')
|
||||
const date = Date.now().toString(16)
|
||||
return callback(err, `${date}-${random}`)
|
||||
})
|
||||
},
|
||||
|
||||
saveOutputFiles(outputFiles, compileDir, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return OutputCacheManager.generateBuildId(function(err, buildId) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return OutputCacheManager.saveOutputFilesInBuildDir(
|
||||
outputFiles,
|
||||
compileDir,
|
||||
buildId,
|
||||
callback
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
saveOutputFilesInBuildDir(outputFiles, compileDir, buildId, callback) {
|
||||
// make a compileDir/CACHE_SUBDIR/build_id directory and
|
||||
// copy all the output files into it
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
|
||||
// Put the files into a new cache subdirectory
|
||||
const cacheDir = Path.join(
|
||||
compileDir,
|
||||
OutputCacheManager.CACHE_SUBDIR,
|
||||
buildId
|
||||
)
|
||||
// Is it a per-user compile? check if compile directory is PROJECTID-USERID
|
||||
const perUser = Path.basename(compileDir).match(
|
||||
/^[0-9a-f]{24}-[0-9a-f]{24}$/
|
||||
)
|
||||
|
||||
// Archive logs in background
|
||||
if (
|
||||
(Settings.clsi != null ? Settings.clsi.archive_logs : undefined) ||
|
||||
(Settings.clsi != null ? Settings.clsi.strace : undefined)
|
||||
) {
|
||||
OutputCacheManager.archiveLogs(outputFiles, compileDir, buildId, function(
|
||||
err
|
||||
) {
|
||||
if (err != null) {
|
||||
return logger.warn({ err }, 'erroring archiving log files')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// make the new cache directory
|
||||
return fse.ensureDir(cacheDir, function(err) {
|
||||
if (err != null) {
|
||||
logger.error(
|
||||
{ err, directory: cacheDir },
|
||||
'error creating cache directory'
|
||||
)
|
||||
return callback(err, outputFiles)
|
||||
} else {
|
||||
// copy all the output files into the new cache directory
|
||||
const results = []
|
||||
return async.mapSeries(
|
||||
outputFiles,
|
||||
function(file, cb) {
|
||||
// don't send dot files as output, express doesn't serve them
|
||||
if (OutputCacheManager._fileIsHidden(file.path)) {
|
||||
logger.debug(
|
||||
{ compileDir, path: file.path },
|
||||
'ignoring dotfile in output'
|
||||
)
|
||||
return cb()
|
||||
}
|
||||
// copy other files into cache directory if valid
|
||||
const newFile = _.clone(file)
|
||||
const [src, dst] = Array.from([
|
||||
Path.join(compileDir, file.path),
|
||||
Path.join(cacheDir, file.path)
|
||||
])
|
||||
return OutputCacheManager._checkFileIsSafe(src, function(
|
||||
err,
|
||||
isSafe
|
||||
) {
|
||||
if (err != null) {
|
||||
return cb(err)
|
||||
}
|
||||
if (!isSafe) {
|
||||
return cb()
|
||||
}
|
||||
return OutputCacheManager._checkIfShouldCopy(src, function(
|
||||
err,
|
||||
shouldCopy
|
||||
) {
|
||||
if (err != null) {
|
||||
return cb(err)
|
||||
}
|
||||
if (!shouldCopy) {
|
||||
return cb()
|
||||
}
|
||||
return OutputCacheManager._copyFile(src, dst, function(err) {
|
||||
if (err != null) {
|
||||
return cb(err)
|
||||
}
|
||||
newFile.build = buildId // attach a build id if we cached the file
|
||||
results.push(newFile)
|
||||
return cb()
|
||||
})
|
||||
})
|
||||
})
|
||||
},
|
||||
function(err) {
|
||||
if (err != null) {
|
||||
// pass back the original files if we encountered *any* error
|
||||
callback(err, outputFiles)
|
||||
// clean up the directory we just created
|
||||
return fse.remove(cacheDir, function(err) {
|
||||
if (err != null) {
|
||||
return logger.error(
|
||||
{ err, dir: cacheDir },
|
||||
'error removing cache dir after failure'
|
||||
)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
// pass back the list of new files in the cache
|
||||
callback(err, results)
|
||||
// let file expiry run in the background, expire all previous files if per-user
|
||||
return OutputCacheManager.expireOutputFiles(cacheRoot, {
|
||||
keep: buildId,
|
||||
limit: perUser ? 1 : null
|
||||
})
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
archiveLogs(outputFiles, compileDir, buildId, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const archiveDir = Path.join(
|
||||
compileDir,
|
||||
OutputCacheManager.ARCHIVE_SUBDIR,
|
||||
buildId
|
||||
)
|
||||
logger.log({ dir: archiveDir }, 'archiving log files for project')
|
||||
return fse.ensureDir(archiveDir, function(err) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return async.mapSeries(
|
||||
outputFiles,
|
||||
function(file, cb) {
|
||||
const [src, dst] = Array.from([
|
||||
Path.join(compileDir, file.path),
|
||||
Path.join(archiveDir, file.path)
|
||||
])
|
||||
return OutputCacheManager._checkFileIsSafe(src, function(
|
||||
err,
|
||||
isSafe
|
||||
) {
|
||||
if (err != null) {
|
||||
return cb(err)
|
||||
}
|
||||
if (!isSafe) {
|
||||
return cb()
|
||||
}
|
||||
return OutputCacheManager._checkIfShouldArchive(src, function(
|
||||
err,
|
||||
shouldArchive
|
||||
) {
|
||||
if (err != null) {
|
||||
return cb(err)
|
||||
}
|
||||
if (!shouldArchive) {
|
||||
return cb()
|
||||
}
|
||||
return OutputCacheManager._copyFile(src, dst, cb)
|
||||
})
|
||||
})
|
||||
},
|
||||
callback
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
expireOutputFiles(cacheRoot, options, callback) {
|
||||
// look in compileDir for build dirs and delete if > N or age of mod time > T
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return fs.readdir(cacheRoot, function(err, results) {
|
||||
if (err != null) {
|
||||
if (err.code === 'ENOENT') {
|
||||
return callback(null)
|
||||
} // cache directory is empty
|
||||
logger.error({ err, project_id: cacheRoot }, 'error clearing cache')
|
||||
return callback(err)
|
||||
}
|
||||
|
||||
const dirs = results.sort().reverse()
|
||||
const currentTime = Date.now()
|
||||
|
||||
const isExpired = function(dir, index) {
|
||||
if ((options != null ? options.keep : undefined) === dir) {
|
||||
return false
|
||||
}
|
||||
// remove any directories over the requested (non-null) limit
|
||||
if (
|
||||
(options != null ? options.limit : undefined) != null &&
|
||||
index > options.limit
|
||||
) {
|
||||
return true
|
||||
}
|
||||
// remove any directories over the hard limit
|
||||
if (index > OutputCacheManager.CACHE_LIMIT) {
|
||||
return true
|
||||
}
|
||||
// we can get the build time from the first part of the directory name DDDD-RRRR
|
||||
// DDDD is date and RRRR is random bytes
|
||||
const dirTime = parseInt(
|
||||
__guard__(dir.split('-'), x => x[0]),
|
||||
16
|
||||
)
|
||||
const age = currentTime - dirTime
|
||||
return age > OutputCacheManager.CACHE_AGE
|
||||
}
|
||||
|
||||
const toRemove = _.filter(dirs, isExpired)
|
||||
|
||||
const removeDir = (dir, cb) =>
|
||||
fse.remove(Path.join(cacheRoot, dir), function(err, result) {
|
||||
logger.log({ cache: cacheRoot, dir }, 'removed expired cache dir')
|
||||
if (err != null) {
|
||||
logger.error({ err, dir }, 'cache remove error')
|
||||
}
|
||||
return cb(err, result)
|
||||
})
|
||||
return async.eachSeries(
|
||||
toRemove,
|
||||
(dir, cb) => removeDir(dir, cb),
|
||||
callback
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
_fileIsHidden(path) {
|
||||
return (path != null ? path.match(/^\.|\/\./) : undefined) != null
|
||||
},
|
||||
|
||||
_checkFileIsSafe(src, callback) {
|
||||
// check if we have a valid file to copy into the cache
|
||||
if (callback == null) {
|
||||
callback = function(error, isSafe) {}
|
||||
}
|
||||
return fs.stat(src, function(err, stats) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
logger.warn(
|
||||
{ err, file: src },
|
||||
'file has disappeared before copying to build cache'
|
||||
)
|
||||
return callback(err, false)
|
||||
} else if (err != null) {
|
||||
// some other problem reading the file
|
||||
logger.error({ err, file: src }, 'stat error for file in cache')
|
||||
return callback(err, false)
|
||||
} else if (!stats.isFile()) {
|
||||
// other filetype - reject it
|
||||
logger.warn(
|
||||
{ src, stat: stats },
|
||||
'nonfile output - refusing to copy to cache'
|
||||
)
|
||||
return callback(null, false)
|
||||
} else {
|
||||
// it's a plain file, ok to copy
|
||||
return callback(null, true)
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_copyFile(src, dst, callback) {
|
||||
// copy output file into the cache
|
||||
return fse.copy(src, dst, function(err) {
|
||||
if ((err != null ? err.code : undefined) === 'ENOENT') {
|
||||
logger.warn(
|
||||
{ err, file: src },
|
||||
'file has disappeared when copying to build cache'
|
||||
)
|
||||
return callback(err, false)
|
||||
} else if (err != null) {
|
||||
logger.error({ err, src, dst }, 'copy error for file in cache')
|
||||
return callback(err)
|
||||
} else {
|
||||
if (
|
||||
Settings.clsi != null ? Settings.clsi.optimiseInDocker : undefined
|
||||
) {
|
||||
// don't run any optimisations on the pdf when they are done
|
||||
// in the docker container
|
||||
return callback()
|
||||
} else {
|
||||
// call the optimiser for the file too
|
||||
return OutputFileOptimiser.optimiseFile(src, dst, callback)
|
||||
}
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_checkIfShouldCopy(src, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(err, shouldCopy) {}
|
||||
}
|
||||
return callback(null, !Path.basename(src).match(/^strace/))
|
||||
},
|
||||
|
||||
_checkIfShouldArchive(src, callback) {
|
||||
let needle
|
||||
if (callback == null) {
|
||||
callback = function(err, shouldCopy) {}
|
||||
}
|
||||
if (Path.basename(src).match(/^strace/)) {
|
||||
return callback(null, true)
|
||||
}
|
||||
if (
|
||||
(Settings.clsi != null ? Settings.clsi.archive_logs : undefined) &&
|
||||
((needle = Path.basename(src)),
|
||||
['output.log', 'output.blg'].includes(needle))
|
||||
) {
|
||||
return callback(null, true)
|
||||
}
|
||||
return callback(null, false)
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
115
app/js/OutputFileFinder.js
Normal file
115
app/js/OutputFileFinder.js
Normal file
@@ -0,0 +1,115 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
no-useless-escape,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputFileFinder
|
||||
const async = require('async')
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
const { spawn } = require('child_process')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
module.exports = OutputFileFinder = {
|
||||
findOutputFiles(resources, directory, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, outputFiles, allFiles) {}
|
||||
}
|
||||
const incomingResources = {}
|
||||
for (const resource of Array.from(resources)) {
|
||||
incomingResources[resource.path] = true
|
||||
}
|
||||
|
||||
return OutputFileFinder._getAllFiles(directory, function(error, allFiles) {
|
||||
if (allFiles == null) {
|
||||
allFiles = []
|
||||
}
|
||||
if (error != null) {
|
||||
logger.err({ err: error }, 'error finding all output files')
|
||||
return callback(error)
|
||||
}
|
||||
const outputFiles = []
|
||||
for (const file of Array.from(allFiles)) {
|
||||
if (!incomingResources[file]) {
|
||||
outputFiles.push({
|
||||
path: file,
|
||||
type: __guard__(file.match(/\.([^\.]+)$/), x => x[1])
|
||||
})
|
||||
}
|
||||
}
|
||||
return callback(null, outputFiles, allFiles)
|
||||
})
|
||||
},
|
||||
|
||||
_getAllFiles(directory, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error, fileList) {}
|
||||
}
|
||||
const callback = function(error, fileList) {
|
||||
_callback(error, fileList)
|
||||
return (_callback = function() {})
|
||||
}
|
||||
|
||||
// don't include clsi-specific files/directories in the output list
|
||||
const EXCLUDE_DIRS = [
|
||||
'-name',
|
||||
'.cache',
|
||||
'-o',
|
||||
'-name',
|
||||
'.archive',
|
||||
'-o',
|
||||
'-name',
|
||||
'.project-*'
|
||||
]
|
||||
const args = [
|
||||
directory,
|
||||
'(',
|
||||
...Array.from(EXCLUDE_DIRS),
|
||||
')',
|
||||
'-prune',
|
||||
'-o',
|
||||
'-type',
|
||||
'f',
|
||||
'-print'
|
||||
]
|
||||
logger.log({ args }, 'running find command')
|
||||
|
||||
const proc = spawn('find', args)
|
||||
let stdout = ''
|
||||
proc.stdout.on('data', chunk => (stdout += chunk.toString()))
|
||||
proc.on('error', callback)
|
||||
return proc.on('close', function(code) {
|
||||
if (code !== 0) {
|
||||
logger.warn(
|
||||
{ directory, code },
|
||||
"find returned error, directory likely doesn't exist"
|
||||
)
|
||||
return callback(null, [])
|
||||
}
|
||||
let fileList = stdout.trim().split('\n')
|
||||
fileList = fileList.map(function(file) {
|
||||
// Strip leading directory
|
||||
let path
|
||||
return (path = Path.relative(directory, file))
|
||||
})
|
||||
return callback(null, fileList)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
104
app/js/OutputFileOptimiser.js
Normal file
104
app/js/OutputFileOptimiser.js
Normal file
@@ -0,0 +1,104 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-undef,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let OutputFileOptimiser
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
const { spawn } = require('child_process')
|
||||
const logger = require('logger-sharelatex')
|
||||
const Metrics = require('./Metrics')
|
||||
const _ = require('underscore')
|
||||
|
||||
module.exports = OutputFileOptimiser = {
|
||||
optimiseFile(src, dst, callback) {
|
||||
// check output file (src) and see if we can optimise it, storing
|
||||
// the result in the build directory (dst)
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
if (src.match(/\/output\.pdf$/)) {
|
||||
return OutputFileOptimiser.checkIfPDFIsOptimised(src, function(
|
||||
err,
|
||||
isOptimised
|
||||
) {
|
||||
if (err != null || isOptimised) {
|
||||
return callback(null)
|
||||
}
|
||||
return OutputFileOptimiser.optimisePDF(src, dst, callback)
|
||||
})
|
||||
} else {
|
||||
return callback(null)
|
||||
}
|
||||
},
|
||||
|
||||
checkIfPDFIsOptimised(file, callback) {
|
||||
const SIZE = 16 * 1024 // check the header of the pdf
|
||||
const result = new Buffer(SIZE)
|
||||
result.fill(0) // prevent leakage of uninitialised buffer
|
||||
return fs.open(file, 'r', function(err, fd) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) =>
|
||||
fs.close(fd, function(errClose) {
|
||||
if (errRead != null) {
|
||||
return callback(errRead)
|
||||
}
|
||||
if (typeof errReadClose !== 'undefined' && errReadClose !== null) {
|
||||
return callback(errClose)
|
||||
}
|
||||
const isOptimised =
|
||||
buffer.toString('ascii').indexOf('/Linearized 1') >= 0
|
||||
return callback(null, isOptimised)
|
||||
})
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
optimisePDF(src, dst, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const tmpOutput = dst + '.opt'
|
||||
const args = ['--linearize', src, tmpOutput]
|
||||
logger.log({ args }, 'running qpdf command')
|
||||
|
||||
const timer = new Metrics.Timer('qpdf')
|
||||
const proc = spawn('qpdf', args)
|
||||
let stdout = ''
|
||||
proc.stdout.on('data', chunk => (stdout += chunk.toString()))
|
||||
callback = _.once(callback) // avoid double call back for error and close event
|
||||
proc.on('error', function(err) {
|
||||
logger.warn({ err, args }, 'qpdf failed')
|
||||
return callback(null)
|
||||
}) // ignore the error
|
||||
return proc.on('close', function(code) {
|
||||
timer.done()
|
||||
if (code !== 0) {
|
||||
logger.warn({ code, args }, 'qpdf returned error')
|
||||
return callback(null) // ignore the error
|
||||
}
|
||||
return fs.rename(tmpOutput, dst, function(err) {
|
||||
if (err != null) {
|
||||
logger.warn(
|
||||
{ tmpOutput, dst },
|
||||
'failed to rename output of qpdf command'
|
||||
)
|
||||
}
|
||||
return callback(null)
|
||||
})
|
||||
})
|
||||
} // ignore the error
|
||||
}
|
||||
163
app/js/ProjectPersistenceManager.js
Normal file
163
app/js/ProjectPersistenceManager.js
Normal file
@@ -0,0 +1,163 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ProjectPersistenceManager
|
||||
const UrlCache = require('./UrlCache')
|
||||
const CompileManager = require('./CompileManager')
|
||||
const db = require('./db')
|
||||
const dbQueue = require('./DbQueue')
|
||||
const async = require('async')
|
||||
const logger = require('logger-sharelatex')
|
||||
const oneDay = 24 * 60 * 60 * 1000
|
||||
const Settings = require('settings-sharelatex')
|
||||
|
||||
module.exports = ProjectPersistenceManager = {
|
||||
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
|
||||
|
||||
markProjectAsJustAccessed(project_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const job = cb =>
|
||||
db.Project.findOrCreate({ where: { project_id } })
|
||||
.spread((project, created) =>
|
||||
project
|
||||
.update({ lastAccessed: new Date() })
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
)
|
||||
.error(cb)
|
||||
return dbQueue.queue.push(job, callback)
|
||||
},
|
||||
|
||||
clearExpiredProjects(callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return ProjectPersistenceManager._findExpiredProjectIds(function(
|
||||
error,
|
||||
project_ids
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
logger.log({ project_ids }, 'clearing expired projects')
|
||||
const jobs = Array.from(project_ids || []).map(project_id =>
|
||||
(project_id => callback =>
|
||||
ProjectPersistenceManager.clearProjectFromCache(project_id, function(
|
||||
err
|
||||
) {
|
||||
if (err != null) {
|
||||
logger.error({ err, project_id }, 'error clearing project')
|
||||
}
|
||||
return callback()
|
||||
}))(project_id)
|
||||
)
|
||||
return async.series(jobs, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return CompileManager.clearExpiredProjects(
|
||||
ProjectPersistenceManager.EXPIRY_TIMEOUT,
|
||||
error => callback()
|
||||
)
|
||||
})
|
||||
})
|
||||
}, // ignore any errors from deleting directories
|
||||
|
||||
clearProject(project_id, user_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
logger.log({ project_id, user_id }, 'clearing project for user')
|
||||
return CompileManager.clearProject(project_id, user_id, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return ProjectPersistenceManager.clearProjectFromCache(
|
||||
project_id,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback()
|
||||
}
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
clearProjectFromCache(project_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
logger.log({ project_id }, 'clearing project from cache')
|
||||
return UrlCache.clearProject(project_id, function(error) {
|
||||
if (error != null) {
|
||||
logger.err({ error, project_id }, 'error clearing project from cache')
|
||||
return callback(error)
|
||||
}
|
||||
return ProjectPersistenceManager._clearProjectFromDatabase(
|
||||
project_id,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{ error, project_id },
|
||||
'error clearing project from database'
|
||||
)
|
||||
}
|
||||
return callback(error)
|
||||
}
|
||||
)
|
||||
})
|
||||
},
|
||||
|
||||
_clearProjectFromDatabase(project_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
logger.log({ project_id }, 'clearing project from database')
|
||||
const job = cb =>
|
||||
db.Project.destroy({ where: { project_id } })
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
return dbQueue.queue.push(job, callback)
|
||||
},
|
||||
|
||||
_findExpiredProjectIds(callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, project_ids) {}
|
||||
}
|
||||
const job = function(cb) {
|
||||
const keepProjectsFrom = new Date(
|
||||
Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT
|
||||
)
|
||||
const q = {}
|
||||
q[db.op.lt] = keepProjectsFrom
|
||||
return db.Project.findAll({ where: { lastAccessed: q } })
|
||||
.then(projects =>
|
||||
cb(
|
||||
null,
|
||||
projects.map(project => project.project_id)
|
||||
)
|
||||
)
|
||||
.error(cb)
|
||||
}
|
||||
|
||||
return dbQueue.queue.push(job, callback)
|
||||
}
|
||||
}
|
||||
|
||||
logger.log(
|
||||
{ EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT },
|
||||
'project assets kept timeout'
|
||||
)
|
||||
217
app/js/RequestParser.js
Normal file
217
app/js/RequestParser.js
Normal file
@@ -0,0 +1,217 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-control-regex,
|
||||
no-throw-literal,
|
||||
no-unused-vars,
|
||||
no-useless-escape,
|
||||
standard/no-callback-literal,
|
||||
valid-typeof,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS205: Consider reworking code to avoid use of IIFEs
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let RequestParser
|
||||
const settings = require('settings-sharelatex')
|
||||
|
||||
module.exports = RequestParser = {
|
||||
VALID_COMPILERS: ['pdflatex', 'latex', 'xelatex', 'lualatex'],
|
||||
MAX_TIMEOUT: 600,
|
||||
|
||||
parse(body, callback) {
|
||||
let resource
|
||||
if (callback == null) {
|
||||
callback = function(error, data) {}
|
||||
}
|
||||
const response = {}
|
||||
|
||||
if (body.compile == null) {
|
||||
return callback('top level object should have a compile attribute')
|
||||
}
|
||||
|
||||
const { compile } = body
|
||||
if (!compile.options) {
|
||||
compile.options = {}
|
||||
}
|
||||
|
||||
try {
|
||||
response.compiler = this._parseAttribute(
|
||||
'compiler',
|
||||
compile.options.compiler,
|
||||
{
|
||||
validValues: this.VALID_COMPILERS,
|
||||
default: 'pdflatex',
|
||||
type: 'string'
|
||||
}
|
||||
)
|
||||
response.timeout = this._parseAttribute(
|
||||
'timeout',
|
||||
compile.options.timeout,
|
||||
{
|
||||
default: RequestParser.MAX_TIMEOUT,
|
||||
type: 'number'
|
||||
}
|
||||
)
|
||||
response.imageName = this._parseAttribute(
|
||||
'imageName',
|
||||
compile.options.imageName,
|
||||
{ type: 'string' }
|
||||
)
|
||||
response.draft = this._parseAttribute('draft', compile.options.draft, {
|
||||
default: false,
|
||||
type: 'boolean'
|
||||
})
|
||||
response.check = this._parseAttribute('check', compile.options.check, {
|
||||
type: 'string'
|
||||
})
|
||||
response.flags = this._parseAttribute('flags', compile.options.flags, {
|
||||
default: [],
|
||||
type: 'object'
|
||||
})
|
||||
|
||||
// The syncType specifies whether the request contains all
|
||||
// resources (full) or only those resources to be updated
|
||||
// in-place (incremental).
|
||||
response.syncType = this._parseAttribute(
|
||||
'syncType',
|
||||
compile.options.syncType,
|
||||
{
|
||||
validValues: ['full', 'incremental'],
|
||||
type: 'string'
|
||||
}
|
||||
)
|
||||
|
||||
// The syncState is an identifier passed in with the request
|
||||
// which has the property that it changes when any resource is
|
||||
// added, deleted, moved or renamed.
|
||||
//
|
||||
// on syncType full the syncState identifier is passed in and
|
||||
// stored
|
||||
//
|
||||
// on syncType incremental the syncState identifier must match
|
||||
// the stored value
|
||||
response.syncState = this._parseAttribute(
|
||||
'syncState',
|
||||
compile.options.syncState,
|
||||
{ type: 'string' }
|
||||
)
|
||||
|
||||
if (response.timeout > RequestParser.MAX_TIMEOUT) {
|
||||
response.timeout = RequestParser.MAX_TIMEOUT
|
||||
}
|
||||
response.timeout = response.timeout * 1000 // milliseconds
|
||||
|
||||
response.resources = (() => {
|
||||
const result = []
|
||||
for (resource of Array.from(compile.resources || [])) {
|
||||
result.push(this._parseResource(resource))
|
||||
}
|
||||
return result
|
||||
})()
|
||||
|
||||
const rootResourcePath = this._parseAttribute(
|
||||
'rootResourcePath',
|
||||
compile.rootResourcePath,
|
||||
{
|
||||
default: 'main.tex',
|
||||
type: 'string'
|
||||
}
|
||||
)
|
||||
const originalRootResourcePath = rootResourcePath
|
||||
const sanitizedRootResourcePath = RequestParser._sanitizePath(
|
||||
rootResourcePath
|
||||
)
|
||||
response.rootResourcePath = RequestParser._checkPath(
|
||||
sanitizedRootResourcePath
|
||||
)
|
||||
|
||||
for (resource of Array.from(response.resources)) {
|
||||
if (resource.path === originalRootResourcePath) {
|
||||
resource.path = sanitizedRootResourcePath
|
||||
}
|
||||
}
|
||||
} catch (error1) {
|
||||
const error = error1
|
||||
return callback(error)
|
||||
}
|
||||
|
||||
return callback(null, response)
|
||||
},
|
||||
|
||||
_parseResource(resource) {
|
||||
let modified
|
||||
if (resource.path == null || typeof resource.path !== 'string') {
|
||||
throw 'all resources should have a path attribute'
|
||||
}
|
||||
|
||||
if (resource.modified != null) {
|
||||
modified = new Date(resource.modified)
|
||||
if (isNaN(modified.getTime())) {
|
||||
throw `resource modified date could not be understood: ${resource.modified}`
|
||||
}
|
||||
}
|
||||
|
||||
if (resource.url == null && resource.content == null) {
|
||||
throw 'all resources should have either a url or content attribute'
|
||||
}
|
||||
if (resource.content != null && typeof resource.content !== 'string') {
|
||||
throw 'content attribute should be a string'
|
||||
}
|
||||
if (resource.url != null && typeof resource.url !== 'string') {
|
||||
throw 'url attribute should be a string'
|
||||
}
|
||||
|
||||
return {
|
||||
path: resource.path,
|
||||
modified,
|
||||
url: resource.url,
|
||||
content: resource.content
|
||||
}
|
||||
},
|
||||
|
||||
_parseAttribute(name, attribute, options) {
|
||||
if (attribute != null) {
|
||||
if (options.validValues != null) {
|
||||
if (options.validValues.indexOf(attribute) === -1) {
|
||||
throw `${name} attribute should be one of: ${options.validValues.join(
|
||||
', '
|
||||
)}`
|
||||
}
|
||||
}
|
||||
if (options.type != null) {
|
||||
if (typeof attribute !== options.type) {
|
||||
throw `${name} attribute should be a ${options.type}`
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (options.default != null) {
|
||||
return options.default
|
||||
}
|
||||
}
|
||||
return attribute
|
||||
},
|
||||
|
||||
_sanitizePath(path) {
|
||||
// See http://php.net/manual/en/function.escapeshellcmd.php
|
||||
return path.replace(
|
||||
/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g,
|
||||
''
|
||||
)
|
||||
},
|
||||
|
||||
_checkPath(path) {
|
||||
// check that the request does not use a relative path
|
||||
for (const dir of Array.from(path.split('/'))) {
|
||||
if (dir === '..') {
|
||||
throw 'relative path in root resource'
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
}
|
||||
154
app/js/ResourceStateManager.js
Normal file
154
app/js/ResourceStateManager.js
Normal file
@@ -0,0 +1,154 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS201: Simplify complex destructure assignments
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ResourceStateManager
|
||||
const Path = require('path')
|
||||
const fs = require('fs')
|
||||
const logger = require('logger-sharelatex')
|
||||
const settings = require('settings-sharelatex')
|
||||
const Errors = require('./Errors')
|
||||
const SafeReader = require('./SafeReader')
|
||||
|
||||
module.exports = ResourceStateManager = {
|
||||
// The sync state is an identifier which must match for an
|
||||
// incremental update to be allowed.
|
||||
//
|
||||
// The initial value is passed in and stored on a full
|
||||
// compile, along with the list of resources..
|
||||
//
|
||||
// Subsequent incremental compiles must come with the same value - if
|
||||
// not they will be rejected with a 409 Conflict response. The
|
||||
// previous list of resources is returned.
|
||||
//
|
||||
// An incremental compile can only update existing files with new
|
||||
// content. The sync state identifier must change if any docs or
|
||||
// files are moved, added, deleted or renamed.
|
||||
|
||||
SYNC_STATE_FILE: '.project-sync-state',
|
||||
SYNC_STATE_MAX_SIZE: 128 * 1024,
|
||||
|
||||
saveProjectState(state, resources, basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE)
|
||||
if (state == null) {
|
||||
// remove the file if no state passed in
|
||||
logger.log({ state, basePath }, 'clearing sync state')
|
||||
return fs.unlink(stateFile, function(err) {
|
||||
if (err != null && err.code !== 'ENOENT') {
|
||||
return callback(err)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
} else {
|
||||
logger.log({ state, basePath }, 'writing sync state')
|
||||
const resourceList = Array.from(resources).map(resource => resource.path)
|
||||
return fs.writeFile(
|
||||
stateFile,
|
||||
[...Array.from(resourceList), `stateHash:${state}`].join('\n'),
|
||||
callback
|
||||
)
|
||||
}
|
||||
},
|
||||
|
||||
checkProjectStateMatches(state, basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, resources) {}
|
||||
}
|
||||
const stateFile = Path.join(basePath, this.SYNC_STATE_FILE)
|
||||
const size = this.SYNC_STATE_MAX_SIZE
|
||||
return SafeReader.readFile(stateFile, size, 'utf8', function(
|
||||
err,
|
||||
result,
|
||||
bytesRead
|
||||
) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
if (bytesRead === size) {
|
||||
logger.error(
|
||||
{ file: stateFile, size, bytesRead },
|
||||
'project state file truncated'
|
||||
)
|
||||
}
|
||||
const array =
|
||||
__guard__(result != null ? result.toString() : undefined, x =>
|
||||
x.split('\n')
|
||||
) || []
|
||||
const adjustedLength = Math.max(array.length, 1)
|
||||
const resourceList = array.slice(0, adjustedLength - 1)
|
||||
const oldState = array[adjustedLength - 1]
|
||||
const newState = `stateHash:${state}`
|
||||
logger.log(
|
||||
{ state, oldState, basePath, stateMatches: newState === oldState },
|
||||
'checking sync state'
|
||||
)
|
||||
if (newState !== oldState) {
|
||||
return callback(
|
||||
new Errors.FilesOutOfSyncError('invalid state for incremental update')
|
||||
)
|
||||
} else {
|
||||
const resources = Array.from(resourceList).map(path => ({ path }))
|
||||
return callback(null, resources)
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
checkResourceFiles(resources, allFiles, basePath, callback) {
|
||||
// check the paths are all relative to current directory
|
||||
let file
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
for (file of Array.from(resources || [])) {
|
||||
for (const dir of Array.from(
|
||||
__guard__(file != null ? file.path : undefined, x => x.split('/'))
|
||||
)) {
|
||||
if (dir === '..') {
|
||||
return callback(new Error('relative path in resource file list'))
|
||||
}
|
||||
}
|
||||
}
|
||||
// check if any of the input files are not present in list of files
|
||||
const seenFile = {}
|
||||
for (file of Array.from(allFiles)) {
|
||||
seenFile[file] = true
|
||||
}
|
||||
const missingFiles = Array.from(resources)
|
||||
.filter(resource => !seenFile[resource.path])
|
||||
.map(resource => resource.path)
|
||||
if ((missingFiles != null ? missingFiles.length : undefined) > 0) {
|
||||
logger.err(
|
||||
{ missingFiles, basePath, allFiles, resources },
|
||||
'missing input files for project'
|
||||
)
|
||||
return callback(
|
||||
new Errors.FilesOutOfSyncError(
|
||||
'resource files missing in incremental update'
|
||||
)
|
||||
)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
352
app/js/ResourceWriter.js
Normal file
352
app/js/ResourceWriter.js
Normal file
@@ -0,0 +1,352 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
no-useless-escape,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ResourceWriter
|
||||
const UrlCache = require('./UrlCache')
|
||||
const Path = require('path')
|
||||
const fs = require('fs')
|
||||
const async = require('async')
|
||||
const OutputFileFinder = require('./OutputFileFinder')
|
||||
const ResourceStateManager = require('./ResourceStateManager')
|
||||
const Metrics = require('./Metrics')
|
||||
const logger = require('logger-sharelatex')
|
||||
const settings = require('settings-sharelatex')
|
||||
|
||||
const parallelFileDownloads = settings.parallelFileDownloads || 1
|
||||
|
||||
module.exports = ResourceWriter = {
|
||||
syncResourcesToDisk(request, basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, resourceList) {}
|
||||
}
|
||||
if (request.syncType === 'incremental') {
|
||||
logger.log(
|
||||
{ project_id: request.project_id, user_id: request.user_id },
|
||||
'incremental sync'
|
||||
)
|
||||
return ResourceStateManager.checkProjectStateMatches(
|
||||
request.syncState,
|
||||
basePath,
|
||||
function(error, resourceList) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return ResourceWriter._removeExtraneousFiles(
|
||||
resourceList,
|
||||
basePath,
|
||||
function(error, outputFiles, allFiles) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return ResourceStateManager.checkResourceFiles(
|
||||
resourceList,
|
||||
allFiles,
|
||||
basePath,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return ResourceWriter.saveIncrementalResourcesToDisk(
|
||||
request.project_id,
|
||||
request.resources,
|
||||
basePath,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null, resourceList)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
} else {
|
||||
logger.log(
|
||||
{ project_id: request.project_id, user_id: request.user_id },
|
||||
'full sync'
|
||||
)
|
||||
return this.saveAllResourcesToDisk(
|
||||
request.project_id,
|
||||
request.resources,
|
||||
basePath,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return ResourceStateManager.saveProjectState(
|
||||
request.syncState,
|
||||
request.resources,
|
||||
basePath,
|
||||
function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null, request.resources)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
},
|
||||
|
||||
saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return this._createDirectory(basePath, error => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const jobs = Array.from(resources).map(resource =>
|
||||
(resource => {
|
||||
return callback =>
|
||||
this._writeResourceToDisk(project_id, resource, basePath, callback)
|
||||
})(resource)
|
||||
)
|
||||
return async.parallelLimit(jobs, parallelFileDownloads, callback)
|
||||
})
|
||||
},
|
||||
|
||||
saveAllResourcesToDisk(project_id, resources, basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return this._createDirectory(basePath, error => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return this._removeExtraneousFiles(resources, basePath, error => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const jobs = Array.from(resources).map(resource =>
|
||||
(resource => {
|
||||
return callback =>
|
||||
this._writeResourceToDisk(
|
||||
project_id,
|
||||
resource,
|
||||
basePath,
|
||||
callback
|
||||
)
|
||||
})(resource)
|
||||
)
|
||||
return async.parallelLimit(jobs, parallelFileDownloads, callback)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
_createDirectory(basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return fs.mkdir(basePath, function(err) {
|
||||
if (err != null) {
|
||||
if (err.code === 'EEXIST') {
|
||||
return callback()
|
||||
} else {
|
||||
logger.log({ err, dir: basePath }, 'error creating directory')
|
||||
return callback(err)
|
||||
}
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_removeExtraneousFiles(resources, basePath, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error, outputFiles, allFiles) {}
|
||||
}
|
||||
const timer = new Metrics.Timer('unlink-output-files')
|
||||
const callback = function(error, ...result) {
|
||||
timer.done()
|
||||
return _callback(error, ...Array.from(result))
|
||||
}
|
||||
|
||||
return OutputFileFinder.findOutputFiles(resources, basePath, function(
|
||||
error,
|
||||
outputFiles,
|
||||
allFiles
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
|
||||
const jobs = []
|
||||
for (const file of Array.from(outputFiles || [])) {
|
||||
;(function(file) {
|
||||
const { path } = file
|
||||
let should_delete = true
|
||||
if (
|
||||
path.match(/^output\./) ||
|
||||
path.match(/\.aux$/) ||
|
||||
path.match(/^cache\//)
|
||||
) {
|
||||
// knitr cache
|
||||
should_delete = false
|
||||
}
|
||||
if (path.match(/^output-.*/)) {
|
||||
// Tikz cached figures (default case)
|
||||
should_delete = false
|
||||
}
|
||||
if (path.match(/\.(pdf|dpth|md5)$/)) {
|
||||
// Tikz cached figures (by extension)
|
||||
should_delete = false
|
||||
}
|
||||
if (
|
||||
path.match(/\.(pygtex|pygstyle)$/) ||
|
||||
path.match(/(^|\/)_minted-[^\/]+\//)
|
||||
) {
|
||||
// minted files/directory
|
||||
should_delete = false
|
||||
}
|
||||
if (
|
||||
path.match(/\.md\.tex$/) ||
|
||||
path.match(/(^|\/)_markdown_[^\/]+\//)
|
||||
) {
|
||||
// markdown files/directory
|
||||
should_delete = false
|
||||
}
|
||||
if (path.match(/-eps-converted-to\.pdf$/)) {
|
||||
// Epstopdf generated files
|
||||
should_delete = false
|
||||
}
|
||||
if (
|
||||
path === 'output.pdf' ||
|
||||
path === 'output.dvi' ||
|
||||
path === 'output.log' ||
|
||||
path === 'output.xdv'
|
||||
) {
|
||||
should_delete = true
|
||||
}
|
||||
if (path === 'output.tex') {
|
||||
// created by TikzManager if present in output files
|
||||
should_delete = true
|
||||
}
|
||||
if (should_delete) {
|
||||
return jobs.push(callback =>
|
||||
ResourceWriter._deleteFileIfNotDirectory(
|
||||
Path.join(basePath, path),
|
||||
callback
|
||||
)
|
||||
)
|
||||
}
|
||||
})(file)
|
||||
}
|
||||
|
||||
return async.series(jobs, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null, outputFiles, allFiles)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
_deleteFileIfNotDirectory(path, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return fs.stat(path, function(error, stat) {
|
||||
if (error != null && error.code === 'ENOENT') {
|
||||
return callback()
|
||||
} else if (error != null) {
|
||||
logger.err(
|
||||
{ err: error, path },
|
||||
'error stating file in deleteFileIfNotDirectory'
|
||||
)
|
||||
return callback(error)
|
||||
} else if (stat.isFile()) {
|
||||
return fs.unlink(path, function(error) {
|
||||
if (error != null) {
|
||||
logger.err(
|
||||
{ err: error, path },
|
||||
'error removing file in deleteFileIfNotDirectory'
|
||||
)
|
||||
return callback(error)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_writeResourceToDisk(project_id, resource, basePath, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return ResourceWriter.checkPath(basePath, resource.path, function(
|
||||
error,
|
||||
path
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return fs.mkdir(Path.dirname(path), { recursive: true }, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
// TODO: Don't overwrite file if it hasn't been modified
|
||||
if (resource.url != null) {
|
||||
return UrlCache.downloadUrlToFile(
|
||||
project_id,
|
||||
resource.url,
|
||||
path,
|
||||
resource.modified,
|
||||
function(err) {
|
||||
if (err != null) {
|
||||
logger.err(
|
||||
{
|
||||
err,
|
||||
project_id,
|
||||
path,
|
||||
resource_url: resource.url,
|
||||
modified: resource.modified
|
||||
},
|
||||
'error downloading file for resources'
|
||||
)
|
||||
Metrics.inc('download-failed')
|
||||
}
|
||||
return callback()
|
||||
}
|
||||
) // try and continue compiling even if http resource can not be downloaded at this time
|
||||
} else {
|
||||
const process = require('process')
|
||||
fs.writeFile(path, resource.content, callback)
|
||||
try {
|
||||
let result
|
||||
return (result = fs.lstatSync(path))
|
||||
} catch (e) {}
|
||||
}
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
checkPath(basePath, resourcePath, callback) {
|
||||
const path = Path.normalize(Path.join(basePath, resourcePath))
|
||||
if (path.slice(0, basePath.length + 1) !== basePath + '/') {
|
||||
return callback(new Error('resource path is outside root directory'))
|
||||
} else {
|
||||
return callback(null, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
60
app/js/SafeReader.js
Normal file
60
app/js/SafeReader.js
Normal file
@@ -0,0 +1,60 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let SafeReader
|
||||
const fs = require('fs')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
module.exports = SafeReader = {
|
||||
// safely read up to size bytes from a file and return result as a
|
||||
// string
|
||||
|
||||
readFile(file, size, encoding, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, result) {}
|
||||
}
|
||||
return fs.open(file, 'r', function(err, fd) {
|
||||
if (err != null && err.code === 'ENOENT') {
|
||||
return callback()
|
||||
}
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
|
||||
// safely return always closing the file
|
||||
const callbackWithClose = (err, ...result) =>
|
||||
fs.close(fd, function(err1) {
|
||||
if (err != null) {
|
||||
return callback(err)
|
||||
}
|
||||
if (err1 != null) {
|
||||
return callback(err1)
|
||||
}
|
||||
return callback(null, ...Array.from(result))
|
||||
})
|
||||
const buff = new Buffer(size, 0) // fill with zeros
|
||||
return fs.read(fd, buff, 0, buff.length, 0, function(
|
||||
err,
|
||||
bytesRead,
|
||||
buffer
|
||||
) {
|
||||
if (err != null) {
|
||||
return callbackWithClose(err)
|
||||
}
|
||||
const result = buffer.toString(encoding, 0, bytesRead)
|
||||
return callbackWithClose(null, result, bytesRead)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
94
app/js/StaticServerForbidSymlinks.js
Normal file
94
app/js/StaticServerForbidSymlinks.js
Normal file
@@ -0,0 +1,94 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
no-cond-assign,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS103: Rewrite code to no longer use __guard__
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let ForbidSymlinks
|
||||
const Path = require('path')
|
||||
const fs = require('fs')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const logger = require('logger-sharelatex')
|
||||
const url = require('url')
|
||||
|
||||
module.exports = ForbidSymlinks = function(staticFn, root, options) {
|
||||
const expressStatic = staticFn(root, options)
|
||||
const basePath = Path.resolve(root)
|
||||
return function(req, res, next) {
|
||||
let file, project_id, result
|
||||
const path = __guard__(url.parse(req.url), x => x.pathname)
|
||||
// check that the path is of the form /project_id_or_name/path/to/file.log
|
||||
if ((result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/))) {
|
||||
project_id = result[1]
|
||||
file = result[2]
|
||||
} else {
|
||||
logger.warn({ path }, 'unrecognized file request')
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
// check that the file does not use a relative path
|
||||
for (const dir of Array.from(file.split('/'))) {
|
||||
if (dir === '..') {
|
||||
logger.warn({ path }, 'attempt to use a relative path')
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
}
|
||||
// check that the requested path is normalized
|
||||
const requestedFsPath = `${basePath}/${project_id}/${file}`
|
||||
if (requestedFsPath !== Path.normalize(requestedFsPath)) {
|
||||
logger.error(
|
||||
{ path: requestedFsPath },
|
||||
'requestedFsPath is not normalized'
|
||||
)
|
||||
return res.sendStatus(404)
|
||||
}
|
||||
// check that the requested path is not a symlink
|
||||
return fs.realpath(requestedFsPath, function(err, realFsPath) {
|
||||
if (err != null) {
|
||||
if (err.code === 'ENOENT') {
|
||||
return res.sendStatus(404)
|
||||
} else {
|
||||
logger.error(
|
||||
{
|
||||
err,
|
||||
requestedFsPath,
|
||||
realFsPath,
|
||||
path: req.params[0],
|
||||
project_id: req.params.project_id
|
||||
},
|
||||
'error checking file access'
|
||||
)
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
} else if (requestedFsPath !== realFsPath) {
|
||||
logger.warn(
|
||||
{
|
||||
requestedFsPath,
|
||||
realFsPath,
|
||||
path: req.params[0],
|
||||
project_id: req.params.project_id
|
||||
},
|
||||
'trying to access a different file (symlink), aborting'
|
||||
)
|
||||
return res.sendStatus(404)
|
||||
} else {
|
||||
return expressStatic(req, res, next)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function __guard__(value, transform) {
|
||||
return typeof value !== 'undefined' && value !== null
|
||||
? transform(value)
|
||||
: undefined
|
||||
}
|
||||
94
app/js/TikzManager.js
Normal file
94
app/js/TikzManager.js
Normal file
@@ -0,0 +1,94 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-unused-vars,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let TikzManager
|
||||
const fs = require('fs')
|
||||
const Path = require('path')
|
||||
const ResourceWriter = require('./ResourceWriter')
|
||||
const SafeReader = require('./SafeReader')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
// for \tikzexternalize or pstool to work the main file needs to match the
|
||||
// jobname. Since we set the -jobname to output, we have to create a
|
||||
// copy of the main file as 'output.tex'.
|
||||
|
||||
module.exports = TikzManager = {
|
||||
checkMainFile(compileDir, mainFile, resources, callback) {
|
||||
// if there's already an output.tex file, we don't want to touch it
|
||||
if (callback == null) {
|
||||
callback = function(error, needsMainFile) {}
|
||||
}
|
||||
for (const resource of Array.from(resources)) {
|
||||
if (resource.path === 'output.tex') {
|
||||
logger.log({ compileDir, mainFile }, 'output.tex already in resources')
|
||||
return callback(null, false)
|
||||
}
|
||||
}
|
||||
// if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
|
||||
return ResourceWriter.checkPath(compileDir, mainFile, function(
|
||||
error,
|
||||
path
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return SafeReader.readFile(path, 65536, 'utf8', function(error, content) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const usesTikzExternalize =
|
||||
(content != null
|
||||
? content.indexOf('\\tikzexternalize')
|
||||
: undefined) >= 0
|
||||
const usesPsTool =
|
||||
(content != null ? content.indexOf('{pstool}') : undefined) >= 0
|
||||
logger.log(
|
||||
{ compileDir, mainFile, usesTikzExternalize, usesPsTool },
|
||||
'checked for packages needing main file as output.tex'
|
||||
)
|
||||
const needsMainFile = usesTikzExternalize || usesPsTool
|
||||
return callback(null, needsMainFile)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
injectOutputFile(compileDir, mainFile, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return ResourceWriter.checkPath(compileDir, mainFile, function(
|
||||
error,
|
||||
path
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return fs.readFile(path, 'utf8', function(error, content) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
logger.log(
|
||||
{ compileDir, mainFile },
|
||||
'copied file to output.tex as project uses packages which require it'
|
||||
)
|
||||
// use wx flag to ensure that output file does not already exist
|
||||
return fs.writeFile(
|
||||
Path.join(compileDir, 'output.tex'),
|
||||
content,
|
||||
{ flag: 'wx' },
|
||||
callback
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
278
app/js/UrlCache.js
Normal file
278
app/js/UrlCache.js
Normal file
@@ -0,0 +1,278 @@
|
||||
/* eslint-disable
|
||||
camelcase,
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS101: Remove unnecessary use of Array.from
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlCache
|
||||
const db = require('./db')
|
||||
const dbQueue = require('./DbQueue')
|
||||
const UrlFetcher = require('./UrlFetcher')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const crypto = require('crypto')
|
||||
const fs = require('fs')
|
||||
const logger = require('logger-sharelatex')
|
||||
const async = require('async')
|
||||
|
||||
module.exports = UrlCache = {
|
||||
downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return UrlCache._ensureUrlIsInCache(
|
||||
project_id,
|
||||
url,
|
||||
lastModified,
|
||||
(error, pathToCachedUrl) => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return UrlCache._copyFile(pathToCachedUrl, destPath, function(error) {
|
||||
if (error != null) {
|
||||
return UrlCache._clearUrlDetails(project_id, url, () =>
|
||||
callback(error)
|
||||
)
|
||||
} else {
|
||||
return callback(error)
|
||||
}
|
||||
})
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
clearProject(project_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return UrlCache._findAllUrlsInProject(project_id, function(error, urls) {
|
||||
logger.log(
|
||||
{ project_id, url_count: urls.length },
|
||||
'clearing project URLs'
|
||||
)
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
const jobs = Array.from(urls || []).map(url =>
|
||||
(url => callback =>
|
||||
UrlCache._clearUrlFromCache(project_id, url, function(error) {
|
||||
if (error != null) {
|
||||
logger.error(
|
||||
{ err: error, project_id, url },
|
||||
'error clearing project URL'
|
||||
)
|
||||
}
|
||||
return callback()
|
||||
}))(url)
|
||||
)
|
||||
return async.series(jobs, callback)
|
||||
})
|
||||
},
|
||||
|
||||
_ensureUrlIsInCache(project_id, url, lastModified, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, pathOnDisk) {}
|
||||
}
|
||||
if (lastModified != null) {
|
||||
// MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
|
||||
// So round down to seconds
|
||||
lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000)
|
||||
}
|
||||
return UrlCache._doesUrlNeedDownloading(
|
||||
project_id,
|
||||
url,
|
||||
lastModified,
|
||||
(error, needsDownloading) => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
if (needsDownloading) {
|
||||
logger.log({ url, lastModified }, 'downloading URL')
|
||||
return UrlFetcher.pipeUrlToFile(
|
||||
url,
|
||||
UrlCache._cacheFilePathForUrl(project_id, url),
|
||||
error => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return UrlCache._updateOrCreateUrlDetails(
|
||||
project_id,
|
||||
url,
|
||||
lastModified,
|
||||
error => {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(
|
||||
null,
|
||||
UrlCache._cacheFilePathForUrl(project_id, url)
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
} else {
|
||||
logger.log({ url, lastModified }, 'URL is up to date in cache')
|
||||
return callback(null, UrlCache._cacheFilePathForUrl(project_id, url))
|
||||
}
|
||||
}
|
||||
)
|
||||
},
|
||||
|
||||
_doesUrlNeedDownloading(project_id, url, lastModified, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, needsDownloading) {}
|
||||
}
|
||||
if (lastModified == null) {
|
||||
return callback(null, true)
|
||||
}
|
||||
return UrlCache._findUrlDetails(project_id, url, function(
|
||||
error,
|
||||
urlDetails
|
||||
) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
if (
|
||||
urlDetails == null ||
|
||||
urlDetails.lastModified == null ||
|
||||
urlDetails.lastModified.getTime() < lastModified.getTime()
|
||||
) {
|
||||
return callback(null, true)
|
||||
} else {
|
||||
return callback(null, false)
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_cacheFileNameForUrl(project_id, url) {
|
||||
return (
|
||||
project_id +
|
||||
':' +
|
||||
crypto
|
||||
.createHash('md5')
|
||||
.update(url)
|
||||
.digest('hex')
|
||||
)
|
||||
},
|
||||
|
||||
_cacheFilePathForUrl(project_id, url) {
|
||||
return `${Settings.path.clsiCacheDir}/${UrlCache._cacheFileNameForUrl(
|
||||
project_id,
|
||||
url
|
||||
)}`
|
||||
},
|
||||
|
||||
_copyFile(from, to, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error) {}
|
||||
}
|
||||
const callbackOnce = function(error) {
|
||||
if (error != null) {
|
||||
logger.error({ err: error, from, to }, 'error copying file from cache')
|
||||
}
|
||||
_callback(error)
|
||||
return (_callback = function() {})
|
||||
}
|
||||
const writeStream = fs.createWriteStream(to)
|
||||
const readStream = fs.createReadStream(from)
|
||||
writeStream.on('error', callbackOnce)
|
||||
readStream.on('error', callbackOnce)
|
||||
writeStream.on('close', callbackOnce)
|
||||
return writeStream.on('open', () => readStream.pipe(writeStream))
|
||||
},
|
||||
|
||||
_clearUrlFromCache(project_id, url, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return UrlCache._clearUrlDetails(project_id, url, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return UrlCache._deleteUrlCacheFromDisk(project_id, url, function(error) {
|
||||
if (error != null) {
|
||||
return callback(error)
|
||||
}
|
||||
return callback(null)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
_deleteUrlCacheFromDisk(project_id, url, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function(
|
||||
error
|
||||
) {
|
||||
if (error != null && error.code !== 'ENOENT') {
|
||||
// no error if the file isn't present
|
||||
return callback(error)
|
||||
} else {
|
||||
return callback()
|
||||
}
|
||||
})
|
||||
},
|
||||
|
||||
_findUrlDetails(project_id, url, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, urlDetails) {}
|
||||
}
|
||||
const job = cb =>
|
||||
db.UrlCache.findOne({ where: { url, project_id } })
|
||||
.then(urlDetails => cb(null, urlDetails))
|
||||
.error(cb)
|
||||
return dbQueue.queue.push(job, callback)
|
||||
},
|
||||
|
||||
_updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const job = cb =>
|
||||
db.UrlCache.findOrCreate({ where: { url, project_id } })
|
||||
.spread((urlDetails, created) =>
|
||||
urlDetails
|
||||
.update({ lastModified })
|
||||
.then(() => cb())
|
||||
.error(cb)
|
||||
)
|
||||
.error(cb)
|
||||
return dbQueue.queue.push(job, callback)
|
||||
},
|
||||
|
||||
_clearUrlDetails(project_id, url, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error) {}
|
||||
}
|
||||
const job = cb =>
|
||||
db.UrlCache.destroy({ where: { url, project_id } })
|
||||
.then(() => cb(null))
|
||||
.error(cb)
|
||||
return dbQueue.queue.push(job, callback)
|
||||
},
|
||||
|
||||
_findAllUrlsInProject(project_id, callback) {
|
||||
if (callback == null) {
|
||||
callback = function(error, urls) {}
|
||||
}
|
||||
const job = cb =>
|
||||
db.UrlCache.findAll({ where: { project_id } })
|
||||
.then(urlEntries =>
|
||||
cb(
|
||||
null,
|
||||
urlEntries.map(entry => entry.url)
|
||||
)
|
||||
)
|
||||
.error(cb)
|
||||
return dbQueue.queue.push(job, callback)
|
||||
}
|
||||
}
|
||||
120
app/js/UrlFetcher.js
Normal file
120
app/js/UrlFetcher.js
Normal file
@@ -0,0 +1,120 @@
|
||||
/* eslint-disable
|
||||
handle-callback-err,
|
||||
no-return-assign,
|
||||
no-unused-vars,
|
||||
node/no-deprecated-api,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* DS207: Consider shorter variations of null checks
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
let UrlFetcher
|
||||
const request = require('request').defaults({ jar: false })
|
||||
const fs = require('fs')
|
||||
const logger = require('logger-sharelatex')
|
||||
const settings = require('settings-sharelatex')
|
||||
const URL = require('url')
|
||||
|
||||
const oneMinute = 60 * 1000
|
||||
|
||||
module.exports = UrlFetcher = {
|
||||
pipeUrlToFile(url, filePath, _callback) {
|
||||
if (_callback == null) {
|
||||
_callback = function(error) {}
|
||||
}
|
||||
const callbackOnce = function(error) {
|
||||
if (timeoutHandler != null) {
|
||||
clearTimeout(timeoutHandler)
|
||||
}
|
||||
_callback(error)
|
||||
return (_callback = function() {})
|
||||
}
|
||||
|
||||
if (settings.filestoreDomainOveride != null) {
|
||||
const p = URL.parse(url).path
|
||||
url = `${settings.filestoreDomainOveride}${p}`
|
||||
}
|
||||
var timeoutHandler = setTimeout(
|
||||
function() {
|
||||
timeoutHandler = null
|
||||
logger.error({ url, filePath }, 'Timed out downloading file to cache')
|
||||
return callbackOnce(
|
||||
new Error(`Timed out downloading file to cache ${url}`)
|
||||
)
|
||||
},
|
||||
// FIXME: maybe need to close fileStream here
|
||||
3 * oneMinute
|
||||
)
|
||||
|
||||
logger.log({ url, filePath }, 'started downloading url to cache')
|
||||
const urlStream = request.get({ url, timeout: oneMinute })
|
||||
urlStream.pause() // stop data flowing until we are ready
|
||||
|
||||
// attach handlers before setting up pipes
|
||||
urlStream.on('error', function(error) {
|
||||
logger.error({ err: error, url, filePath }, 'error downloading url')
|
||||
return callbackOnce(
|
||||
error || new Error(`Something went wrong downloading the URL ${url}`)
|
||||
)
|
||||
})
|
||||
|
||||
urlStream.on('end', () =>
|
||||
logger.log({ url, filePath }, 'finished downloading file into cache')
|
||||
)
|
||||
|
||||
return urlStream.on('response', function(res) {
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||||
const fileStream = fs.createWriteStream(filePath)
|
||||
|
||||
// attach handlers before setting up pipes
|
||||
fileStream.on('error', function(error) {
|
||||
logger.error(
|
||||
{ err: error, url, filePath },
|
||||
'error writing file into cache'
|
||||
)
|
||||
return fs.unlink(filePath, function(err) {
|
||||
if (err != null) {
|
||||
logger.err({ err, filePath }, 'error deleting file from cache')
|
||||
}
|
||||
return callbackOnce(error)
|
||||
})
|
||||
})
|
||||
|
||||
fileStream.on('finish', function() {
|
||||
logger.log({ url, filePath }, 'finished writing file into cache')
|
||||
return callbackOnce()
|
||||
})
|
||||
|
||||
fileStream.on('pipe', () =>
|
||||
logger.log({ url, filePath }, 'piping into filestream')
|
||||
)
|
||||
|
||||
urlStream.pipe(fileStream)
|
||||
return urlStream.resume() // now we are ready to handle the data
|
||||
} else {
|
||||
logger.error(
|
||||
{ statusCode: res.statusCode, url, filePath },
|
||||
'unexpected status code downloading url to cache'
|
||||
)
|
||||
// https://nodejs.org/api/http.html#http_class_http_clientrequest
|
||||
// If you add a 'response' event handler, then you must consume
|
||||
// the data from the response object, either by calling
|
||||
// response.read() whenever there is a 'readable' event, or by
|
||||
// adding a 'data' handler, or by calling the .resume()
|
||||
// method. Until the data is consumed, the 'end' event will not
|
||||
// fire. Also, until the data is read it will consume memory
|
||||
// that can eventually lead to a 'process out of memory' error.
|
||||
urlStream.resume() // discard the data
|
||||
return callbackOnce(
|
||||
new Error(
|
||||
`URL returned non-success status code: ${res.statusCode} ${url}`
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
67
app/js/db.js
Normal file
67
app/js/db.js
Normal file
@@ -0,0 +1,67 @@
|
||||
/* eslint-disable
|
||||
no-console,
|
||||
*/
|
||||
// TODO: This file was created by bulk-decaffeinate.
|
||||
// Fix any style issues and re-enable lint.
|
||||
/*
|
||||
* decaffeinate suggestions:
|
||||
* DS102: Remove unnecessary code created because of implicit returns
|
||||
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
||||
*/
|
||||
const Sequelize = require('sequelize')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const _ = require('underscore')
|
||||
const logger = require('logger-sharelatex')
|
||||
|
||||
const options = _.extend({ logging: false }, Settings.mysql.clsi)
|
||||
|
||||
logger.log({ dbPath: Settings.mysql.clsi.storage }, 'connecting to db')
|
||||
|
||||
const sequelize = new Sequelize(
|
||||
Settings.mysql.clsi.database,
|
||||
Settings.mysql.clsi.username,
|
||||
Settings.mysql.clsi.password,
|
||||
options
|
||||
)
|
||||
|
||||
if (Settings.mysql.clsi.dialect === 'sqlite') {
|
||||
logger.log('running PRAGMA journal_mode=WAL;')
|
||||
sequelize.query('PRAGMA journal_mode=WAL;')
|
||||
sequelize.query('PRAGMA synchronous=OFF;')
|
||||
sequelize.query('PRAGMA read_uncommitted = true;')
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
UrlCache: sequelize.define(
|
||||
'UrlCache',
|
||||
{
|
||||
url: Sequelize.STRING,
|
||||
project_id: Sequelize.STRING,
|
||||
lastModified: Sequelize.DATE
|
||||
},
|
||||
{
|
||||
indexes: [{ fields: ['url', 'project_id'] }, { fields: ['project_id'] }]
|
||||
}
|
||||
),
|
||||
|
||||
Project: sequelize.define(
|
||||
'Project',
|
||||
{
|
||||
project_id: { type: Sequelize.STRING, primaryKey: true },
|
||||
lastAccessed: Sequelize.DATE
|
||||
},
|
||||
{
|
||||
indexes: [{ fields: ['lastAccessed'] }]
|
||||
}
|
||||
),
|
||||
|
||||
op: Sequelize.Op,
|
||||
|
||||
sync() {
|
||||
logger.log({ dbPath: Settings.mysql.clsi.storage }, 'syncing db schema')
|
||||
return sequelize
|
||||
.sync()
|
||||
.then(() => logger.log('db sync complete'))
|
||||
.catch(err => console.log(err, 'error syncing'))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user