Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a35df6d829 | ||
|
|
1a794d804a | ||
|
|
24e20a79f4 | ||
|
|
c47f49e24b | ||
|
|
65f2f23cf6 | ||
|
|
8d7d637eed | ||
|
|
7551bc3135 | ||
|
|
75ef0d6581 | ||
|
|
31f62c7a7b | ||
|
|
3a4dd9df50 | ||
|
|
916b4cb40b | ||
|
|
37cc9f3715 | ||
|
|
0692e964ef | ||
|
|
198e1ef492 | ||
|
|
280d64cf60 | ||
|
|
e7ed8d786a | ||
|
|
81e85de169 | ||
|
|
151ea99639 | ||
|
|
b8cdd4fa85 | ||
|
|
163a33674b | ||
|
|
67bfeacab8 | ||
|
|
1923352e66 | ||
|
|
f37004cec6 | ||
|
|
1a7500f102 | ||
|
|
90cda12ed9 | ||
|
|
c84bd4fa3f | ||
|
|
84f3d3061d | ||
|
|
2c4fbd10ed | ||
|
|
ff94a76eb9 | ||
|
|
92338ab419 | ||
|
|
5b2031b84f | ||
|
|
94397854c6 | ||
|
|
6bf8c22d78 | ||
|
|
b4f0da0c42 | ||
|
|
4886620d8a | ||
|
|
fc674370bd | ||
|
|
b418ea201b | ||
|
|
7f9c9176a9 | ||
|
|
af86745112 | ||
|
|
22e8ee59af | ||
|
|
225a12fcd2 |
@@ -50,6 +50,7 @@ module.exports = (grunt) ->
|
|||||||
unit:
|
unit:
|
||||||
options:
|
options:
|
||||||
reporter: "spec"
|
reporter: "spec"
|
||||||
|
grep: grunt.option("grep")
|
||||||
src: ["test/unit/js/**/*.js"]
|
src: ["test/unit/js/**/*.js"]
|
||||||
acceptance:
|
acceptance:
|
||||||
options:
|
options:
|
||||||
|
|||||||
43
app.coffee
43
app.coffee
@@ -4,11 +4,15 @@ logger = require "logger-sharelatex"
|
|||||||
logger.initialize("clsi")
|
logger.initialize("clsi")
|
||||||
smokeTest = require "smoke-test-sharelatex"
|
smokeTest = require "smoke-test-sharelatex"
|
||||||
|
|
||||||
|
Path = require "path"
|
||||||
|
fs = require "fs"
|
||||||
|
|
||||||
Metrics = require "metrics-sharelatex"
|
Metrics = require "metrics-sharelatex"
|
||||||
Metrics.initialize("clsi")
|
Metrics.initialize("clsi")
|
||||||
Metrics.open_sockets.monitor(logger)
|
Metrics.open_sockets.monitor(logger)
|
||||||
|
|
||||||
ProjectPersistenceManager = require "./app/js/ProjectPersistenceManager"
|
ProjectPersistenceManager = require "./app/js/ProjectPersistenceManager"
|
||||||
|
OutputCacheManager = require "./app/js/OutputCacheManager"
|
||||||
|
|
||||||
require("./app/js/db").sync()
|
require("./app/js/db").sync()
|
||||||
|
|
||||||
@@ -21,7 +25,7 @@ app.use Metrics.http.monitor(logger)
|
|||||||
# Compile requests can take longer than the default two
|
# Compile requests can take longer than the default two
|
||||||
# minutes (including file download time), so bump up the
|
# minutes (including file download time), so bump up the
|
||||||
# timeout a bit.
|
# timeout a bit.
|
||||||
TIMEOUT = threeMinutes = 3 * 60 * 1000
|
TIMEOUT = 6 * 60 * 1000
|
||||||
app.use (req, res, next) ->
|
app.use (req, res, next) ->
|
||||||
req.setTimeout TIMEOUT
|
req.setTimeout TIMEOUT
|
||||||
res.setTimeout TIMEOUT
|
res.setTimeout TIMEOUT
|
||||||
@@ -33,9 +37,31 @@ app.delete "/project/:project_id", CompileController.clearCache
|
|||||||
app.get "/project/:project_id/sync/code", CompileController.syncFromCode
|
app.get "/project/:project_id/sync/code", CompileController.syncFromCode
|
||||||
app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf
|
app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf
|
||||||
|
|
||||||
staticServer = express.static(Settings.path.compilesDir)
|
ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks"
|
||||||
|
|
||||||
|
# create a static server which does not allow access to any symlinks
|
||||||
|
# avoids possible mismatch of root directory between middleware check
|
||||||
|
# and serving the files
|
||||||
|
staticServer = ForbidSymlinks express.static, Settings.path.compilesDir, setHeaders: (res, path, stat) ->
|
||||||
|
if Path.basename(path) == "output.pdf"
|
||||||
|
res.set("Content-Type", "application/pdf")
|
||||||
|
# Calculate an etag in the same way as nginx
|
||||||
|
# https://github.com/tj/send/issues/65
|
||||||
|
etag = (path, stat) ->
|
||||||
|
'"' + Math.ceil(+stat.mtime / 1000).toString(16) +
|
||||||
|
'-' + Number(stat.size).toString(16) + '"'
|
||||||
|
res.set("Etag", etag(path, stat))
|
||||||
|
else
|
||||||
|
# Force plain treatment of other file types to prevent hosting of HTTP/JS files
|
||||||
|
# that could be used in same-origin/XSS attacks.
|
||||||
|
res.set("Content-Type", "text/plain")
|
||||||
|
|
||||||
app.get "/project/:project_id/output/*", (req, res, next) ->
|
app.get "/project/:project_id/output/*", (req, res, next) ->
|
||||||
req.url = "/#{req.params.project_id}/#{req.params[0]}"
|
if req.query?.build? && req.query.build.match(OutputCacheManager.BUILD_REGEX)
|
||||||
|
# for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
|
||||||
|
req.url = "/#{req.params.project_id}/" + OutputCacheManager.path(req.query.build, "/#{req.params[0]}")
|
||||||
|
else
|
||||||
|
req.url = "/#{req.params.project_id}/#{req.params[0]}"
|
||||||
staticServer(req, res, next)
|
staticServer(req, res, next)
|
||||||
|
|
||||||
app.get "/status", (req, res, next) ->
|
app.get "/status", (req, res, next) ->
|
||||||
@@ -60,9 +86,18 @@ app.get "/health_check", (req, res)->
|
|||||||
res.contentType(resCacher?.setContentType)
|
res.contentType(resCacher?.setContentType)
|
||||||
res.send resCacher?.code, resCacher?.body
|
res.send resCacher?.code, resCacher?.body
|
||||||
|
|
||||||
|
profiler = require "v8-profiler"
|
||||||
|
app.get "/profile", (req, res) ->
|
||||||
|
time = parseInt(req.query.time || "1000")
|
||||||
|
profiler.startProfiling("test")
|
||||||
|
setTimeout () ->
|
||||||
|
profile = profiler.stopProfiling("test")
|
||||||
|
res.json(profile)
|
||||||
|
, time
|
||||||
|
|
||||||
app.use (error, req, res, next) ->
|
app.use (error, req, res, next) ->
|
||||||
logger.error err: error, "server error"
|
logger.error err: error, "server error"
|
||||||
res.send 500
|
res.send error?.statusCode || 500
|
||||||
|
|
||||||
app.listen port = (Settings.internal?.clsi?.port or 3013), host = (Settings.internal?.clsi?.host or "localhost"), (error) ->
|
app.listen port = (Settings.internal?.clsi?.port or 3013), host = (Settings.internal?.clsi?.host or "localhost"), (error) ->
|
||||||
logger.log "CLSI listening on #{host}:#{port}"
|
logger.log "CLSI listening on #{host}:#{port}"
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ module.exports = CompileController =
|
|||||||
outputFiles: outputFiles.map (file) ->
|
outputFiles: outputFiles.map (file) ->
|
||||||
url: "#{Settings.apis.clsi.url}/project/#{request.project_id}/output/#{file.path}"
|
url: "#{Settings.apis.clsi.url}/project/#{request.project_id}/output/#{file.path}"
|
||||||
type: file.type
|
type: file.type
|
||||||
|
build: file.build
|
||||||
}
|
}
|
||||||
|
|
||||||
clearCache: (req, res, next = (error) ->) ->
|
clearCache: (req, res, next = (error) ->) ->
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
ResourceWriter = require "./ResourceWriter"
|
ResourceWriter = require "./ResourceWriter"
|
||||||
LatexRunner = require "./LatexRunner"
|
LatexRunner = require "./LatexRunner"
|
||||||
OutputFileFinder = require "./OutputFileFinder"
|
OutputFileFinder = require "./OutputFileFinder"
|
||||||
|
OutputCacheManager = require "./OutputCacheManager"
|
||||||
Settings = require("settings-sharelatex")
|
Settings = require("settings-sharelatex")
|
||||||
Path = require "path"
|
Path = require "path"
|
||||||
logger = require "logger-sharelatex"
|
logger = require "logger-sharelatex"
|
||||||
@@ -32,7 +33,8 @@ module.exports = CompileManager =
|
|||||||
|
|
||||||
OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) ->
|
OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) ->
|
||||||
return callback(error) if error?
|
return callback(error) if error?
|
||||||
callback null, outputFiles
|
OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) ->
|
||||||
|
callback null, newOutputFiles
|
||||||
|
|
||||||
clearProject: (project_id, _callback = (error) ->) ->
|
clearProject: (project_id, _callback = (error) ->) ->
|
||||||
callback = (error) ->
|
callback = (error) ->
|
||||||
|
|||||||
113
app/coffee/OutputCacheManager.coffee
Normal file
113
app/coffee/OutputCacheManager.coffee
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
async = require "async"
|
||||||
|
fs = require "fs"
|
||||||
|
fse = require "fs-extra"
|
||||||
|
Path = require "path"
|
||||||
|
logger = require "logger-sharelatex"
|
||||||
|
_ = require "underscore"
|
||||||
|
|
||||||
|
OutputFileOptimiser = require "./OutputFileOptimiser"
|
||||||
|
|
||||||
|
module.exports = OutputCacheManager =
|
||||||
|
CACHE_SUBDIR: '.cache/clsi'
|
||||||
|
BUILD_REGEX: /^[0-9a-f]+$/ # build id is Date.now() converted to hex
|
||||||
|
CACHE_LIMIT: 2 # maximum number of cache directories
|
||||||
|
CACHE_AGE: 60*60*1000 # up to one hour old
|
||||||
|
|
||||||
|
path: (buildId, file) ->
|
||||||
|
# used by static server, given build id return '.cache/clsi/buildId'
|
||||||
|
if buildId.match OutputCacheManager.BUILD_REGEX
|
||||||
|
return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
|
||||||
|
else
|
||||||
|
# for invalid build id, return top level
|
||||||
|
return file
|
||||||
|
|
||||||
|
saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) ->
|
||||||
|
# make a compileDir/CACHE_SUBDIR/build_id directory and
|
||||||
|
# copy all the output files into it
|
||||||
|
cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
|
||||||
|
# Put the files into a new cache subdirectory
|
||||||
|
buildId = Date.now().toString(16)
|
||||||
|
cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
|
||||||
|
# let file expiry run in the background
|
||||||
|
OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId}
|
||||||
|
|
||||||
|
checkFile = (src, callback) ->
|
||||||
|
# check if we have a valid file to copy into the cache
|
||||||
|
fs.stat src, (err, stats) ->
|
||||||
|
if err?
|
||||||
|
# some problem reading the file
|
||||||
|
logger.error err: err, file: src, "stat error for file in cache"
|
||||||
|
callback(err)
|
||||||
|
else if not stats.isFile()
|
||||||
|
# other filetype - reject it
|
||||||
|
logger.error err: err, src: src, dst: dst, stat: stats, "nonfile output - refusing to copy to cache"
|
||||||
|
callback(new Error("output file is not a file"), file)
|
||||||
|
else
|
||||||
|
# it's a plain file, ok to copy
|
||||||
|
callback(null)
|
||||||
|
|
||||||
|
copyFile = (src, dst, callback) ->
|
||||||
|
# copy output file into the cache
|
||||||
|
fse.copy src, dst, (err) ->
|
||||||
|
if err?
|
||||||
|
logger.error err: err, src: src, dst: dst, "copy error for file in cache"
|
||||||
|
callback(err)
|
||||||
|
else
|
||||||
|
# call the optimiser for the file too
|
||||||
|
OutputFileOptimiser.optimiseFile src, dst, callback
|
||||||
|
|
||||||
|
# make the new cache directory
|
||||||
|
fse.ensureDir cacheDir, (err) ->
|
||||||
|
if err?
|
||||||
|
logger.error err: err, directory: cacheDir, "error creating cache directory"
|
||||||
|
callback(err, outputFiles)
|
||||||
|
else
|
||||||
|
# copy all the output files into the new cache directory
|
||||||
|
async.mapSeries outputFiles, (file, cb) ->
|
||||||
|
newFile = _.clone(file)
|
||||||
|
[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
|
||||||
|
checkFile src, (err) ->
|
||||||
|
copyFile src, dst, (err) ->
|
||||||
|
if not err?
|
||||||
|
newFile.build = buildId # attach a build id if we cached the file
|
||||||
|
cb(err, newFile)
|
||||||
|
, (err, results) ->
|
||||||
|
if err?
|
||||||
|
# pass back the original files if we encountered *any* error
|
||||||
|
callback(err, outputFiles)
|
||||||
|
else
|
||||||
|
# pass back the list of new files in the cache
|
||||||
|
callback(err, results)
|
||||||
|
|
||||||
|
expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
|
||||||
|
# look in compileDir for build dirs and delete if > N or age of mod time > T
|
||||||
|
fs.readdir cacheRoot, (err, results) ->
|
||||||
|
if err?
|
||||||
|
return callback(null) if err.code == 'ENOENT' # cache directory is empty
|
||||||
|
logger.error err: err, project_id: cacheRoot, "error clearing cache"
|
||||||
|
return callback(err)
|
||||||
|
|
||||||
|
dirs = results.sort().reverse()
|
||||||
|
currentTime = Date.now()
|
||||||
|
|
||||||
|
isExpired = (dir, index) ->
|
||||||
|
return false if options?.keep == dir
|
||||||
|
# remove any directories over the hard limit
|
||||||
|
return true if index > OutputCacheManager.CACHE_LIMIT
|
||||||
|
# we can get the build time from the directory name
|
||||||
|
dirTime = parseInt(dir, 16)
|
||||||
|
age = currentTime - dirTime
|
||||||
|
return age > OutputCacheManager.CACHE_AGE
|
||||||
|
|
||||||
|
toRemove = _.filter(dirs, isExpired)
|
||||||
|
|
||||||
|
removeDir = (dir, cb) ->
|
||||||
|
fse.remove Path.join(cacheRoot, dir), (err, result) ->
|
||||||
|
logger.log cache: cacheRoot, dir: dir, "removed expired cache dir"
|
||||||
|
if err?
|
||||||
|
logger.error err: err, dir: dir, "cache remove error"
|
||||||
|
cb(err, result)
|
||||||
|
|
||||||
|
async.eachSeries toRemove, (dir, cb) ->
|
||||||
|
removeDir dir, cb
|
||||||
|
, callback
|
||||||
@@ -1,58 +1,49 @@
|
|||||||
async = require "async"
|
async = require "async"
|
||||||
fs = require "fs"
|
fs = require "fs"
|
||||||
Path = require "path"
|
Path = require "path"
|
||||||
wrench = require "wrench"
|
spawn = require("child_process").spawn
|
||||||
|
logger = require "logger-sharelatex"
|
||||||
|
|
||||||
module.exports = OutputFileFinder =
|
module.exports = OutputFileFinder =
|
||||||
findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
|
findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
|
||||||
incomingResources = {}
|
incomingResources = {}
|
||||||
for resource in resources
|
for resource in resources
|
||||||
incomingResources[resource.path] = true
|
incomingResources[resource.path] = true
|
||||||
|
|
||||||
|
logger.log directory: directory, "getting output files"
|
||||||
|
|
||||||
OutputFileFinder._getAllFiles directory, (error, allFiles) ->
|
OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
|
||||||
|
return callback(error) if error?
|
||||||
jobs = []
|
jobs = []
|
||||||
outputFiles = []
|
outputFiles = []
|
||||||
for file in allFiles
|
for file in allFiles
|
||||||
do (file) ->
|
if !incomingResources[file]
|
||||||
jobs.push (callback) ->
|
outputFiles.push {
|
||||||
if incomingResources[file.path]
|
path: file
|
||||||
return callback()
|
type: file.match(/\.([^\.]+)$/)?[1]
|
||||||
else
|
}
|
||||||
OutputFileFinder._isDirectory Path.join(directory, file.path), (error, directory) ->
|
callback null, outputFiles
|
||||||
return callback(error) if error?
|
|
||||||
if !directory
|
|
||||||
outputFiles.push file
|
|
||||||
callback()
|
|
||||||
|
|
||||||
async.series jobs, (error) ->
|
_getAllFiles: (directory, _callback = (error, fileList) ->) ->
|
||||||
return callback(error) if error?
|
callback = (error, fileList) ->
|
||||||
callback null, outputFiles
|
_callback(error, fileList)
|
||||||
|
|
||||||
_isDirectory: (path, callback = (error, directory) ->) ->
|
|
||||||
fs.stat path, (error, stat) ->
|
|
||||||
callback error, stat?.isDirectory()
|
|
||||||
|
|
||||||
_getAllFiles: (directory, _callback = (error, outputFiles) ->) ->
|
|
||||||
callback = (error, outputFiles) ->
|
|
||||||
_callback(error, outputFiles)
|
|
||||||
_callback = () ->
|
_callback = () ->
|
||||||
|
|
||||||
|
args = [directory, "-name", ".cache", "-prune", "-o", "-type", "f", "-print"]
|
||||||
|
logger.log args: args, "running find command"
|
||||||
|
|
||||||
outputFiles = []
|
proc = spawn("find", args)
|
||||||
|
stdout = ""
|
||||||
wrench.readdirRecursive directory, (error, files) =>
|
proc.stdout.on "data", (chunk) ->
|
||||||
if error?
|
stdout += chunk.toString()
|
||||||
if error.code == "ENOENT"
|
proc.on "error", callback
|
||||||
# Directory doesn't exist, which is not a problem
|
proc.on "close", (code) ->
|
||||||
return callback(null, [])
|
if code != 0
|
||||||
else
|
logger.warn {directory, code}, "find returned error, directory likely doesn't exist"
|
||||||
return callback(error)
|
return callback null, []
|
||||||
|
fileList = stdout.trim().split("\n")
|
||||||
# readdirRecursive returns multiple times and finishes with a null response
|
fileList = fileList.map (file) ->
|
||||||
if !files?
|
# Strip leading directory
|
||||||
return callback(null, outputFiles)
|
path = Path.relative(directory, file)
|
||||||
|
return callback null, fileList
|
||||||
for file in files
|
|
||||||
outputFiles.push
|
|
||||||
path: file
|
|
||||||
type: file.match(/\.([^\.]+)$/)?[1]
|
|
||||||
|
|
||||||
|
|||||||
37
app/coffee/OutputFileOptimiser.coffee
Normal file
37
app/coffee/OutputFileOptimiser.coffee
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
fs = require "fs"
|
||||||
|
Path = require "path"
|
||||||
|
spawn = require("child_process").spawn
|
||||||
|
logger = require "logger-sharelatex"
|
||||||
|
_ = require "underscore"
|
||||||
|
|
||||||
|
module.exports = OutputFileOptimiser =
|
||||||
|
|
||||||
|
optimiseFile: (src, dst, callback = (error) ->) ->
|
||||||
|
# check output file (src) and see if we can optimise it, storing
|
||||||
|
# the result in the build directory (dst)
|
||||||
|
if src.match(/\.pdf$/)
|
||||||
|
OutputFileOptimiser.optimisePDF src, dst, callback
|
||||||
|
else
|
||||||
|
callback (null)
|
||||||
|
|
||||||
|
optimisePDF: (src, dst, callback = (error) ->) ->
|
||||||
|
tmpOutput = dst + '.opt'
|
||||||
|
args = ["--linearize", src, tmpOutput]
|
||||||
|
logger.log args: args, "running qpdf command"
|
||||||
|
|
||||||
|
proc = spawn("qpdf", args)
|
||||||
|
stdout = ""
|
||||||
|
proc.stdout.on "data", (chunk) ->
|
||||||
|
stdout += chunk.toString()
|
||||||
|
callback = _.once(callback) # avoid double call back for error and close event
|
||||||
|
proc.on "error", (err) ->
|
||||||
|
logger.warn {err, args}, "qpdf failed"
|
||||||
|
callback(null) # ignore the error
|
||||||
|
proc.on "close", (code) ->
|
||||||
|
if code != 0
|
||||||
|
logger.warn {code, args}, "qpdf returned error"
|
||||||
|
return callback(null) # ignore the error
|
||||||
|
fs.rename tmpOutput, dst, (err) ->
|
||||||
|
if err?
|
||||||
|
logger.warn {tmpOutput, dst}, "failed to rename output of qpdf command"
|
||||||
|
callback(null) # ignore the error
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
module.exports = RequestParser =
|
module.exports = RequestParser =
|
||||||
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
|
VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
|
||||||
MAX_TIMEOUT: 60
|
MAX_TIMEOUT: 300
|
||||||
|
|
||||||
parse: (body, callback = (error, data) ->) ->
|
parse: (body, callback = (error, data) ->) ->
|
||||||
response = {}
|
response = {}
|
||||||
@@ -27,10 +27,12 @@ module.exports = RequestParser =
|
|||||||
response.timeout = response.timeout * 1000 # milliseconds
|
response.timeout = response.timeout * 1000 # milliseconds
|
||||||
|
|
||||||
response.resources = (@_parseResource(resource) for resource in (compile.resources or []))
|
response.resources = (@_parseResource(resource) for resource in (compile.resources or []))
|
||||||
response.rootResourcePath = @_parseAttribute "rootResourcePath",
|
|
||||||
|
rootResourcePath = @_parseAttribute "rootResourcePath",
|
||||||
compile.rootResourcePath
|
compile.rootResourcePath
|
||||||
default: "main.tex"
|
default: "main.tex"
|
||||||
type: "string"
|
type: "string"
|
||||||
|
response.rootResourcePath = RequestParser._sanitizePath(rootResourcePath)
|
||||||
catch error
|
catch error
|
||||||
return callback error
|
return callback error
|
||||||
|
|
||||||
@@ -72,3 +74,6 @@ module.exports = RequestParser =
|
|||||||
throw "Default not implemented"
|
throw "Default not implemented"
|
||||||
return attribute
|
return attribute
|
||||||
|
|
||||||
|
_sanitizePath: (path) ->
|
||||||
|
# See http://php.net/manual/en/function.escapeshellcmd.php
|
||||||
|
path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "")
|
||||||
|
|||||||
24
app/coffee/StaticServerForbidSymlinks.coffee
Normal file
24
app/coffee/StaticServerForbidSymlinks.coffee
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
Path = require("path")
|
||||||
|
fs = require("fs")
|
||||||
|
Settings = require("settings-sharelatex")
|
||||||
|
logger = require("logger-sharelatex")
|
||||||
|
url = require "url"
|
||||||
|
|
||||||
|
module.exports = ForbidSymlinks = (staticFn, root, options) ->
|
||||||
|
expressStatic = staticFn root, options
|
||||||
|
basePath = Path.resolve(root)
|
||||||
|
return (req, res, next) ->
|
||||||
|
path = url.parse(req.url)?.pathname
|
||||||
|
requestedFsPath = Path.normalize("#{basePath}/#{path}")
|
||||||
|
fs.realpath requestedFsPath, (err, realFsPath)->
|
||||||
|
if err?
|
||||||
|
logger.warn err:err, requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "error checking file access"
|
||||||
|
if err.code == 'ENOENT'
|
||||||
|
return res.sendStatus(404)
|
||||||
|
else
|
||||||
|
return res.sendStatus(500)
|
||||||
|
else if requestedFsPath != realFsPath
|
||||||
|
logger.warn requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "trying to access a different file (symlink), aborting"
|
||||||
|
return res.sendStatus(404)
|
||||||
|
else
|
||||||
|
expressStatic(req, res, next)
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "node-clsi",
|
"name": "node-clsi",
|
||||||
"description": "A Node.js implementation of the CLSI LaTeX web-API",
|
"description": "A Node.js implementation of the CLSI LaTeX web-API",
|
||||||
"version": "0.1.1",
|
"version": "0.1.4",
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://github.com/sharelatex/clsi-sharelatex.git"
|
"url": "https://github.com/sharelatex/clsi-sharelatex.git"
|
||||||
@@ -21,7 +21,10 @@
|
|||||||
"smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v1.0.0",
|
"smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v1.0.0",
|
||||||
"sqlite3": "~2.2.0",
|
"sqlite3": "~2.2.0",
|
||||||
"express": "^4.2.0",
|
"express": "^4.2.0",
|
||||||
"body-parser": "^1.2.0"
|
"body-parser": "^1.2.0",
|
||||||
|
"fs-extra": "^0.16.3",
|
||||||
|
"underscore": "^1.8.2",
|
||||||
|
"v8-profiler": "^5.2.4"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"mocha": "1.10.0",
|
"mocha": "1.10.0",
|
||||||
|
|||||||
Binary file not shown.
@@ -1,17 +0,0 @@
|
|||||||
Section Title
|
|
||||||
-------------
|
|
||||||
|
|
||||||
* List item one
|
|
||||||
* List item two
|
|
||||||
|
|
||||||
: Sample grid table.
|
|
||||||
|
|
||||||
+---------------+---------------+--------------------+
|
|
||||||
| Fruit | Price | Advantages |
|
|
||||||
+===============+===============+====================+
|
|
||||||
| Bananas | $1.34 | - built-in wrapper |
|
|
||||||
| | | - bright color |
|
|
||||||
+---------------+---------------+--------------------+
|
|
||||||
| Oranges | $2.10 | - cures scurvy |
|
|
||||||
| | | - tasty |
|
|
||||||
+---------------+---------------+--------------------+
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
\documentclass{article}
|
|
||||||
\usepackage{longtable}
|
|
||||||
\usepackage{booktabs, multicol, multirow}
|
|
||||||
|
|
||||||
\begin{document}
|
|
||||||
|
|
||||||
\input{chapters/chapter1}
|
|
||||||
|
|
||||||
\end{document}
|
|
||||||
Binary file not shown.
@@ -1,23 +0,0 @@
|
|||||||
% Title
|
|
||||||
% Author
|
|
||||||
% Date
|
|
||||||
|
|
||||||
Chapter title
|
|
||||||
=============
|
|
||||||
|
|
||||||
Section Title
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Hello world. Have a nice table:
|
|
||||||
|
|
||||||
: Sample grid table.
|
|
||||||
|
|
||||||
+---------------+---------------+--------------------+
|
|
||||||
| Fruit | Price | Advantages |
|
|
||||||
+===============+===============+====================+
|
|
||||||
| Bananas | $1.34 | - built-in wrapper |
|
|
||||||
| | | - bright color |
|
|
||||||
+---------------+---------------+--------------------+
|
|
||||||
| Oranges | $2.10 | - cures scurvy |
|
|
||||||
| | | - tasty |
|
|
||||||
+---------------+---------------+--------------------+
|
|
||||||
Binary file not shown.
@@ -36,9 +36,11 @@ describe "CompileController", ->
|
|||||||
@output_files = [{
|
@output_files = [{
|
||||||
path: "output.pdf"
|
path: "output.pdf"
|
||||||
type: "pdf"
|
type: "pdf"
|
||||||
|
build: 1234
|
||||||
}, {
|
}, {
|
||||||
path: "output.log"
|
path: "output.log"
|
||||||
type: "log"
|
type: "log"
|
||||||
|
build: 1234
|
||||||
}]
|
}]
|
||||||
@RequestParser.parse = sinon.stub().callsArgWith(1, null, @request)
|
@RequestParser.parse = sinon.stub().callsArgWith(1, null, @request)
|
||||||
@ProjectPersistenceManager.markProjectAsJustAccessed = sinon.stub().callsArg(1)
|
@ProjectPersistenceManager.markProjectAsJustAccessed = sinon.stub().callsArg(1)
|
||||||
@@ -73,6 +75,7 @@ describe "CompileController", ->
|
|||||||
outputFiles: @output_files.map (file) =>
|
outputFiles: @output_files.map (file) =>
|
||||||
url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/output/#{file.path}"
|
url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/output/#{file.path}"
|
||||||
type: file.type
|
type: file.type
|
||||||
|
build: file.build
|
||||||
)
|
)
|
||||||
.should.equal true
|
.should.equal true
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ describe "CompileManager", ->
|
|||||||
"./LatexRunner": @LatexRunner = {}
|
"./LatexRunner": @LatexRunner = {}
|
||||||
"./ResourceWriter": @ResourceWriter = {}
|
"./ResourceWriter": @ResourceWriter = {}
|
||||||
"./OutputFileFinder": @OutputFileFinder = {}
|
"./OutputFileFinder": @OutputFileFinder = {}
|
||||||
|
"./OutputCacheManager": @OutputCacheManager = {}
|
||||||
"settings-sharelatex": @Settings = { path: compilesDir: "/compiles/dir" }
|
"settings-sharelatex": @Settings = { path: compilesDir: "/compiles/dir" }
|
||||||
"logger-sharelatex": @logger = { log: sinon.stub() }
|
"logger-sharelatex": @logger = { log: sinon.stub() }
|
||||||
"child_process": @child_process = {}
|
"child_process": @child_process = {}
|
||||||
@@ -26,6 +27,15 @@ describe "CompileManager", ->
|
|||||||
path: "output.pdf"
|
path: "output.pdf"
|
||||||
type: "pdf"
|
type: "pdf"
|
||||||
}]
|
}]
|
||||||
|
@build_files = [{
|
||||||
|
path: "output.log"
|
||||||
|
type: "log"
|
||||||
|
build: 1234
|
||||||
|
}, {
|
||||||
|
path: "output.pdf"
|
||||||
|
type: "pdf"
|
||||||
|
build: 1234
|
||||||
|
}]
|
||||||
@request =
|
@request =
|
||||||
resources: @resources = "mock-resources"
|
resources: @resources = "mock-resources"
|
||||||
rootResourcePath: @rootResourcePath = "main.tex"
|
rootResourcePath: @rootResourcePath = "main.tex"
|
||||||
@@ -37,6 +47,7 @@ describe "CompileManager", ->
|
|||||||
@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3)
|
@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3)
|
||||||
@LatexRunner.runLatex = sinon.stub().callsArg(2)
|
@LatexRunner.runLatex = sinon.stub().callsArg(2)
|
||||||
@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
|
@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
|
||||||
|
@OutputCacheManager.saveOutputFiles = sinon.stub().callsArgWith(2, null, @build_files)
|
||||||
@CompileManager.doCompile @request, @callback
|
@CompileManager.doCompile @request, @callback
|
||||||
|
|
||||||
it "should write the resources to disk", ->
|
it "should write the resources to disk", ->
|
||||||
@@ -60,7 +71,8 @@ describe "CompileManager", ->
|
|||||||
.should.equal true
|
.should.equal true
|
||||||
|
|
||||||
it "should return the output files", ->
|
it "should return the output files", ->
|
||||||
@callback.calledWith(null, @output_files).should.equal true
|
console.log 'output_files', @build_files
|
||||||
|
@callback.calledWith(null, @build_files).should.equal true
|
||||||
|
|
||||||
describe "clearProject", ->
|
describe "clearProject", ->
|
||||||
describe "succesfully", ->
|
describe "succesfully", ->
|
||||||
|
|||||||
@@ -4,29 +4,26 @@ require('chai').should()
|
|||||||
modulePath = require('path').join __dirname, '../../../app/js/OutputFileFinder'
|
modulePath = require('path').join __dirname, '../../../app/js/OutputFileFinder'
|
||||||
path = require "path"
|
path = require "path"
|
||||||
expect = require("chai").expect
|
expect = require("chai").expect
|
||||||
|
EventEmitter = require("events").EventEmitter
|
||||||
|
|
||||||
describe "OutputFileFinder", ->
|
describe "OutputFileFinder", ->
|
||||||
beforeEach ->
|
beforeEach ->
|
||||||
@OutputFileFinder = SandboxedModule.require modulePath, requires:
|
@OutputFileFinder = SandboxedModule.require modulePath, requires:
|
||||||
"fs": @fs = {}
|
"fs": @fs = {}
|
||||||
"wrench": @wrench = {}
|
"child_process": spawn: @spawn = sinon.stub()
|
||||||
|
"logger-sharelatex": { log: sinon.stub(), warn: sinon.stub() }
|
||||||
@directory = "/test/dir"
|
@directory = "/test/dir"
|
||||||
@callback = sinon.stub()
|
@callback = sinon.stub()
|
||||||
|
|
||||||
describe "findOutputFiles", ->
|
describe "findOutputFiles", ->
|
||||||
beforeEach ->
|
beforeEach ->
|
||||||
@resource_path = "resource/path.tex"
|
@resource_path = "resource/path.tex"
|
||||||
@output_paths = ["output.pdf", "extra", "extra/file.tex"]
|
@output_paths = ["output.pdf", "extra/file.tex"]
|
||||||
|
@all_paths = @output_paths.concat [@resource_path]
|
||||||
@resources = [
|
@resources = [
|
||||||
path: @resource_path = "resource/path.tex"
|
path: @resource_path = "resource/path.tex"
|
||||||
]
|
]
|
||||||
@OutputFileFinder._isDirectory = (dirPath, callback = (error, directory) ->) =>
|
@OutputFileFinder._getAllFiles = sinon.stub().callsArgWith(1, null, @all_paths)
|
||||||
callback null, dirPath == path.join(@directory, "extra")
|
|
||||||
|
|
||||||
@wrench.readdirRecursive = (dir, callback) =>
|
|
||||||
callback(null, [@resource_path].concat(@output_paths))
|
|
||||||
callback(null, null)
|
|
||||||
sinon.spy @wrench, "readdirRecursive"
|
|
||||||
@OutputFileFinder.findOutputFiles @resources, @directory, (error, @outputFiles) =>
|
@OutputFileFinder.findOutputFiles @resources, @directory, (error, @outputFiles) =>
|
||||||
|
|
||||||
it "should only return the output files, not directories or resource paths", ->
|
it "should only return the output files, not directories or resource paths", ->
|
||||||
@@ -37,5 +34,35 @@ describe "OutputFileFinder", ->
|
|||||||
path: "extra/file.tex",
|
path: "extra/file.tex",
|
||||||
type: "tex"
|
type: "tex"
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
describe "_getAllFiles", ->
|
||||||
|
beforeEach ->
|
||||||
|
@proc = new EventEmitter()
|
||||||
|
@proc.stdout = new EventEmitter()
|
||||||
|
@spawn.returns @proc
|
||||||
|
@directory = "/base/dir"
|
||||||
|
@OutputFileFinder._getAllFiles @directory, @callback
|
||||||
|
|
||||||
|
describe "successfully", ->
|
||||||
|
beforeEach ->
|
||||||
|
@proc.stdout.emit(
|
||||||
|
"data",
|
||||||
|
["/base/dir/main.tex", "/base/dir/chapters/chapter1.tex"].join("\n") + "\n"
|
||||||
|
)
|
||||||
|
@proc.emit "close", 0
|
||||||
|
|
||||||
|
it "should call the callback with the relative file paths", ->
|
||||||
|
@callback.calledWith(
|
||||||
|
null,
|
||||||
|
["main.tex", "chapters/chapter1.tex"]
|
||||||
|
).should.equal true
|
||||||
|
|
||||||
|
describe "when the directory doesn't exist", ->
|
||||||
|
beforeEach ->
|
||||||
|
@proc.emit "close", 1
|
||||||
|
|
||||||
|
it "should call the callback with a blank array", ->
|
||||||
|
@callback.calledWith(
|
||||||
|
null,
|
||||||
|
[]
|
||||||
|
).should.equal true
|
||||||
|
|||||||
@@ -204,6 +204,13 @@ describe "RequestParser", ->
|
|||||||
@callback.calledWith("rootResourcePath attribute should be a string")
|
@callback.calledWith("rootResourcePath attribute should be a string")
|
||||||
.should.equal true
|
.should.equal true
|
||||||
|
|
||||||
|
describe "with a root resource path that needs escaping", ->
|
||||||
|
beforeEach ->
|
||||||
|
@validRequest.compile.rootResourcePath = "`rm -rf foo`.tex"
|
||||||
|
@RequestParser.parse @validRequest, @callback
|
||||||
|
@data = @callback.args[0][1]
|
||||||
|
|
||||||
|
it "should return the escaped resource", ->
|
||||||
|
@data.rootResourcePath.should.equal "rm -rf foo.tex"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
82
test/unit/coffee/StaticServerForbidSymlinksTests.coffee
Normal file
82
test/unit/coffee/StaticServerForbidSymlinksTests.coffee
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
should = require('chai').should()
|
||||||
|
SandboxedModule = require('sandboxed-module')
|
||||||
|
assert = require('assert')
|
||||||
|
path = require('path')
|
||||||
|
sinon = require('sinon')
|
||||||
|
modulePath = path.join __dirname, "../../../app/js/StaticServerForbidSymlinks"
|
||||||
|
expect = require("chai").expect
|
||||||
|
|
||||||
|
describe "StaticServerForbidSymlinks", ->
|
||||||
|
|
||||||
|
beforeEach ->
|
||||||
|
|
||||||
|
@settings =
|
||||||
|
path:
|
||||||
|
compilesDir: "/compiles/here"
|
||||||
|
|
||||||
|
@fs = {}
|
||||||
|
@ForbidSymlinks = SandboxedModule.require modulePath, requires:
|
||||||
|
"settings-sharelatex":@settings
|
||||||
|
"logger-sharelatex":
|
||||||
|
log:->
|
||||||
|
warn:->
|
||||||
|
"fs":@fs
|
||||||
|
|
||||||
|
@dummyStatic = (rootDir, options) ->
|
||||||
|
return (req, res, next) ->
|
||||||
|
# console.log "dummyStatic serving file", rootDir, "called with", req.url
|
||||||
|
# serve it
|
||||||
|
next()
|
||||||
|
|
||||||
|
@StaticServerForbidSymlinks = @ForbidSymlinks @dummyStatic, @settings.path.compilesDir
|
||||||
|
@req =
|
||||||
|
params:
|
||||||
|
project_id:"12345"
|
||||||
|
|
||||||
|
@res = {}
|
||||||
|
@req.url = "/12345/output.pdf"
|
||||||
|
|
||||||
|
|
||||||
|
describe "sending a normal file through", ->
|
||||||
|
beforeEach ->
|
||||||
|
@fs.realpath = sinon.stub().callsArgWith(1, null, "#{@settings.path.compilesDir}/#{@req.params.project_id}/output.pdf")
|
||||||
|
|
||||||
|
it "should call next", (done)->
|
||||||
|
@res.sendStatus = (resCode)->
|
||||||
|
resCode.should.equal 200
|
||||||
|
done()
|
||||||
|
@StaticServerForbidSymlinks @req, @res, done
|
||||||
|
|
||||||
|
|
||||||
|
describe "with a missing file", ->
|
||||||
|
beforeEach ->
|
||||||
|
@fs.realpath = sinon.stub().callsArgWith(1, {code: 'ENOENT'}, "#{@settings.path.compilesDir}/#{@req.params.project_id}/unknown.pdf")
|
||||||
|
|
||||||
|
it "should send a 404", (done)->
|
||||||
|
@res.sendStatus = (resCode)->
|
||||||
|
resCode.should.equal 404
|
||||||
|
done()
|
||||||
|
@StaticServerForbidSymlinks @req, @res
|
||||||
|
|
||||||
|
|
||||||
|
describe "with a symlink file", ->
|
||||||
|
beforeEach ->
|
||||||
|
@fs.realpath = sinon.stub().callsArgWith(1, null, "/etc/#{@req.params.project_id}/output.pdf")
|
||||||
|
|
||||||
|
it "should send a 404", (done)->
|
||||||
|
@res.sendStatus = (resCode)->
|
||||||
|
resCode.should.equal 404
|
||||||
|
done()
|
||||||
|
@StaticServerForbidSymlinks @req, @res
|
||||||
|
|
||||||
|
describe "with an error from fs.realpath", ->
|
||||||
|
|
||||||
|
beforeEach ->
|
||||||
|
@fs.realpath = sinon.stub().callsArgWith(1, "error")
|
||||||
|
|
||||||
|
it "should send a 500", (done)->
|
||||||
|
@res.sendStatus = (resCode)->
|
||||||
|
resCode.should.equal 500
|
||||||
|
done()
|
||||||
|
@StaticServerForbidSymlinks @req, @res
|
||||||
|
|
||||||
Reference in New Issue
Block a user