Test updated logger on clsi

Fixed NPE when Settings.clsi is defined but Settings.clsi.docker is not

.nvmrc

@@ -1 +1 @@
-10.19.0
+10.21.0

Dockerfile

@@ -2,7 +2,7 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
 
-FROM node:10.19.0 as base
+FROM node:10.21.0 as base
 
 WORKDIR /app
 COPY install_deps.sh /app

app.js

@@ -5,7 +5,7 @@
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
-let tenMinutes
+const tenMinutes = 10 * 60 * 1000
 const Metrics = require('metrics-sharelatex')
 Metrics.initialize('clsi')
 
@@ -17,7 +17,7 @@ if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
   logger.initializeErrorReporting(Settings.sentry.dsn)
 }
 
-const smokeTest = require('smoke-test-sharelatex')
+const smokeTest = require('./test/smoke/js/SmokeTests')
 const ContentTypeMapper = require('./app/js/ContentTypeMapper')
 const Errors = require('./app/js/Errors')
 
@@ -49,31 +49,29 @@ app.use(function(req, res, next) {
   return next()
 })
 
-app.param('project_id', function(req, res, next, project_id) {
-  if (project_id != null ? project_id.match(/^[a-zA-Z0-9_-]+$/) : undefined) {
+app.param('project_id', function(req, res, next, projectId) {
+  if (projectId != null ? projectId.match(/^[a-zA-Z0-9_-]+$/) : undefined) {
     return next()
   } else {
     return next(new Error('invalid project id'))
   }
 })
 
-app.param('user_id', function(req, res, next, user_id) {
-  if (user_id != null ? user_id.match(/^[0-9a-f]{24}$/) : undefined) {
+app.param('user_id', function(req, res, next, userId) {
+  if (userId != null ? userId.match(/^[0-9a-f]{24}$/) : undefined) {
     return next()
   } else {
     return next(new Error('invalid user id'))
   }
 })
 
-app.param('build_id', function(req, res, next, build_id) {
+app.param('build_id', function(req, res, next, buildId) {
   if (
-    build_id != null
-      ? build_id.match(OutputCacheManager.BUILD_REGEX)
-      : undefined
+    buildId != null ? buildId.match(OutputCacheManager.BUILD_REGEX) : undefined
   ) {
     return next()
   } else {
-    return next(new Error(`invalid build id ${build_id}`))
+    return next(new Error(`invalid build id ${buildId}`))
   }
 })
 
@@ -192,69 +190,49 @@ app.get('/oops', function(req, res, next) {
 
 app.get('/status', (req, res, next) => res.send('CLSI is alive\n'))
 
-const resCacher = {
-  contentType(setContentType) {
-    this.setContentType = setContentType
-  },
-  send(code, body) {
-    this.code = code
-    this.body = body
-  },
-
-  // default the server to be down
-  code: 500,
-  body: {},
-  setContentType: 'application/json'
-}
-
-let shutdownTime
+Settings.processTooOld = false
 if (Settings.processLifespanLimitMs) {
   Settings.processLifespanLimitMs +=
     Settings.processLifespanLimitMs * (Math.random() / 10)
-  shutdownTime = Date.now() + Settings.processLifespanLimitMs
-  logger.info('Lifespan limited to ', shutdownTime)
-}
+  logger.info(
+    'Lifespan limited to ',
+    Date.now() + Settings.processLifespanLimitMs
+  )
 
-const checkIfProcessIsTooOld = function(cont) {
-  if (shutdownTime && shutdownTime < Date.now()) {
+  setTimeout(() => {
     logger.log('shutting down, process is too old')
-    resCacher.send = function() {}
-    resCacher.code = 500
-    resCacher.body = { processToOld: true }
-  } else {
-    cont()
-  }
+    Settings.processTooOld = true
+  }, Settings.processLifespanLimitMs)
 }
 
+function runSmokeTest() {
+  if (Settings.processTooOld) return
+  logger.log('running smoke tests')
+  smokeTest.triggerRun(err => {
+    if (err) logger.error({ err }, 'smoke tests failed')
+    setTimeout(runSmokeTest, 30 * 1000)
+  })
+}
 if (Settings.smokeTest) {
-  const runSmokeTest = function() {
-    checkIfProcessIsTooOld(function() {
-      logger.log('running smoke tests')
-      smokeTest.run(
-        require.resolve(__dirname + '/test/smoke/js/SmokeTests.js')
-      )({}, resCacher)
-      return setTimeout(runSmokeTest, 30 * 1000)
-    })
-  }
   runSmokeTest()
 }
 
 app.get('/health_check', function(req, res) {
-  res.contentType(resCacher.setContentType)
-  return res.status(resCacher.code).send(resCacher.body)
+  if (Settings.processTooOld) {
+    return res.status(500).json({ processTooOld: true })
+  }
+  smokeTest.sendLastResult(res)
 })
 
-app.get('/smoke_test_force', (req, res) =>
-  smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))(
-    req,
-    res
-  )
-)
+app.get('/smoke_test_force', (req, res) => smokeTest.sendNewResult(res))
 
 app.use(function(error, req, res, next) {
   if (error instanceof Errors.NotFoundError) {
-    logger.warn({ err: error, url: req.url }, 'not found error')
+    logger.log({ err: error, url: req.url }, 'not found error')
     return res.sendStatus(404)
+  } else if (error.code === 'EPIPE') {
+    // inspect container returns EPIPE when shutting down
+    return res.sendStatus(503) // send 503 Unavailable response
   } else {
     logger.error({ err: error, url: req.url }, 'server error')
     return res.sendStatus((error != null ? error.statusCode : undefined) || 500)
@@ -331,38 +309,40 @@ const host =
     x1 => x1.host
   ) || 'localhost'
 
-const load_tcp_port = Settings.internal.load_balancer_agent.load_port
-const load_http_port = Settings.internal.load_balancer_agent.local_port
+const loadTcpPort = Settings.internal.load_balancer_agent.load_port
+const loadHttpPort = Settings.internal.load_balancer_agent.local_port
 
 if (!module.parent) {
   // Called directly
-  app.listen(port, host, error =>
-    logger.info(`CLSI starting up, listening on ${host}:${port}`)
-  )
-
-  loadTcpServer.listen(load_tcp_port, host, function(error) {
-    if (error != null) {
-      throw error
+  app.listen(port, host, error => {
+    if (error) {
+      logger.fatal({ error }, `Error starting CLSI on ${host}:${port}`)
+    } else {
+      logger.info(`CLSI starting up, listening on ${host}:${port}`)
     }
-    return logger.info(`Load tcp agent listening on load port ${load_tcp_port}`)
   })
 
-  loadHttpServer.listen(load_http_port, host, function(error) {
+  loadTcpServer.listen(loadTcpPort, host, function(error) {
     if (error != null) {
       throw error
     }
-    return logger.info(
-      `Load http agent listening on load port ${load_http_port}`
-    )
+    return logger.info(`Load tcp agent listening on load port ${loadTcpPort}`)
+  })
+
+  loadHttpServer.listen(loadHttpPort, host, function(error) {
+    if (error != null) {
+      throw error
+    }
+    return logger.info(`Load http agent listening on load port ${loadHttpPort}`)
   })
 }
 
 module.exports = app
 
-setInterval(
-  () => ProjectPersistenceManager.clearExpiredProjects(),
-  (tenMinutes = 10 * 60 * 1000)
-)
+setInterval(() => {
+  ProjectPersistenceManager.refreshExpiryTimeout()
+  ProjectPersistenceManager.clearExpiredProjects()
+}, tenMinutes)
 
 function __guard__(value, transform) {
   return typeof value !== 'undefined' && value !== null

app/js/CompileController.js

@@ -55,6 +55,10 @@ module.exports = CompileController = {
             } else if (error instanceof Errors.FilesOutOfSyncError) {
               code = 409 // Http 409 Conflict
               status = 'retry'
+            } else if (error && error.code === 'EPIPE') {
+              // docker returns EPIPE when shutting down
+              code = 503 // send 503 Unavailable response
+              status = 'unavailable'
             } else if (error != null ? error.terminated : undefined) {
               status = 'terminated'
             } else if (error != null ? error.validate : undefined) {
@@ -214,6 +218,15 @@ module.exports = CompileController = {
     const { project_id } = req.params
     const { user_id } = req.params
     const { image } = req.query
+    if (
+      image &&
+      Settings.clsi &&
+      Settings.clsi.docker &&
+      Settings.clsi.docker.allowedImages &&
+      !Settings.clsi.docker.allowedImages.includes(image)
+    ) {
+      return res.status(400).send('invalid image')
+    }
     logger.log({ image, file, project_id }, 'word count request')
 
     return CompileManager.wordcount(project_id, user_id, file, image, function(

app/js/CompileManager.js

@@ -142,6 +142,10 @@ module.exports = CompileManager = {
         )
       // set up environment variables for chktex
       const env = {}
+      if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') {
+        // override default texlive openout_any environment variable
+        env.openout_any = Settings.texliveOpenoutAny
+      }
       // only run chktex on LaTeX files (not knitr .Rtex files or any others)
       const isLaTeXFile =
         request.rootResourcePath != null
@@ -195,7 +199,8 @@ module.exports = CompileManager = {
               timeout: request.timeout,
               image: request.imageName,
               flags: request.flags,
-              environment: env
+              environment: env,
+              compileGroup: request.compileGroup
             },
             function(error, output, stats, timings) {
               // request was for validation only
@@ -334,7 +339,7 @@ module.exports = CompileManager = {
       proc.on('error', callback)
 
       let stderr = ''
-      proc.stderr.on('data', chunk => (stderr += chunk.toString()))
+      proc.stderr.setEncoding('utf8').on('data', chunk => (stderr += chunk))
 
       return proc.on('close', function(code) {
         if (code === 0) {
@@ -426,30 +431,18 @@ module.exports = CompileManager = {
     const compileDir = getCompileDir(project_id, user_id)
     const synctex_path = `${base_dir}/output.pdf`
     const command = ['code', synctex_path, file_path, line, column]
-    return fse.ensureDir(compileDir, function(error) {
+    CompileManager._runSynctex(project_id, user_id, command, function(
+      error,
+      stdout
+    ) {
       if (error != null) {
-        logger.err(
-          { error, project_id, user_id, file_name },
-          'error ensuring dir for sync from code'
-        )
         return callback(error)
       }
-      return CompileManager._runSynctex(project_id, user_id, command, function(
-        error,
-        stdout
-      ) {
-        if (error != null) {
-          return callback(error)
-        }
-        logger.log(
-          { project_id, user_id, file_name, line, column, command, stdout },
-          'synctex code output'
-        )
-        return callback(
-          null,
-          CompileManager._parseSynctexFromCodeOutput(stdout)
-        )
-      })
+      logger.log(
+        { project_id, user_id, file_name, line, column, command, stdout },
+        'synctex code output'
+      )
+      return callback(null, CompileManager._parseSynctexFromCodeOutput(stdout))
     })
   },
 
@@ -462,53 +455,39 @@ module.exports = CompileManager = {
     const base_dir = Settings.path.synctexBaseDir(compileName)
     const synctex_path = `${base_dir}/output.pdf`
     const command = ['pdf', synctex_path, page, h, v]
-    return fse.ensureDir(compileDir, function(error) {
+    CompileManager._runSynctex(project_id, user_id, command, function(
+      error,
+      stdout
+    ) {
       if (error != null) {
-        logger.err(
-          { error, project_id, user_id, file_name },
-          'error ensuring dir for sync to code'
-        )
         return callback(error)
       }
-      return CompileManager._runSynctex(project_id, user_id, command, function(
-        error,
-        stdout
-      ) {
-        if (error != null) {
-          return callback(error)
-        }
-        logger.log(
-          { project_id, user_id, page, h, v, stdout },
-          'synctex pdf output'
-        )
-        return callback(
-          null,
-          CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
-        )
-      })
+      logger.log(
+        { project_id, user_id, page, h, v, stdout },
+        'synctex pdf output'
+      )
+      return callback(
+        null,
+        CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
+      )
     })
   },
 
-  _checkFileExists(path, callback) {
+  _checkFileExists(dir, filename, callback) {
     if (callback == null) {
       callback = function(error) {}
     }
-    const synctexDir = Path.dirname(path)
-    const synctexFile = Path.join(synctexDir, 'output.synctex.gz')
-    return fs.stat(synctexDir, function(error, stats) {
+    const file = Path.join(dir, filename)
+    return fs.stat(dir, function(error, stats) {
       if ((error != null ? error.code : undefined) === 'ENOENT') {
-        return callback(
-          new Errors.NotFoundError('called synctex with no output directory')
-        )
+        return callback(new Errors.NotFoundError('no output directory'))
       }
       if (error != null) {
         return callback(error)
       }
-      return fs.stat(synctexFile, function(error, stats) {
+      return fs.stat(file, function(error, stats) {
         if ((error != null ? error.code : undefined) === 'ENOENT') {
-          return callback(
-            new Errors.NotFoundError('called synctex with no output file')
-          )
+          return callback(new Errors.NotFoundError('no output file'))
         }
         if (error != null) {
           return callback(error)
@@ -532,24 +511,33 @@ module.exports = CompileManager = {
     const directory = getCompileDir(project_id, user_id)
     const timeout = 60 * 1000 // increased to allow for large projects
     const compileName = getCompileName(project_id, user_id)
-    return CommandRunner.run(
-      compileName,
-      command,
-      directory,
-      Settings.clsi != null ? Settings.clsi.docker.image : undefined,
-      timeout,
-      {},
-      function(error, output) {
-        if (error != null) {
-          logger.err(
-            { err: error, command, project_id, user_id },
-            'error running synctex'
-          )
-          return callback(error)
-        }
-        return callback(null, output.stdout)
+    const compileGroup = 'synctex'
+    CompileManager._checkFileExists(directory, 'output.synctex.gz', error => {
+      if (error) {
+        return callback(error)
       }
-    )
+      return CommandRunner.run(
+        compileName,
+        command,
+        directory,
+        Settings.clsi && Settings.clsi.docker
+          ? Settings.clsi.docker.image
+          : undefined,
+        timeout,
+        {},
+        compileGroup,
+        function(error, output) {
+          if (error != null) {
+            logger.err(
+              { err: error, command, project_id, user_id },
+              'error running synctex'
+            )
+            return callback(error)
+          }
+          return callback(null, output.stdout)
+        }
+      )
+    })
   },
 
   _parseSynctexFromCodeOutput(output) {
@@ -602,6 +590,7 @@ module.exports = CompileManager = {
     const compileDir = getCompileDir(project_id, user_id)
     const timeout = 60 * 1000
     const compileName = getCompileName(project_id, user_id)
+    const compileGroup = 'wordcount'
     return fse.ensureDir(compileDir, function(error) {
       if (error != null) {
         logger.err(
@@ -617,6 +606,7 @@ module.exports = CompileManager = {
         image,
         timeout,
         {},
+        compileGroup,
         function(error) {
           if (error != null) {
             return callback(error)

app/js/DockerRunner.js

@@ -25,7 +25,7 @@ const async = require('async')
 const LockManager = require('./DockerLockManager')
 const fs = require('fs')
 const Path = require('path')
-const _ = require('underscore')
+const _ = require('lodash')
 
 logger.info('using docker runner')
 
@@ -44,7 +44,16 @@ module.exports = DockerRunner = {
   ERR_EXITED: new Error('exited'),
   ERR_TIMED_OUT: new Error('container timed out'),
 
-  run(project_id, command, directory, image, timeout, environment, callback) {
+  run(
+    project_id,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    callback
+  ) {
     let name
     if (callback == null) {
       callback = function(error, output) {}
@@ -77,6 +86,13 @@ module.exports = DockerRunner = {
       ;({ image } = Settings.clsi.docker)
     }
 
+    if (
+      Settings.clsi.docker.allowedImages &&
+      !Settings.clsi.docker.allowedImages.includes(image)
+    ) {
+      return callback(new Error('image not allowed'))
+    }
+
     if (Settings.texliveImageNameOveride != null) {
       const img = image.split('/')
       image = `${Settings.texliveImageNameOveride}/${img[2]}`
@@ -87,7 +103,8 @@ module.exports = DockerRunner = {
       image,
       volumes,
       timeout,
-      environment
+      environment,
+      compileGroup
     )
     const fingerprint = DockerRunner._fingerprintContainer(options)
     options.name = name = `project-${project_id}-${fingerprint}`
@@ -223,7 +240,14 @@ module.exports = DockerRunner = {
     )
   },
 
-  _getContainerOptions(command, image, volumes, timeout, environment) {
+  _getContainerOptions(
+    command,
+    image,
+    volumes,
+    timeout,
+    environment,
+    compileGroup
+  ) {
     let m, year
     let key, value, hostVol, dockerVol
     const timeoutInSeconds = timeout / 1000
@@ -310,6 +334,23 @@ module.exports = DockerRunner = {
       options.HostConfig.Runtime = Settings.clsi.docker.runtime
     }
 
+    if (Settings.clsi.docker.Readonly) {
+      options.HostConfig.ReadonlyRootfs = true
+      options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' }
+    }
+
+    // Allow per-compile group overriding of individual settings
+    if (
+      Settings.clsi.docker.compileGroupConfig &&
+      Settings.clsi.docker.compileGroupConfig[compileGroup]
+    ) {
+      const override = Settings.clsi.docker.compileGroupConfig[compileGroup]
+      let key
+      for (key in override) {
+        _.set(options, key, override[key])
+      }
+    }
+
     return options
   },
 
@@ -632,6 +673,9 @@ module.exports = DockerRunner = {
             ttl
           ) {
             if (name.slice(0, 9) === '/project-' && ttl <= 0) {
+              // strip the / prefix
+              // the LockManager uses the plain container name
+              name = name.slice(1)
               return jobs.push(cb =>
                 DockerRunner.destroyContainer(name, id, false, () => cb())
               )

app/js/LatexRunner.js

@@ -19,6 +19,7 @@ const Settings = require('settings-sharelatex')
 const logger = require('logger-sharelatex')
 const Metrics = require('./Metrics')
 const CommandRunner = require('./CommandRunner')
+const fs = require('fs')
 
 const ProcessTable = {} // table of currently running jobs (pids or docker container names)
 
@@ -35,7 +36,8 @@ module.exports = LatexRunner = {
       timeout,
       image,
       environment,
-      flags
+      flags,
+      compileGroup
     } = options
     if (!compiler) {
       compiler = 'pdflatex'
@@ -45,7 +47,15 @@ module.exports = LatexRunner = {
     } // milliseconds
 
     logger.log(
-      { directory, compiler, timeout, mainFile, environment, flags },
+      {
+        directory,
+        compiler,
+        timeout,
+        mainFile,
+        environment,
+        flags,
+        compileGroup
+      },
       'starting compile'
     )
 
@@ -78,6 +88,7 @@ module.exports = LatexRunner = {
       image,
       timeout,
       environment,
+      compileGroup,
       function(error, output) {
         delete ProcessTable[id]
         if (error != null) {
@@ -127,11 +138,39 @@ module.exports = LatexRunner = {
               : undefined,
             x5 => x5[1]
           ) || 0
-        return callback(error, output, stats, timings)
+        // record output files
+        LatexRunner.writeLogOutput(project_id, directory, output, () => {
+          return callback(error, output, stats, timings)
+        })
       }
     ))
   },
 
+  writeLogOutput(project_id, directory, output, callback) {
+    if (!output) {
+      return callback()
+    }
+    // internal method for writing non-empty log files
+    function _writeFile(file, content, cb) {
+      if (content && content.length > 0) {
+        fs.writeFile(file, content, err => {
+          if (err) {
+            logger.error({ project_id, file }, 'error writing log file') // don't fail on error
+          }
+          cb()
+        })
+      } else {
+        cb()
+      }
+    }
+    // write stdout and stderr, ignoring errors
+    _writeFile(Path.join(directory, 'output.stdout'), output.stdout, () => {
+      _writeFile(Path.join(directory, 'output.stderr'), output.stderr, () => {
+        callback()
+      })
+    })
+  },
+
   killLatex(project_id, callback) {
     if (callback == null) {
       callback = function(error) {}

app/js/LocalCommandRunner.js

@@ -15,15 +15,27 @@
  */
 let CommandRunner
 const { spawn } = require('child_process')
+const _ = require('underscore')
 const logger = require('logger-sharelatex')
 
 logger.info('using standard command runner')
 
 module.exports = CommandRunner = {
-  run(project_id, command, directory, image, timeout, environment, callback) {
+  run(
+    project_id,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    callback
+  ) {
     let key, value
     if (callback == null) {
       callback = function(error) {}
+    } else {
+      callback = _.once(callback)
     }
     command = Array.from(command).map(arg =>
       arg.toString().replace('$COMPILE_DIR', directory)
@@ -46,7 +58,7 @@ module.exports = CommandRunner = {
     const proc = spawn(command[0], command.slice(1), { cwd: directory, env })
 
     let stdout = ''
-    proc.stdout.on('data', data => (stdout += data))
+    proc.stdout.setEncoding('utf8').on('data', data => (stdout += data))
 
     proc.on('error', function(err) {
       logger.err(

app/js/OutputCacheManager.js

@@ -19,7 +19,7 @@ const fs = require('fs')
 const fse = require('fs-extra')
 const Path = require('path')
 const logger = require('logger-sharelatex')
-const _ = require('underscore')
+const _ = require('lodash')
 const Settings = require('settings-sharelatex')
 const crypto = require('crypto')
 

app/js/OutputFileFinder.js

@@ -87,7 +87,7 @@ module.exports = OutputFileFinder = {
 
     const proc = spawn('find', args)
     let stdout = ''
-    proc.stdout.on('data', chunk => (stdout += chunk.toString()))
+    proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
     proc.on('error', callback)
     return proc.on('close', function(code) {
       if (code !== 0) {

app/js/OutputFileOptimiser.js

@@ -19,7 +19,7 @@ const Path = require('path')
 const { spawn } = require('child_process')
 const logger = require('logger-sharelatex')
 const Metrics = require('./Metrics')
-const _ = require('underscore')
+const _ = require('lodash')
 
 module.exports = OutputFileOptimiser = {
   optimiseFile(src, dst, callback) {
@@ -45,8 +45,7 @@ module.exports = OutputFileOptimiser = {
 
   checkIfPDFIsOptimised(file, callback) {
     const SIZE = 16 * 1024 // check the header of the pdf
-    const result = new Buffer(SIZE)
-    result.fill(0) // prevent leakage of uninitialised buffer
+    const result = Buffer.alloc(SIZE) // fills with zeroes by default
     return fs.open(file, 'r', function(err, fd) {
       if (err != null) {
         return callback(err)
@@ -78,7 +77,7 @@ module.exports = OutputFileOptimiser = {
     const timer = new Metrics.Timer('qpdf')
     const proc = spawn('qpdf', args)
     let stdout = ''
-    proc.stdout.on('data', chunk => (stdout += chunk.toString()))
+    proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
     callback = _.once(callback) // avoid double call back for error and close event
     proc.on('error', function(err) {
       logger.warn({ err, args }, 'qpdf failed')

app/js/ProjectPersistenceManager.js

@@ -20,10 +20,32 @@ const async = require('async')
 const logger = require('logger-sharelatex')
 const oneDay = 24 * 60 * 60 * 1000
 const Settings = require('settings-sharelatex')
+const diskusage = require('diskusage')
 
 module.exports = ProjectPersistenceManager = {
   EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
 
+  refreshExpiryTimeout(callback) {
+    if (callback == null) {
+      callback = function(error) {}
+    }
+    diskusage.check('/', function(err, stats) {
+      if (err) {
+        logger.err({ err: err }, 'error getting disk usage')
+        return callback(err)
+      }
+      const lowDisk = stats.available / stats.total < 0.1
+      const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
+      if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
+        logger.warn(
+          { stats: stats },
+          'disk running low on space, modifying EXPIRY_TIMEOUT'
+        )
+        ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
+      }
+      callback()
+    })
+  },
   markProjectAsJustAccessed(project_id, callback) {
     if (callback == null) {
       callback = function(error) {}

app/js/RequestParser.js

@@ -61,7 +61,13 @@ module.exports = RequestParser = {
       response.imageName = this._parseAttribute(
         'imageName',
         compile.options.imageName,
-        { type: 'string' }
+        {
+          type: 'string',
+          validValues:
+            settings.clsi &&
+            settings.clsi.docker &&
+            settings.clsi.docker.allowedImages
+        }
       )
       response.draft = this._parseAttribute('draft', compile.options.draft, {
         default: false,
@@ -74,7 +80,17 @@ module.exports = RequestParser = {
         default: [],
         type: 'object'
       })
-
+      if (settings.allowedCompileGroups) {
+        response.compileGroup = this._parseAttribute(
+          'compileGroup',
+          compile.options.compileGroup,
+          {
+            validValues: settings.allowedCompileGroups,
+            default: '',
+            type: 'string'
+          }
+        )
+      }
       // The syncType specifies whether the request contains all
       // resources (full) or only those resources to be updated
       // in-place (incremental).

app/js/ResourceWriter.js

@@ -231,7 +231,9 @@ module.exports = ResourceWriter = {
             path === 'output.pdf' ||
             path === 'output.dvi' ||
             path === 'output.log' ||
-            path === 'output.xdv'
+            path === 'output.xdv' ||
+            path === 'output.stdout' ||
+            path === 'output.stderr'
           ) {
             should_delete = true
           }

app/js/SafeReader.js

@@ -43,7 +43,7 @@ module.exports = SafeReader = {
           }
           return callback(null, ...Array.from(result))
         })
-      const buff = new Buffer(size, 0) // fill with zeros
+      const buff = Buffer.alloc(size) // fills with zeroes by default
       return fs.read(fd, buff, 0, buff.length, 0, function(
         err,
         bytesRead,

app/js/UrlCache.js

@@ -95,7 +95,7 @@ module.exports = UrlCache = {
         }
         if (needsDownloading) {
           logger.log({ url, lastModified }, 'downloading URL')
-          return UrlFetcher.pipeUrlToFile(
+          return UrlFetcher.pipeUrlToFileWithRetry(
             url,
             UrlCache._cacheFilePathForUrl(project_id, url),
             error => {

app/js/UrlFetcher.js

@@ -18,10 +18,18 @@ const fs = require('fs')
 const logger = require('logger-sharelatex')
 const settings = require('settings-sharelatex')
 const URL = require('url')
+const async = require('async')
 
 const oneMinute = 60 * 1000
 
 module.exports = UrlFetcher = {
+  pipeUrlToFileWithRetry(url, filePath, callback) {
+    const doDownload = function(cb) {
+      UrlFetcher.pipeUrlToFile(url, filePath, cb)
+    }
+    async.retry(3, doDownload, callback)
+  },
+
   pipeUrlToFile(url, filePath, _callback) {
     if (_callback == null) {
       _callback = function(error) {}

app/js/db.js

@@ -10,7 +10,7 @@
  */
 const Sequelize = require('sequelize')
 const Settings = require('settings-sharelatex')
-const _ = require('underscore')
+const _ = require('lodash')
 const logger = require('logger-sharelatex')
 
 const options = _.extend({ logging: false }, Settings.mysql.clsi)

buildscript.txt

@@ -6,6 +6,6 @@ clsi
 --env-add=
 --env-pass-through=TEXLIVE_IMAGE
 --language=es
---node-version=10.19.0
+--node-version=10.21.0
 --public-repo=True
 --script-version=2.1.0

config/settings.defaults.js

@@ -9,7 +9,7 @@ module.exports = {
       username: 'clsi',
       dialect: 'sqlite',
       storage:
-        process.env.SQLITE_PATH || Path.resolve(__dirname + '/../db/db.sqlite'),
+        process.env.SQLITE_PATH || Path.resolve(__dirname, '../db/db.sqlite'),
       pool: {
         max: 1,
         min: 1
@@ -26,10 +26,10 @@ module.exports = {
     parseInt(process.env.PROCESS_LIFE_SPAN_LIMIT_MS) || 60 * 60 * 24 * 1000 * 2,
 
   path: {
-    compilesDir: Path.resolve(__dirname + '/../compiles'),
-    clsiCacheDir: Path.resolve(__dirname + '/../cache'),
-    synctexBaseDir(project_id) {
-      return Path.join(this.compilesDir, project_id)
+    compilesDir: Path.resolve(__dirname, '../compiles'),
+    clsiCacheDir: Path.resolve(__dirname, '../cache'),
+    synctexBaseDir(projectId) {
+      return Path.join(this.compilesDir, projectId)
     }
   },
 
@@ -57,13 +57,25 @@ module.exports = {
   parallelSqlQueryLimit: process.env.FILESTORE_PARALLEL_SQL_QUERY_LIMIT || 1,
   filestoreDomainOveride: process.env.FILESTORE_DOMAIN_OVERRIDE,
   texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
+  texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
   sentry: {
     dsn: process.env.SENTRY_DSN
   }
 }
 
+if (process.env.ALLOWED_COMPILE_GROUPS) {
+  try {
+    module.exports.allowedCompileGroups = process.env.ALLOWED_COMPILE_GROUPS.split(
+      ' '
+    )
+  } catch (error) {
+    console.error(error, 'could not apply allowed compile group setting')
+    process.exit(1)
+  }
+}
+
 if (process.env.DOCKER_RUNNER) {
-  let seccomp_profile_path
+  let seccompProfilePath
   module.exports.clsi = {
     dockerRunner: process.env.DOCKER_RUNNER === 'true',
     docker: {
@@ -76,22 +88,56 @@ if (process.env.DOCKER_RUNNER) {
       socketPath: '/var/run/docker.sock',
       user: process.env.TEXLIVE_IMAGE_USER || 'tex'
     },
+    optimiseInDocker: true,
     expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
     checkProjectsIntervalMs: 10 * 60 * 1000
   }
 
   try {
-    seccomp_profile_path = Path.resolve(
-      __dirname + '/../seccomp/clsi-profile.json'
+    // Override individual docker settings using path-based keys, e.g.:
+    // compileGroupDockerConfigs = {
+    //    priority: { 'HostConfig.CpuShares': 100 }
+    //    beta: { 'dotted.path.here', 'value'}
+    // }
+    const compileGroupConfig = JSON.parse(
+      process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}'
     )
-    module.exports.clsi.docker.seccomp_profile = JSON.stringify(
-      JSON.parse(require('fs').readFileSync(seccomp_profile_path))
+    // Automatically clean up wordcount and synctex containers
+    const defaultCompileGroupConfig = {
+      wordcount: { 'HostConfig.AutoRemove': true },
+      synctex: { 'HostConfig.AutoRemove': true }
+    }
+    module.exports.clsi.docker.compileGroupConfig = Object.assign(
+      defaultCompileGroupConfig,
+      compileGroupConfig
     )
   } catch (error) {
-    console.log(
-      error,
-      `could not load seccom profile from ${seccomp_profile_path}`
+    console.error(error, 'could not apply compile group docker configs')
+    process.exit(1)
+  }
+
+  try {
+    seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
+    module.exports.clsi.docker.seccomp_profile = JSON.stringify(
+      JSON.parse(require('fs').readFileSync(seccompProfilePath))
     )
+  } catch (error) {
+    console.error(
+      error,
+      `could not load seccomp profile from ${seccompProfilePath}`
+    )
+    process.exit(1)
+  }
+
+  if (process.env.ALLOWED_IMAGES) {
+    try {
+      module.exports.clsi.docker.allowedImages = process.env.ALLOWED_IMAGES.split(
+        ' '
+      )
+    } catch (error) {
+      console.error(error, 'could not apply allowed images setting')
+      process.exit(1)
+    }
   }
 
   module.exports.path.synctexBaseDir = () => '/compile'

docker-compose-config.yml

@@ -3,6 +3,7 @@ version: "2.3"
 services:
   dev:
     environment:
+      ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1"
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
       TEXLIVE_IMAGE_USER: "tex"
       SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
@@ -18,6 +19,7 @@ services:
       
   ci:
     environment:
+      ALLOWED_IMAGES: ${TEXLIVE_IMAGE}
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
       TEXLIVE_IMAGE_USER: "tex"
       SHARELATEX_CONFIG: /app/config/settings.defaults.coffee

package.json

@@ -21,21 +21,21 @@
   "dependencies": {
     "async": "3.2.0",
     "body-parser": "^1.19.0",
+    "diskusage": "^1.1.3",
     "dockerode": "^3.1.0",
     "express": "^4.17.1",
     "fs-extra": "^8.1.0",
     "heapdump": "^0.3.15",
     "lockfile": "^1.0.4",
-    "logger-sharelatex": "^1.9.1",
+    "lodash": "^4.17.15",
+    "logger-sharelatex": "git+https://github.com/overleaf/logger-module.git#csh-metadata-fallback-node-fetch",
     "lynx": "0.2.0",
     "metrics-sharelatex": "^2.6.0",
     "mysql": "^2.18.1",
     "request": "^2.88.2",
     "sequelize": "^5.21.5",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0",
-    "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
     "sqlite3": "^4.1.1",
-    "underscore": "^1.9.2",
     "v8-profiler-node8": "^6.1.1",
     "wrench": "~1.5.9"
   },

test/acceptance/js/AllowedImageNames.js

@@ -0,0 +1,102 @@
+const Client = require('./helpers/Client')
+const ClsiApp = require('./helpers/ClsiApp')
+const { expect } = require('chai')
+
+describe('AllowedImageNames', function() {
+  beforeEach(function(done) {
+    this.project_id = Client.randomId()
+    this.request = {
+      options: {
+        imageName: undefined
+      },
+      resources: [
+        {
+          path: 'main.tex',
+          content: `\
+\\documentclass{article}
+\\begin{document}
+Hello world
+\\end{document}\
+`
+        }
+      ]
+    }
+    ClsiApp.ensureRunning(done)
+  })
+
+  describe('with a valid name', function() {
+    beforeEach(function(done) {
+      this.request.options.imageName = process.env.TEXLIVE_IMAGE
+
+      Client.compile(this.project_id, this.request, (error, res, body) => {
+        this.error = error
+        this.res = res
+        this.body = body
+        done(error)
+      })
+    })
+    it('should return success', function() {
+      expect(this.res.statusCode).to.equal(200)
+    })
+
+    it('should return a PDF', function() {
+      let pdf
+      try {
+        pdf = Client.getOutputFile(this.body, 'pdf')
+      } catch (e) {}
+      expect(pdf).to.exist
+    })
+  })
+
+  describe('with an invalid name', function() {
+    beforeEach(function(done) {
+      this.request.options.imageName = 'something/evil:1337'
+      Client.compile(this.project_id, this.request, (error, res, body) => {
+        this.error = error
+        this.res = res
+        this.body = body
+        done(error)
+      })
+    })
+    it('should return non success', function() {
+      expect(this.res.statusCode).to.not.equal(200)
+    })
+
+    it('should not return a PDF', function() {
+      let pdf
+      try {
+        pdf = Client.getOutputFile(this.body, 'pdf')
+      } catch (e) {}
+      expect(pdf).to.not.exist
+    })
+  })
+
+  describe('wordcount', function() {
+    beforeEach(function(done) {
+      Client.compile(this.project_id, this.request, done)
+    })
+    it('should error out with an invalid imageName', function() {
+      Client.wordcountWithImage(
+        this.project_id,
+        'main.tex',
+        'something/evil:1337',
+        (error, result) => {
+          expect(String(error)).to.include('statusCode=400')
+        }
+      )
+    })
+
+    it('should produce a texcout a valid imageName', function() {
+      Client.wordcountWithImage(
+        this.project_id,
+        'main.tex',
+        process.env.TEXLIVE_IMAGE,
+        (error, result) => {
+          expect(error).to.not.exist
+          expect(result).to.exist
+          expect(result.texcount).to.exist
+        }
+      )
+    })
+  })
+})

test/acceptance/js/ExampleDocumentTests.js

@@ -235,6 +235,7 @@ describe('Example Documents', function() {
                 ) === 'failure'
               ) {
                 console.log('DEBUG: error', error, 'body', JSON.stringify(body))
+                return done(new Error('Compile failed'))
               }
               const pdf = Client.getOutputFile(body, 'pdf')
               return downloadAndComparePdf(
@@ -263,6 +264,7 @@ describe('Example Documents', function() {
                 ) === 'failure'
               ) {
                 console.log('DEBUG: error', error, 'body', JSON.stringify(body))
+                return done(new Error('Compile failed'))
               }
               const pdf = Client.getOutputFile(body, 'pdf')
               return downloadAndComparePdf(

test/acceptance/js/SynctexTests.js

@@ -69,7 +69,7 @@ Hello world
     })
   })
 
-  return describe('from pdf to code', function() {
+  describe('from pdf to code', function() {
     return it('should return the correct location', function(done) {
       return Client.syncFromPdf(
         this.project_id,
@@ -88,4 +88,104 @@ Hello world
       )
     })
   })
+
+  describe('when the project directory is not available', function() {
+    before(function() {
+      this.other_project_id = Client.randomId()
+    })
+    describe('from code to pdf', function() {
+      it('should return a 404 response', function(done) {
+        return Client.syncFromCode(
+          this.other_project_id,
+          'main.tex',
+          3,
+          5,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+    describe('from pdf to code', function() {
+      it('should return a 404 response', function(done) {
+        return Client.syncFromPdf(
+          this.other_project_id,
+          1,
+          100,
+          200,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+  })
+
+  describe('when the synctex file is not available', function() {
+    before(function(done) {
+      this.broken_project_id = Client.randomId()
+      const content = 'this is not valid tex' // not a valid tex file
+      this.request = {
+        resources: [
+          {
+            path: 'main.tex',
+            content
+          }
+        ]
+      }
+      Client.compile(
+        this.broken_project_id,
+        this.request,
+        (error, res, body) => {
+          this.error = error
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    describe('from code to pdf', function() {
+      it('should return a 404 response', function(done) {
+        return Client.syncFromCode(
+          this.broken_project_id,
+          'main.tex',
+          3,
+          5,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+    describe('from pdf to code', function() {
+      it('should return a 404 response', function(done) {
+        return Client.syncFromPdf(
+          this.broken_project_id,
+          1,
+          100,
+          200,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+  })
 })

test/acceptance/js/UrlCachingTests.js

@@ -11,7 +11,6 @@
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
 const Client = require('./helpers/Client')
-const request = require('request')
 require('chai').should()
 const sinon = require('sinon')
 const ClsiApp = require('./helpers/ClsiApp')

test/acceptance/js/helpers/Client.js

@@ -81,13 +81,14 @@ module.exports = Client = {
           file,
           line,
           column
-        }
+        },
+        json: true
       },
       (error, response, body) => {
         if (error != null) {
           return callback(error)
         }
-        return callback(null, JSON.parse(body))
+        return callback(null, body)
       }
     )
   },
@@ -103,13 +104,14 @@ module.exports = Client = {
           page,
           h,
           v
-        }
+        },
+        json: true
       },
       (error, response, body) => {
         if (error != null) {
           return callback(error)
         }
-        return callback(null, JSON.parse(body))
+        return callback(null, body)
       }
     )
   },
@@ -187,6 +189,11 @@ module.exports = Client = {
   },
 
   wordcount(project_id, file, callback) {
+    const image = undefined
+    Client.wordcountWithImage(project_id, file, image, callback)
+  },
+
+  wordcountWithImage(project_id, file, image, callback) {
     if (callback == null) {
       callback = function(error, pdfPositions) {}
     }
@@ -194,6 +201,7 @@ module.exports = Client = {
       {
         url: `${this.host}/project/${project_id}/wordcount`,
         qs: {
+          image,
           file
         }
       },
@@ -201,6 +209,9 @@ module.exports = Client = {
         if (error != null) {
           return callback(error)
         }
+        if (response.statusCode !== 200) {
+          return callback(new Error(`statusCode=${response.statusCode}`))
+        }
         return callback(null, JSON.parse(body))
       }
     )

test/load/js/loadTest.js

@@ -13,7 +13,7 @@ const request = require('request')
 const Settings = require('settings-sharelatex')
 const async = require('async')
 const fs = require('fs')
-const _ = require('underscore')
+const _ = require('lodash')
 const concurentCompiles = 5
 const totalCompiles = 50
 

test/smoke/js/SmokeTests.js

@@ -1,20 +1,3 @@
-/* eslint-disable
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS101: Remove unnecessary use of Array.from
- * DS102: Remove unnecessary code created because of implicit returns
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-const chai = require('chai')
-if (Object.prototype.should == null) {
-  chai.should()
-}
-const { expect } = chai
 const request = require('request')
 const Settings = require('settings-sharelatex')
 
@@ -23,9 +6,35 @@ const buildUrl = path =>
 
 const url = buildUrl(`project/smoketest-${process.pid}/compile`)
 
-describe('Running a compile', function() {
-  before(function(done) {
-    return request.post(
+module.exports = {
+  sendNewResult(res) {
+    this._run(error => this._sendResponse(res, error))
+  },
+  sendLastResult(res) {
+    this._sendResponse(res, this._lastError)
+  },
+  triggerRun(cb) {
+    this._run(error => {
+      this._lastError = error
+      cb(error)
+    })
+  },
+
+  _lastError: new Error('SmokeTestsPending'),
+  _sendResponse(res, error) {
+    let code, body
+    if (error) {
+      code = 500
+      body = error.message
+    } else {
+      code = 200
+      body = 'OK'
+    }
+    res.contentType('text/plain')
+    res.status(code).send(body)
+  },
+  _run(done) {
+    request.post(
       {
         url,
         json: {
@@ -50,7 +59,7 @@ describe('Running a compile', function() {
       \\pgfmathsetmacro{\\dy}{rand*0.1}% A random variance in the y coordinate,
                                      % gives a hight fill to the lipid
       \\pgfmathsetmacro{\\rot}{rand*0.1}% A random variance in the
-                                      % molecule orientation      
+                                      % molecule orientation
       \\shade[ball color=red] ({\\i+\\dx+\\rot},{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)}) circle(0.45);
       \\shade[ball color=gray] (\\i+\\dx,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-0.9}) circle(0.45);
       \\shade[ball color=gray] (\\i+\\dx-\\rot,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-1.8}) circle(0.45);
@@ -72,29 +81,22 @@ describe('Running a compile', function() {
         }
       },
       (error, response, body) => {
-        this.error = error
-        this.response = response
-        this.body = body
-        return done()
+        if (error) return done(error)
+        if (!body || !body.compile || !body.compile.outputFiles) {
+          return done(new Error('response payload incomplete'))
+        }
+
+        let pdfFound = false
+        let logFound = false
+        for (const file of body.compile.outputFiles) {
+          if (file.type === 'pdf') pdfFound = true
+          if (file.type === 'log') logFound = true
+        }
+
+        if (!pdfFound) return done(new Error('no pdf returned'))
+        if (!logFound) return done(new Error('no log returned'))
+        done()
       }
     )
-  })
-
-  it('should return the pdf', function() {
-    for (const file of Array.from(this.body.compile.outputFiles)) {
-      if (file.type === 'pdf') {
-        return
-      }
-    }
-    throw new Error('no pdf returned')
-  })
-
-  return it('should return the log', function() {
-    for (const file of Array.from(this.body.compile.outputFiles)) {
-      if (file.type === 'log') {
-        return
-      }
-    }
-    throw new Error('no log returned')
-  })
-})
+  }
+}

test/unit/js/CompileControllerTests.js

@@ -12,6 +12,7 @@
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 require('chai').should()
+const { expect } = require('chai')
 const modulePath = require('path').join(
   __dirname,
   '../../../app/js/CompileController'
@@ -287,21 +288,60 @@ describe('CompileController', function() {
       this.CompileManager.wordcount = sinon
         .stub()
         .callsArgWith(4, null, (this.texcount = ['mock-texcount']))
-      return this.CompileController.wordcount(this.req, this.res, this.next)
     })
 
     it('should return the word count of a file', function() {
+      this.CompileController.wordcount(this.req, this.res, this.next)
       return this.CompileManager.wordcount
         .calledWith(this.project_id, undefined, this.file, this.image)
         .should.equal(true)
     })
 
-    return it('should return the texcount info', function() {
+    it('should return the texcount info', function() {
+      this.CompileController.wordcount(this.req, this.res, this.next)
       return this.res.json
         .calledWith({
           texcount: this.texcount
         })
         .should.equal(true)
     })
+
+    describe('when allowedImages is set', function() {
+      beforeEach(function() {
+        this.Settings.clsi = { docker: {} }
+        this.Settings.clsi.docker.allowedImages = [
+          'repo/image:tag1',
+          'repo/image:tag2'
+        ]
+        this.res.send = sinon.stub()
+        this.res.status = sinon.stub().returns({ send: this.res.send })
+      })
+
+      describe('with an invalid image', function() {
+        beforeEach(function() {
+          this.req.query.image = 'something/evil:1337'
+          this.CompileController.wordcount(this.req, this.res, this.next)
+        })
+        it('should return a 400', function() {
+          expect(this.res.status.calledWith(400)).to.equal(true)
+        })
+        it('should not run the query', function() {
+          expect(this.CompileManager.wordcount.called).to.equal(false)
+        })
+      })
+
+      describe('with a valid image', function() {
+        beforeEach(function() {
+          this.req.query.image = 'repo/image:tag1'
+          this.CompileController.wordcount(this.req, this.res, this.next)
+        })
+        it('should not return a 400', function() {
+          expect(this.res.status.calledWith(400)).to.equal(false)
+        })
+        it('should run the query', function() {
+          expect(this.CompileManager.wordcount.called).to.equal(true)
+        })
+      })
+    })
   })
 })

test/unit/js/CompileManagerTests.js

@@ -160,7 +160,8 @@ describe('CompileManager', function() {
         compiler: (this.compiler = 'pdflatex'),
         timeout: (this.timeout = 42000),
         imageName: (this.image = 'example.com/image'),
-        flags: (this.flags = ['-file-line-error'])
+        flags: (this.flags = ['-file-line-error']),
+        compileGroup: (this.compileGroup = 'compile-group')
       }
       this.env = {}
       this.Settings.compileDir = 'compiles'
@@ -199,7 +200,8 @@ describe('CompileManager', function() {
             timeout: this.timeout,
             image: this.image,
             flags: this.flags,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
@@ -253,7 +255,8 @@ describe('CompileManager', function() {
               CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16',
               CHKTEX_EXIT_ON_ERROR: 1,
               CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000'
-            }
+            },
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
@@ -275,7 +278,8 @@ describe('CompileManager', function() {
             timeout: this.timeout,
             image: this.image,
             flags: this.flags,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
@@ -294,6 +298,7 @@ describe('CompileManager', function() {
         this.proc = new EventEmitter()
         this.proc.stdout = new EventEmitter()
         this.proc.stderr = new EventEmitter()
+        this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr)
         this.child_process.spawn = sinon.stub().returns(this.proc)
         this.CompileManager.clearProject(
           this.project_id,
@@ -328,6 +333,7 @@ describe('CompileManager', function() {
         this.proc = new EventEmitter()
         this.proc.stdout = new EventEmitter()
         this.proc.stderr = new EventEmitter()
+        this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr)
         this.child_process.spawn = sinon.stub().returns(this.proc)
         this.CompileManager.clearProject(
           this.project_id,
@@ -382,7 +388,7 @@ describe('CompileManager', function() {
         this.stdout = `NODE\t${this.page}\t${this.h}\t${this.v}\t${this.width}\t${this.height}\n`
         this.CommandRunner.run = sinon
           .stub()
-          .callsArgWith(6, null, { stdout: this.stdout })
+          .callsArgWith(7, null, { stdout: this.stdout })
         return this.CompileManager.syncFromCode(
           this.project_id,
           this.user_id,
@@ -441,7 +447,7 @@ describe('CompileManager', function() {
         this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}\t${this.line}\t${this.column}\n`
         this.CommandRunner.run = sinon
           .stub()
-          .callsArgWith(6, null, { stdout: this.stdout })
+          .callsArgWith(7, null, { stdout: this.stdout })
         return this.CompileManager.syncFromPdf(
           this.project_id,
           this.user_id,
@@ -483,7 +489,7 @@ describe('CompileManager', function() {
 
   return describe('wordcount', function() {
     beforeEach(function() {
-      this.CommandRunner.run = sinon.stub().callsArg(6)
+      this.CommandRunner.run = sinon.stub().callsArg(7)
       this.fs.readFile = sinon
         .stub()
         .callsArgWith(

test/unit/js/DockerRunnerTests.js

@@ -69,7 +69,8 @@ describe('DockerRunner', function() {
             return runner(callback)
           }
         }
-      }
+      },
+      globals: { Math } // used by lodash
     })
     this.Docker = Docker
     this.getContainer = Docker.prototype.getContainer
@@ -85,6 +86,7 @@ describe('DockerRunner', function() {
     this.project_id = 'project-id-123'
     this.volumes = { '/local/compile/directory': '/compile' }
     this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
+    this.compileGroup = 'compile-group'
     return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
   })
 
@@ -121,6 +123,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           (err, output) => {
             this.callback(err, output)
             return done()
@@ -170,6 +173,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -218,6 +222,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -251,6 +256,7 @@ describe('DockerRunner', function() {
           null,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -267,7 +273,7 @@ describe('DockerRunner', function() {
       })
     })
 
-    return describe('with image override', function() {
+    describe('with image override', function() {
       beforeEach(function() {
         this.Settings.texliveImageNameOveride = 'overrideimage.com/something'
         this.DockerRunner._runAndWaitForContainer = sinon
@@ -280,6 +286,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -289,6 +296,120 @@ describe('DockerRunner', function() {
         return image.should.equal('overrideimage.com/something/image:2016.2')
       })
     })
+
+    describe('with image restriction', function() {
+      beforeEach(function() {
+        this.Settings.clsi.docker.allowedImages = [
+          'repo/image:tag1',
+          'repo/image:tag2'
+        ]
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+      })
+
+      describe('with a valid image', function() {
+        beforeEach(function() {
+          this.DockerRunner.run(
+            this.project_id,
+            this.command,
+            this.directory,
+            'repo/image:tag1',
+            this.timeout,
+            this.env,
+            this.compileGroup,
+            this.callback
+          )
+        })
+
+        it('should setup the container', function() {
+          this.DockerRunner._getContainerOptions.called.should.equal(true)
+        })
+      })
+
+      describe('with a invalid image', function() {
+        beforeEach(function() {
+          this.DockerRunner.run(
+            this.project_id,
+            this.command,
+            this.directory,
+            'something/different:evil',
+            this.timeout,
+            this.env,
+            this.compileGroup,
+            this.callback
+          )
+        })
+
+        it('should call the callback with an error', function() {
+          const err = new Error('image not allowed')
+          this.callback.called.should.equal(true)
+          this.callback.args[0][0].message.should.equal(err.message)
+        })
+
+        it('should not setup the container', function() {
+          this.DockerRunner._getContainerOptions.called.should.equal(false)
+        })
+      })
+    })
+  })
+
+  describe('run with _getOptions', function() {
+    beforeEach(function(done) {
+      // this.DockerRunner._getContainerOptions = sinon
+      //   .stub()
+      //   .returns((this.options = { mockoptions: 'foo' }))
+      this.DockerRunner._fingerprintContainer = sinon
+        .stub()
+        .returns((this.fingerprint = 'fingerprint'))
+
+      this.name = `project-${this.project_id}-${this.fingerprint}`
+
+      this.command = ['mock', 'command', '--outdir=$COMPILE_DIR']
+      this.command_with_dir = ['mock', 'command', '--outdir=/compile']
+      this.timeout = 42000
+      return done()
+    })
+
+    describe('when a compile group config is set', function() {
+      beforeEach(function() {
+        this.Settings.clsi.docker.compileGroupConfig = {
+          'compile-group': {
+            'HostConfig.newProperty': 'new-property'
+          },
+          'other-group': { otherProperty: 'other-property' }
+        }
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+        return this.DockerRunner.run(
+          this.project_id,
+          this.command,
+          this.directory,
+          this.image,
+          this.timeout,
+          this.env,
+          this.compileGroup,
+          this.callback
+        )
+      })
+
+      it('should set the docker options for the compile group', function() {
+        const options = this.DockerRunner._runAndWaitForContainer.lastCall
+          .args[0]
+        return expect(options.HostConfig).to.deep.include({
+          Binds: ['/local/compile/directory:/compile:rw'],
+          LogConfig: { Type: 'none', Config: {} },
+          CapDrop: 'ALL',
+          SecurityOpt: ['no-new-privileges'],
+          newProperty: 'new-property'
+        })
+      })
+
+      return it('should call the callback', function() {
+        return this.callback.calledWith(null, this.output).should.equal(true)
+      })
+    })
   })
 
   describe('_runAndWaitForContainer', function() {
@@ -357,8 +478,8 @@ describe('DockerRunner', function() {
         return this.DockerRunner.startContainer(
           this.options,
           this.volumes,
-          this.callback,
-          () => {}
+          () => {},
+          this.callback
         )
       })
 
@@ -630,19 +751,19 @@ describe('DockerRunner', function() {
     it('should destroy old containers', function() {
       this.DockerRunner.destroyContainer.callCount.should.equal(1)
       return this.DockerRunner.destroyContainer
-        .calledWith('/project-old-container-name', 'old-container-id')
+        .calledWith('project-old-container-name', 'old-container-id')
         .should.equal(true)
     })
 
     it('should not destroy new containers', function() {
       return this.DockerRunner.destroyContainer
-        .calledWith('/project-new-container-name', 'new-container-id')
+        .calledWith('project-new-container-name', 'new-container-id')
         .should.equal(false)
     })
 
     it('should not destroy non-project containers', function() {
       return this.DockerRunner.destroyContainer
-        .calledWith('/totally-not-a-project-container', 'some-random-id')
+        .calledWith('totally-not-a-project-container', 'some-random-id')
         .should.equal(false)
     })
 

test/unit/js/LatexRunnerTests.js

@@ -37,7 +37,10 @@ describe('LatexRunner', function() {
             done() {}
           })
         },
-        './CommandRunner': (this.CommandRunner = {})
+        './CommandRunner': (this.CommandRunner = {}),
+        fs: (this.fs = {
+          writeFile: sinon.stub().callsArg(2)
+        })
       }
     })
 
@@ -45,6 +48,7 @@ describe('LatexRunner', function() {
     this.mainFile = 'main-file.tex'
     this.compiler = 'pdflatex'
     this.image = 'example.com/image'
+    this.compileGroup = 'compile-group'
     this.callback = sinon.stub()
     this.project_id = 'project-id-123'
     return (this.env = { foo: '123' })
@@ -52,7 +56,10 @@ describe('LatexRunner', function() {
 
   return describe('runLatex', function() {
     beforeEach(function() {
-      return (this.CommandRunner.run = sinon.stub().callsArg(6))
+      return (this.CommandRunner.run = sinon.stub().callsArgWith(7, null, {
+        stdout: 'this is stdout',
+        stderr: 'this is stderr'
+      }))
     })
 
     describe('normally', function() {
@@ -65,13 +72,14 @@ describe('LatexRunner', function() {
             compiler: this.compiler,
             timeout: (this.timeout = 42000),
             image: this.image,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           },
           this.callback
         )
       })
 
-      return it('should run the latex command', function() {
+      it('should run the latex command', function() {
         return this.CommandRunner.run
           .calledWith(
             this.project_id,
@@ -79,10 +87,20 @@ describe('LatexRunner', function() {
             this.directory,
             this.image,
             this.timeout,
-            this.env
+            this.env,
+            this.compileGroup
           )
           .should.equal(true)
       })
+
+      it('should record the stdout and stderr', function() {
+        this.fs.writeFile
+          .calledWith(this.directory + '/' + 'output.stdout', 'this is stdout')
+          .should.equal(true)
+        this.fs.writeFile
+          .calledWith(this.directory + '/' + 'output.stderr', 'this is stderr')
+          .should.equal(true)
+      })
     })
 
     describe('with an .Rtex main file', function() {

test/unit/js/OutputFileFinderTests.js

@@ -70,6 +70,7 @@ describe('OutputFileFinder', function() {
     beforeEach(function() {
       this.proc = new EventEmitter()
       this.proc.stdout = new EventEmitter()
+      this.proc.stdout.setEncoding = sinon.stub().returns(this.proc.stdout)
       this.spawn.returns(this.proc)
       this.directory = '/base/dir'
       return this.OutputFileFinder._getAllFiles(this.directory, this.callback)

test/unit/js/OutputFileOptimiserTests.js

@@ -30,7 +30,8 @@ describe('OutputFileOptimiser', function() {
         child_process: { spawn: (this.spawn = sinon.stub()) },
         'logger-sharelatex': { log: sinon.stub(), warn: sinon.stub() },
         './Metrics': {}
-      }
+      },
+      globals: { Math } // used by lodash
     })
     this.directory = '/test/dir'
     return (this.callback = sinon.stub())
@@ -124,7 +125,7 @@ describe('OutputFileOptimiser', function() {
       this.fs.read = sinon
         .stub()
         .withArgs(this.fd)
-        .yields(null, 100, new Buffer('hello /Linearized 1'))
+        .yields(null, 100, Buffer.from('hello /Linearized 1'))
       this.fs.close = sinon
         .stub()
         .withArgs(this.fd)
@@ -140,7 +141,7 @@ describe('OutputFileOptimiser', function() {
         this.fs.read = sinon
           .stub()
           .withArgs(this.fd)
-          .yields(null, 100, new Buffer('hello /Linearized 1'))
+          .yields(null, 100, Buffer.from('hello /Linearized 1'))
         return this.OutputFileOptimiser.checkIfPDFIsOptimised(
           this.src,
           this.callback
@@ -169,7 +170,7 @@ describe('OutputFileOptimiser', function() {
         this.fs.read = sinon
           .stub()
           .withArgs(this.fd)
-          .yields(null, 100, new Buffer('hello not linearized 1'))
+          .yields(null, 100, Buffer.from('hello not linearized 1'))
         return this.OutputFileOptimiser.checkIfPDFIsOptimised(
           this.src,
           this.callback

test/unit/js/ProjectPersistenceManagerTests.js

@@ -14,6 +14,7 @@
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 require('chai').should()
+const assert = require('chai').assert
 const modulePath = require('path').join(
   __dirname,
   '../../../app/js/ProjectPersistenceManager'
@@ -26,7 +27,15 @@ describe('ProjectPersistenceManager', function() {
       requires: {
         './UrlCache': (this.UrlCache = {}),
         './CompileManager': (this.CompileManager = {}),
-        'logger-sharelatex': (this.logger = { log: sinon.stub() }),
+        diskusage: (this.diskusage = { check: sinon.stub() }),
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          warn: sinon.stub(),
+          err: sinon.stub()
+        }),
+        'settings-sharelatex': (this.settings = {
+          project_cache_length_ms: 1000
+        }),
         './db': (this.db = {})
       }
     })
@@ -35,6 +44,57 @@ describe('ProjectPersistenceManager', function() {
     return (this.user_id = '1234')
   })
 
+  describe('refreshExpiryTimeout', function() {
+    it('should leave expiry alone if plenty of disk', function(done) {
+      this.diskusage.check.callsArgWith(1, null, {
+        available: 40,
+        total: 100
+      })
+
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
+          this.settings.project_cache_length_ms
+        )
+        done()
+      })
+    })
+
+    it('should drop EXPIRY_TIMEOUT 10% if low disk usage', function(done) {
+      this.diskusage.check.callsArgWith(1, null, {
+        available: 5,
+        total: 100
+      })
+
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
+        done()
+      })
+    })
+
+    it('should not drop EXPIRY_TIMEOUT to below 50% of project_cache_length_ms', function(done) {
+      this.diskusage.check.callsArgWith(1, null, {
+        available: 5,
+        total: 100
+      })
+      this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
+        done()
+      })
+    })
+
+    it('should not modify EXPIRY_TIMEOUT if there is an error getting disk values', function(done) {
+      this.diskusage.check.callsArgWith(1, 'Error', {
+        available: 5,
+        total: 100
+      })
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(1000)
+        done()
+      })
+    })
+  })
+
   describe('clearExpiredProjects', function() {
     beforeEach(function() {
       this.project_ids = ['project-id-1', 'project-id-2']

test/unit/js/RequestParserTests.js

@@ -114,6 +114,48 @@ describe('RequestParser', function() {
     })
   })
 
+  describe('when image restrictions are present', function() {
+    beforeEach(function() {
+      this.settings.clsi = { docker: {} }
+      this.settings.clsi.docker.allowedImages = [
+        'repo/name:tag1',
+        'repo/name:tag2'
+      ]
+    })
+
+    describe('with imageName set to something invalid', function() {
+      beforeEach(function() {
+        const request = this.validRequest
+        request.compile.options.imageName = 'something/different:latest'
+        this.RequestParser.parse(request, (error, data) => {
+          this.error = error
+          this.data = data
+        })
+      })
+
+      it('should throw an error for imageName', function() {
+        expect(String(this.error)).to.include(
+          'imageName attribute should be one of'
+        )
+      })
+    })
+
+    describe('with imageName set to something valid', function() {
+      beforeEach(function() {
+        const request = this.validRequest
+        request.compile.options.imageName = 'repo/name:tag1'
+        this.RequestParser.parse(request, (error, data) => {
+          this.error = error
+          this.data = data
+        })
+      })
+
+      it('should set the imageName', function() {
+        this.data.imageName.should.equal('repo/name:tag1')
+      })
+    })
+  })
+
   describe('with flags set', function() {
     beforeEach(function() {
       this.validRequest.compile.options.flags = ['-file-line-error']

test/unit/js/ResourceWriterTests.js

@@ -230,6 +230,12 @@ describe('ResourceWriter', function() {
         {
           path: '_markdown_main/30893013dec5d869a415610079774c2f.md.tex',
           type: 'tex'
+        },
+        {
+          path: 'output.stdout'
+        },
+        {
+          path: 'output.stderr'
         }
       ]
       this.resources = 'mock-resources'
@@ -256,6 +262,18 @@ describe('ResourceWriter', function() {
         .should.equal(true)
     })
 
+    it('should delete the stdout log file', function() {
+      return this.ResourceWriter._deleteFileIfNotDirectory
+        .calledWith(path.join(this.basePath, 'output.stdout'))
+        .should.equal(true)
+    })
+
+    it('should delete the stderr log file', function() {
+      return this.ResourceWriter._deleteFileIfNotDirectory
+        .calledWith(path.join(this.basePath, 'output.stderr'))
+        .should.equal(true)
+    })
+
     it('should delete the extra files', function() {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'extra/file.tex'))

test/unit/js/UrlCacheTests.js

@@ -160,7 +160,7 @@ describe('UrlCache', function() {
 
   describe('_ensureUrlIsInCache', function() {
     beforeEach(function() {
-      this.UrlFetcher.pipeUrlToFile = sinon.stub().callsArg(2)
+      this.UrlFetcher.pipeUrlToFileWithRetry = sinon.stub().callsArg(2)
       return (this.UrlCache._updateOrCreateUrlDetails = sinon
         .stub()
         .callsArg(3))
@@ -190,7 +190,7 @@ describe('UrlCache', function() {
       })
 
       it('should download the URL to the cache file', function() {
-        return this.UrlFetcher.pipeUrlToFile
+        return this.UrlFetcher.pipeUrlToFileWithRetry
           .calledWith(
             this.url,
             this.UrlCache._cacheFilePathForUrl(this.project_id, this.url)
@@ -232,7 +232,7 @@ describe('UrlCache', function() {
       })
 
       it('should not download the URL to the cache file', function() {
-        return this.UrlFetcher.pipeUrlToFile.called.should.equal(false)
+        return this.UrlFetcher.pipeUrlToFileWithRetry.called.should.equal(false)
       })
 
       return it('should return the callback with the cache file path', function() {

test/unit/js/UrlFetcherTests.js

@@ -33,72 +33,64 @@ describe('UrlFetcher', function() {
       }
     }))
   })
-
-  it('should turn off the cookie jar in request', function() {
-    return this.defaults.calledWith({ jar: false }).should.equal(true)
-  })
-
-  describe('rewrite url domain if filestoreDomainOveride is set', function() {
-    beforeEach(function() {
-      this.path = '/path/to/file/on/disk'
-      this.request.get = sinon
-        .stub()
-        .returns((this.urlStream = new EventEmitter()))
-      this.urlStream.pipe = sinon.stub()
-      this.urlStream.pause = sinon.stub()
-      this.urlStream.resume = sinon.stub()
-      this.fs.createWriteStream = sinon
-        .stub()
-        .returns((this.fileStream = new EventEmitter()))
-      return (this.fs.unlink = (file, callback) => callback())
+  describe('pipeUrlToFileWithRetry', function() {
+    this.beforeEach(function() {
+      this.UrlFetcher.pipeUrlToFile = sinon.stub()
     })
 
-    it('should use the normal domain when override not set', function(done) {
-      this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
-        this.request.get.args[0][0].url.should.equal(this.url)
-        return done()
+    it('should call pipeUrlToFile', function(done) {
+      this.UrlFetcher.pipeUrlToFile.callsArgWith(2)
+      this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, err => {
+        expect(err).to.equal(undefined)
+        this.UrlFetcher.pipeUrlToFile.called.should.equal(true)
+        done()
       })
-      this.res = { statusCode: 200 }
-      this.urlStream.emit('response', this.res)
-      this.urlStream.emit('end')
-      return this.fileStream.emit('finish')
     })
 
-    return it('should use override domain when filestoreDomainOveride is set', function(done) {
-      this.settings.filestoreDomainOveride = '192.11.11.11'
-      this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
-        this.request.get.args[0][0].url.should.equal(
-          '192.11.11.11/file/here?query=string'
-        )
-        return done()
+    it('should call pipeUrlToFile multiple times on error', function(done) {
+      const error = new Error("couldn't download file")
+      this.UrlFetcher.pipeUrlToFile.callsArgWith(2, error)
+      this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, err => {
+        expect(err).to.equal(error)
+        this.UrlFetcher.pipeUrlToFile.callCount.should.equal(3)
+        done()
+      })
+    })
+
+    it('should call pipeUrlToFile twice if only 1 error', function(done) {
+      this.UrlFetcher.pipeUrlToFile.onCall(0).callsArgWith(2, 'error')
+      this.UrlFetcher.pipeUrlToFile.onCall(1).callsArgWith(2)
+      this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, err => {
+        expect(err).to.equal(undefined)
+        this.UrlFetcher.pipeUrlToFile.callCount.should.equal(2)
+        done()
       })
-      this.res = { statusCode: 200 }
-      this.urlStream.emit('response', this.res)
-      this.urlStream.emit('end')
-      return this.fileStream.emit('finish')
     })
   })
 
-  return describe('pipeUrlToFile', function() {
-    beforeEach(function(done) {
-      this.path = '/path/to/file/on/disk'
-      this.request.get = sinon
-        .stub()
-        .returns((this.urlStream = new EventEmitter()))
-      this.urlStream.pipe = sinon.stub()
-      this.urlStream.pause = sinon.stub()
-      this.urlStream.resume = sinon.stub()
-      this.fs.createWriteStream = sinon
-        .stub()
-        .returns((this.fileStream = new EventEmitter()))
-      this.fs.unlink = (file, callback) => callback()
-      return done()
+  describe('pipeUrlToFile', function() {
+    it('should turn off the cookie jar in request', function() {
+      return this.defaults.calledWith({ jar: false }).should.equal(true)
     })
 
-    describe('successfully', function() {
-      beforeEach(function(done) {
+    describe('rewrite url domain if filestoreDomainOveride is set', function() {
+      beforeEach(function() {
+        this.path = '/path/to/file/on/disk'
+        this.request.get = sinon
+          .stub()
+          .returns((this.urlStream = new EventEmitter()))
+        this.urlStream.pipe = sinon.stub()
+        this.urlStream.pause = sinon.stub()
+        this.urlStream.resume = sinon.stub()
+        this.fs.createWriteStream = sinon
+          .stub()
+          .returns((this.fileStream = new EventEmitter()))
+        return (this.fs.unlink = (file, callback) => callback())
+      })
+
+      it('should use the normal domain when override not set', function(done) {
         this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
-          this.callback()
+          this.request.get.args[0][0].url.should.equal(this.url)
           return done()
         })
         this.res = { statusCode: 200 }
@@ -107,67 +99,113 @@ describe('UrlFetcher', function() {
         return this.fileStream.emit('finish')
       })
 
-      it('should request the URL', function() {
-        return this.request.get
-          .calledWith(sinon.match({ url: this.url }))
-          .should.equal(true)
-      })
-
-      it('should open the file for writing', function() {
-        return this.fs.createWriteStream
-          .calledWith(this.path)
-          .should.equal(true)
-      })
-
-      it('should pipe the URL to the file', function() {
-        return this.urlStream.pipe
-          .calledWith(this.fileStream)
-          .should.equal(true)
-      })
-
-      return it('should call the callback', function() {
-        return this.callback.called.should.equal(true)
-      })
-    })
-
-    describe('with non success status code', function() {
-      beforeEach(function(done) {
-        this.UrlFetcher.pipeUrlToFile(this.url, this.path, err => {
-          this.callback(err)
+      return it('should use override domain when filestoreDomainOveride is set', function(done) {
+        this.settings.filestoreDomainOveride = '192.11.11.11'
+        this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
+          this.request.get.args[0][0].url.should.equal(
+            '192.11.11.11/file/here?query=string'
+          )
           return done()
         })
-        this.res = { statusCode: 404 }
+        this.res = { statusCode: 200 }
         this.urlStream.emit('response', this.res)
-        return this.urlStream.emit('end')
-      })
-
-      it('should call the callback with an error', function() {
-        this.callback.calledWith(sinon.match(Error)).should.equal(true)
-
-        const message = this.callback.args[0][0].message
-        expect(message).to.include('URL returned non-success status code: 404')
+        this.urlStream.emit('end')
+        return this.fileStream.emit('finish')
       })
     })
 
-    return describe('with error', function() {
+    return describe('pipeUrlToFile', function() {
       beforeEach(function(done) {
-        this.UrlFetcher.pipeUrlToFile(this.url, this.path, err => {
-          this.callback(err)
-          return done()
+        this.path = '/path/to/file/on/disk'
+        this.request.get = sinon
+          .stub()
+          .returns((this.urlStream = new EventEmitter()))
+        this.urlStream.pipe = sinon.stub()
+        this.urlStream.pause = sinon.stub()
+        this.urlStream.resume = sinon.stub()
+        this.fs.createWriteStream = sinon
+          .stub()
+          .returns((this.fileStream = new EventEmitter()))
+        this.fs.unlink = (file, callback) => callback()
+        return done()
+      })
+
+      describe('successfully', function() {
+        beforeEach(function(done) {
+          this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
+            this.callback()
+            return done()
+          })
+          this.res = { statusCode: 200 }
+          this.urlStream.emit('response', this.res)
+          this.urlStream.emit('end')
+          return this.fileStream.emit('finish')
+        })
+
+        it('should request the URL', function() {
+          return this.request.get
+            .calledWith(sinon.match({ url: this.url }))
+            .should.equal(true)
+        })
+
+        it('should open the file for writing', function() {
+          return this.fs.createWriteStream
+            .calledWith(this.path)
+            .should.equal(true)
+        })
+
+        it('should pipe the URL to the file', function() {
+          return this.urlStream.pipe
+            .calledWith(this.fileStream)
+            .should.equal(true)
+        })
+
+        return it('should call the callback', function() {
+          return this.callback.called.should.equal(true)
         })
-        return this.urlStream.emit(
-          'error',
-          (this.error = new Error('something went wrong'))
-        )
       })
 
-      it('should call the callback with the error', function() {
-        return this.callback.calledWith(this.error).should.equal(true)
+      describe('with non success status code', function() {
+        beforeEach(function(done) {
+          this.UrlFetcher.pipeUrlToFile(this.url, this.path, err => {
+            this.callback(err)
+            return done()
+          })
+          this.res = { statusCode: 404 }
+          this.urlStream.emit('response', this.res)
+          return this.urlStream.emit('end')
+        })
+
+        it('should call the callback with an error', function() {
+          this.callback.calledWith(sinon.match(Error)).should.equal(true)
+
+          const message = this.callback.args[0][0].message
+          expect(message).to.include(
+            'URL returned non-success status code: 404'
+          )
+        })
       })
 
-      return it('should only call the callback once, even if end is called', function() {
-        this.urlStream.emit('end')
-        return this.callback.calledOnce.should.equal(true)
+      return describe('with error', function() {
+        beforeEach(function(done) {
+          this.UrlFetcher.pipeUrlToFile(this.url, this.path, err => {
+            this.callback(err)
+            return done()
+          })
+          return this.urlStream.emit(
+            'error',
+            (this.error = new Error('something went wrong'))
+          )
+        })
+
+        it('should call the callback with the error', function() {
+          return this.callback.calledWith(this.error).should.equal(true)
+        })
+
+        return it('should only call the callback once, even if end is called', function() {
+          this.urlStream.emit('end')
+          return this.callback.calledOnce.should.equal(true)
+        })
       })
     })
   })