add possible rest api clients

Bump node-fetch from 2.6.0 to 2.6.1

.eslintrc

@@ -8,7 +8,7 @@
     "prettier/standard"
   ],
   "parserOptions": {
-    "ecmaVersion": 2017
+    "ecmaVersion": 2018
   },
   "plugins": [
     "mocha",

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      # Docker images use the branch name and do not support slashes in tags
+      # https://github.com/overleaf/google-ops/issues/822
+      # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator
+      separator: "-"
+
+    # Block informal upgrades -- security upgrades use a separate queue.
+    # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit
+    open-pull-requests-limit: 0
+
+    # currently assign team-magma to all dependabot PRs - this may change in
+    # future if we reorganise teams
+    labels:
+      - "dependencies"
+      - "Team-Magma"

.gitignore

@@ -11,3 +11,6 @@ db.sqlite-wal
 db.sqlite-shm
 config/*
 npm-debug.log
+
+# managed by dev-environment$ bin/update_build_scripts
+.npmrc

.nvmrc

@@ -1 +1 @@
-10.19.0
+10.22.1

Dockerfile

@@ -2,7 +2,7 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
 
-FROM node:10.19.0 as base
+FROM node:10.22.1 as base
 
 WORKDIR /app
 COPY install_deps.sh /app
@@ -15,12 +15,10 @@ FROM base as app
 #wildcard as some files may not be in all repos
 COPY package*.json npm-shrink*.json /app/
 
-RUN npm install --quiet
+RUN npm ci --quiet
 
 COPY . /app
 
-
-
 FROM base
 
 COPY --from=app /app /app

Jenkinsfile

@@ -1,131 +0,0 @@
-String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
-
-pipeline {
-  agent any
-
-  environment {
-    GIT_PROJECT = "clsi"
-    JENKINS_WORKFLOW = "clsi-sharelatex"
-    TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline"
-    GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT"
-  }
-
-  triggers {
-    pollSCM('* * * * *')
-    cron(cron_string)
-  }
-
-  stages {
-
-    stage('Install') {
-      steps {
-        withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
-          sh "curl $GIT_API_URL \
-            --data '{ \
-            \"state\" : \"pending\", \
-            \"target_url\": \"$TARGET_URL\", \
-            \"description\": \"Your build is underway\", \
-            \"context\": \"ci/jenkins\" }' \
-            -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
-        }
-      }
-    }
-
-    stage('Build') {
-      steps {
-        sh 'make build'
-      }
-    }
-
-    stage('Linting') {
-      steps {
-        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format'
-        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint'
-      }
-    }
-
-    stage('Unit Tests') {
-      steps {
-        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
-      }
-    }
-
-    stage('Acceptance Tests') {
-      steps {
-        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
-      }
-    }
-
-    stage('Package and docker push') {
-      steps {
-        sh 'echo ${BUILD_NUMBER} > build_number.txt'
-        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
-        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
-        
-        withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
-          sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
-        }
-        sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
-        sh 'docker logout https://gcr.io/overleaf-ops'
-        
-      }
-    }
-
-    stage('Publish to s3') {
-      steps {
-        sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
-        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
-            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
-        }
-        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
-            // The deployment process uses this file to figure out the latest build
-            s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
-        }
-      }
-    }
-  }
-
-  post {
-    always {
-      sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
-      sh 'make clean'
-    }
-
-    success {
-      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
-        sh "curl $GIT_API_URL \
-          --data '{ \
-          \"state\" : \"success\", \
-          \"target_url\": \"$TARGET_URL\", \
-          \"description\": \"Your build succeeded!\", \
-          \"context\": \"ci/jenkins\" }' \
-          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
-      }
-    }
-
-    failure {
-      mail(from: "${EMAIL_ALERT_FROM}",
-           to: "${EMAIL_ALERT_TO}",
-           subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
-           body: "Build: ${BUILD_URL}")
-      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
-        sh "curl $GIT_API_URL \
-          --data '{ \
-          \"state\" : \"failure\", \
-          \"target_url\": \"$TARGET_URL\", \
-          \"description\": \"Your build failed\", \
-          \"context\": \"ci/jenkins\" }' \
-          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
-      }
-    }
-  }
-
-  // The options directive is for configuration that applies to the whole job.
-  options {
-    // we'd like to make sure remove old builds, so we don't fill up our storage!
-    buildDiscarder(logRotator(numToKeepStr:'50'))
-
-    // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
-    timeout(time: 30, unit: 'MINUTES')
-  }
-}

Makefile

@@ -25,13 +25,13 @@ clean:
 	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
 format:
-	$(DOCKER_COMPOSE) run --rm test_unit npm run format
+	$(DOCKER_COMPOSE) run --rm test_unit npm run --silent format
 
 format_fix:
-	$(DOCKER_COMPOSE) run --rm test_unit npm run format:fix
+	$(DOCKER_COMPOSE) run --rm test_unit npm run --silent format:fix
 
 lint:
-	$(DOCKER_COMPOSE) run --rm test_unit npm run lint
+	$(DOCKER_COMPOSE) run --rm test_unit npm run --silent lint
 
 test: format lint test_unit test_acceptance
 

README.md

@@ -1,6 +1,8 @@
 overleaf/clsi
 ===============
 
+**Note:** Original repo here: https://github.com/overleaf/clsi
+
 A web api for compiling LaTeX documents in the cloud
 
 The Common LaTeX Service Interface (CLSI) provides a RESTful interface to traditional LaTeX tools (or, more generally, any command line tool for composing marked-up documents into a display format such as PDF or HTML). The CLSI listens on the following ports by default:
@@ -34,20 +36,21 @@ The CLSI can be installed and set up as part of the entire [Overleaf stack](http
 
     $ git clone git@github.com:overleaf/clsi.git
     
-Then install the require npm modules:
+Then install the require npm modules and run:
 
     $ npm install
+    $ npm start
     
-Then compile the coffee script source files:
-
-    $ grunt install
-    
-Finally, (after configuring your local database - see the Config section), run the CLSI service:
-
-    $ grunt run
     
 The CLSI should then be running at http://localhost:3013.
 
+**Note:** to install texlive-full on ubuntu:
+    $ sudo apt install texlive-full
+
+Possible REST API clients to test are:
+* Postman
+* Insomnia
+    
 Config
 ------
 

app.js

@@ -5,7 +5,7 @@
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
-let tenMinutes
+const tenMinutes = 10 * 60 * 1000
 const Metrics = require('metrics-sharelatex')
 Metrics.initialize('clsi')
 
@@ -17,7 +17,7 @@ if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) {
   logger.initializeErrorReporting(Settings.sentry.dsn)
 }
 
-const smokeTest = require('smoke-test-sharelatex')
+const smokeTest = require('./test/smoke/js/SmokeTests')
 const ContentTypeMapper = require('./app/js/ContentTypeMapper')
 const Errors = require('./app/js/Errors')
 
@@ -42,38 +42,36 @@ app.use(Metrics.http.monitor(logger))
 // minutes (including file download time), so bump up the
 // timeout a bit.
 const TIMEOUT = 10 * 60 * 1000
-app.use(function(req, res, next) {
+app.use(function (req, res, next) {
   req.setTimeout(TIMEOUT)
   res.setTimeout(TIMEOUT)
   res.removeHeader('X-Powered-By')
   return next()
 })
 
-app.param('project_id', function(req, res, next, project_id) {
-  if (project_id != null ? project_id.match(/^[a-zA-Z0-9_-]+$/) : undefined) {
+app.param('project_id', function (req, res, next, projectId) {
+  if (projectId != null ? projectId.match(/^[a-zA-Z0-9_-]+$/) : undefined) {
     return next()
   } else {
     return next(new Error('invalid project id'))
   }
 })
 
-app.param('user_id', function(req, res, next, user_id) {
-  if (user_id != null ? user_id.match(/^[0-9a-f]{24}$/) : undefined) {
+app.param('user_id', function (req, res, next, userId) {
+  if (userId != null ? userId.match(/^[0-9a-f]{24}$/) : undefined) {
     return next()
   } else {
     return next(new Error('invalid user id'))
   }
 })
 
-app.param('build_id', function(req, res, next, build_id) {
+app.param('build_id', function (req, res, next, buildId) {
   if (
-    build_id != null
-      ? build_id.match(OutputCacheManager.BUILD_REGEX)
-      : undefined
+    buildId != null ? buildId.match(OutputCacheManager.BUILD_REGEX) : undefined
   ) {
     return next()
   } else {
-    return next(new Error(`invalid build id ${build_id}`))
+    return next(new Error(`invalid build id ${buildId}`))
   }
 })
 
@@ -136,19 +134,18 @@ const staticServer = ForbidSymlinks(express.static, Settings.path.compilesDir, {
   }
 })
 
-app.get('/project/:project_id/user/:user_id/build/:build_id/output/*', function(
-  req,
-  res,
-  next
-) {
+app.get(
+  '/project/:project_id/user/:user_id/build/:build_id/output/*',
+  function (req, res, next) {
     // for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
     req.url =
       `/${req.params.project_id}-${req.params.user_id}/` +
       OutputCacheManager.path(req.params.build_id, `/${req.params[0]}`)
     return staticServer(req, res, next)
-})
+  }
+)
 
-app.get('/project/:project_id/build/:build_id/output/*', function(
+app.get('/project/:project_id/build/:build_id/output/*', function (
   req,
   res,
   next
@@ -160,7 +157,7 @@ app.get('/project/:project_id/build/:build_id/output/*', function(
   return staticServer(req, res, next)
 })
 
-app.get('/project/:project_id/user/:user_id/output/*', function(
+app.get('/project/:project_id/user/:user_id/output/*', function (
   req,
   res,
   next
@@ -170,7 +167,7 @@ app.get('/project/:project_id/user/:user_id/output/*', function(
   return staticServer(req, res, next)
 })
 
-app.get('/project/:project_id/output/*', function(req, res, next) {
+app.get('/project/:project_id/output/*', function (req, res, next) {
   if (
     (req.query != null ? req.query.build : undefined) != null &&
     req.query.build.match(OutputCacheManager.BUILD_REGEX)
@@ -185,76 +182,56 @@ app.get('/project/:project_id/output/*', function(req, res, next) {
   return staticServer(req, res, next)
 })
 
-app.get('/oops', function(req, res, next) {
+app.get('/oops', function (req, res, next) {
   logger.error({ err: 'hello' }, 'test error')
   return res.send('error\n')
 })
 
 app.get('/status', (req, res, next) => res.send('CLSI is alive\n'))
 
-const resCacher = {
-  contentType(setContentType) {
-    this.setContentType = setContentType
-  },
-  send(code, body) {
-    this.code = code
-    this.body = body
-  },
-
-  // default the server to be down
-  code: 500,
-  body: {},
-  setContentType: 'application/json'
-}
-
-let shutdownTime
+Settings.processTooOld = false
 if (Settings.processLifespanLimitMs) {
   Settings.processLifespanLimitMs +=
     Settings.processLifespanLimitMs * (Math.random() / 10)
-  shutdownTime = Date.now() + Settings.processLifespanLimitMs
-  logger.info('Lifespan limited to ', shutdownTime)
-}
+  logger.info(
+    'Lifespan limited to ',
+    Date.now() + Settings.processLifespanLimitMs
+  )
 
-const checkIfProcessIsTooOld = function(cont) {
-  if (shutdownTime && shutdownTime < Date.now()) {
+  setTimeout(() => {
     logger.log('shutting down, process is too old')
-    resCacher.send = function() {}
-    resCacher.code = 500
-    resCacher.body = { processToOld: true }
-  } else {
-    cont()
-  }
+    Settings.processTooOld = true
+  }, Settings.processLifespanLimitMs)
 }
 
-if (Settings.smokeTest) {
-  const runSmokeTest = function() {
-    checkIfProcessIsTooOld(function() {
+function runSmokeTest() {
+  if (Settings.processTooOld) return
   logger.log('running smoke tests')
-      smokeTest.run(
-        require.resolve(__dirname + '/test/smoke/js/SmokeTests.js')
-      )({}, resCacher)
-      return setTimeout(runSmokeTest, 30 * 1000)
+  smokeTest.triggerRun((err) => {
+    if (err) logger.error({ err }, 'smoke tests failed')
+    setTimeout(runSmokeTest, 30 * 1000)
   })
-  }
+}
+if (Settings.smokeTest) {
   runSmokeTest()
 }
 
-app.get('/health_check', function(req, res) {
-  res.contentType(resCacher.setContentType)
-  return res.status(resCacher.code).send(resCacher.body)
+app.get('/health_check', function (req, res) {
+  if (Settings.processTooOld) {
+    return res.status(500).json({ processTooOld: true })
+  }
+  smokeTest.sendLastResult(res)
 })
 
-app.get('/smoke_test_force', (req, res) =>
-  smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))(
-    req,
-    res
-  )
-)
+app.get('/smoke_test_force', (req, res) => smokeTest.sendNewResult(res))
 
-app.use(function(error, req, res, next) {
+app.use(function (error, req, res, next) {
   if (error instanceof Errors.NotFoundError) {
-    logger.warn({ err: error, url: req.url }, 'not found error')
+    logger.log({ err: error, url: req.url }, 'not found error')
     return res.sendStatus(404)
+  } else if (error.code === 'EPIPE') {
+    // inspect container returns EPIPE when shutting down
+    return res.sendStatus(503) // send 503 Unavailable response
   } else {
     logger.error({ err: error, url: req.url }, 'server error')
     return res.sendStatus((error != null ? error.statusCode : undefined) || 500)
@@ -266,8 +243,8 @@ const os = require('os')
 
 let STATE = 'up'
 
-const loadTcpServer = net.createServer(function(socket) {
-  socket.on('error', function(err) {
+const loadTcpServer = net.createServer(function (socket) {
+  socket.on('error', function (err) {
     if (err.code === 'ECONNRESET') {
       // this always comes up, we don't know why
       return
@@ -302,19 +279,19 @@ const loadTcpServer = net.createServer(function(socket) {
 
 const loadHttpServer = express()
 
-loadHttpServer.post('/state/up', function(req, res, next) {
+loadHttpServer.post('/state/up', function (req, res, next) {
   STATE = 'up'
   logger.info('getting message to set server to down')
   return res.sendStatus(204)
 })
 
-loadHttpServer.post('/state/down', function(req, res, next) {
+loadHttpServer.post('/state/down', function (req, res, next) {
   STATE = 'down'
   logger.info('getting message to set server to down')
   return res.sendStatus(204)
 })
 
-loadHttpServer.post('/state/maint', function(req, res, next) {
+loadHttpServer.post('/state/maint', function (req, res, next) {
   STATE = 'maint'
   logger.info('getting message to set server to maint')
   return res.sendStatus(204)
@@ -323,46 +300,48 @@ loadHttpServer.post('/state/maint', function(req, res, next) {
 const port =
   __guard__(
     Settings.internal != null ? Settings.internal.clsi : undefined,
-    x => x.port
+    (x) => x.port
   ) || 3013
 const host =
   __guard__(
     Settings.internal != null ? Settings.internal.clsi : undefined,
-    x1 => x1.host
+    (x1) => x1.host
   ) || 'localhost'
 
-const load_tcp_port = Settings.internal.load_balancer_agent.load_port
-const load_http_port = Settings.internal.load_balancer_agent.local_port
+const loadTcpPort = Settings.internal.load_balancer_agent.load_port
+const loadHttpPort = Settings.internal.load_balancer_agent.local_port
 
 if (!module.parent) {
   // Called directly
-  app.listen(port, host, error =>
+  app.listen(port, host, (error) => {
+    if (error) {
+      logger.fatal({ error }, `Error starting CLSI on ${host}:${port}`)
+    } else {
       logger.info(`CLSI starting up, listening on ${host}:${port}`)
-  )
-
-  loadTcpServer.listen(load_tcp_port, host, function(error) {
-    if (error != null) {
-      throw error
     }
-    return logger.info(`Load tcp agent listening on load port ${load_tcp_port}`)
   })
 
-  loadHttpServer.listen(load_http_port, host, function(error) {
+  loadTcpServer.listen(loadTcpPort, host, function (error) {
     if (error != null) {
       throw error
     }
-    return logger.info(
-      `Load http agent listening on load port ${load_http_port}`
-    )
+    return logger.info(`Load tcp agent listening on load port ${loadTcpPort}`)
+  })
+
+  loadHttpServer.listen(loadHttpPort, host, function (error) {
+    if (error != null) {
+      throw error
+    }
+    return logger.info(`Load http agent listening on load port ${loadHttpPort}`)
   })
 }
 
 module.exports = app
 
-setInterval(
-  () => ProjectPersistenceManager.clearExpiredProjects(),
-  (tenMinutes = 10 * 60 * 1000)
-)
+setInterval(() => {
+  ProjectPersistenceManager.refreshExpiryTimeout()
+  ProjectPersistenceManager.clearExpiredProjects()
+}, tenMinutes)
 
 function __guard__(value, transform) {
   return typeof value !== 'undefined' && value !== null

app/js/CompileController.js

@@ -24,10 +24,10 @@ const Errors = require('./Errors')
 module.exports = CompileController = {
   compile(req, res, next) {
     if (next == null) {
-      next = function(error) {}
+      next = function (error) {}
     }
     const timer = new Metrics.Timer('compile-request')
-    return RequestParser.parse(req.body, function(error, request) {
+    return RequestParser.parse(req.body, function (error, request) {
       if (error != null) {
         return next(error)
       }
@@ -37,11 +37,11 @@ module.exports = CompileController = {
       }
       return ProjectPersistenceManager.markProjectAsJustAccessed(
         request.project_id,
-        function(error) {
+        function (error) {
           if (error != null) {
             return next(error)
           }
-          return CompileManager.doCompileWithLock(request, function(
+          return CompileManager.doCompileWithLock(request, function (
             error,
             outputFiles
           ) {
@@ -55,6 +55,10 @@ module.exports = CompileController = {
             } else if (error instanceof Errors.FilesOutOfSyncError) {
               code = 409 // Http 409 Conflict
               status = 'retry'
+            } else if (error && error.code === 'EPIPE') {
+              // docker returns EPIPE when shutting down
+              code = 503 // send 503 Unavailable response
+              status = 'unavailable'
             } else if (error != null ? error.terminated : undefined) {
               status = 'terminated'
             } else if (error != null ? error.validate : undefined) {
@@ -112,7 +116,7 @@ module.exports = CompileController = {
               compile: {
                 status,
                 error: (error != null ? error.message : undefined) || error,
-                outputFiles: outputFiles.map(file => ({
+                outputFiles: outputFiles.map((file) => ({
                   url:
                     `${Settings.apis.clsi.url}/project/${request.project_id}` +
                     (request.user_id != null
@@ -134,7 +138,7 @@ module.exports = CompileController = {
 
   stopCompile(req, res, next) {
     const { project_id, user_id } = req.params
-    return CompileManager.stopCompile(project_id, user_id, function(error) {
+    return CompileManager.stopCompile(project_id, user_id, function (error) {
       if (error != null) {
         return next(error)
       }
@@ -144,12 +148,12 @@ module.exports = CompileController = {
 
   clearCache(req, res, next) {
     if (next == null) {
-      next = function(error) {}
+      next = function (error) {}
     }
     return ProjectPersistenceManager.clearProject(
       req.params.project_id,
       req.params.user_id,
-      function(error) {
+      function (error) {
         if (error != null) {
           return next(error)
         }
@@ -160,7 +164,7 @@ module.exports = CompileController = {
 
   syncFromCode(req, res, next) {
     if (next == null) {
-      next = function(error) {}
+      next = function (error) {}
     }
     const { file } = req.query
     const line = parseInt(req.query.line, 10)
@@ -173,7 +177,7 @@ module.exports = CompileController = {
       file,
       line,
       column,
-      function(error, pdfPositions) {
+      function (error, pdfPositions) {
         if (error != null) {
           return next(error)
         }
@@ -186,37 +190,50 @@ module.exports = CompileController = {
 
   syncFromPdf(req, res, next) {
     if (next == null) {
-      next = function(error) {}
+      next = function (error) {}
     }
     const page = parseInt(req.query.page, 10)
     const h = parseFloat(req.query.h)
     const v = parseFloat(req.query.v)
     const { project_id } = req.params
     const { user_id } = req.params
-    return CompileManager.syncFromPdf(project_id, user_id, page, h, v, function(
-      error,
-      codePositions
-    ) {
+    return CompileManager.syncFromPdf(
+      project_id,
+      user_id,
+      page,
+      h,
+      v,
+      function (error, codePositions) {
         if (error != null) {
           return next(error)
         }
         return res.json({
           code: codePositions
         })
-    })
+      }
+    )
   },
 
   wordcount(req, res, next) {
     if (next == null) {
-      next = function(error) {}
+      next = function (error) {}
     }
     const file = req.query.file || 'main.tex'
     const { project_id } = req.params
     const { user_id } = req.params
     const { image } = req.query
+    if (
+      image &&
+      Settings.clsi &&
+      Settings.clsi.docker &&
+      Settings.clsi.docker.allowedImages &&
+      !Settings.clsi.docker.allowedImages.includes(image)
+    ) {
+      return res.status(400).send('invalid image')
+    }
     logger.log({ image, file, project_id }, 'word count request')
 
-    return CompileManager.wordcount(project_id, user_id, file, image, function(
+    return CompileManager.wordcount(project_id, user_id, file, image, function (
       error,
       result
     ) {
@@ -231,7 +248,7 @@ module.exports = CompileController = {
 
   status(req, res, next) {
     if (next == null) {
-      next = function(error) {}
+      next = function (error) {}
     }
     return res.send('OK')
   }

app/js/CompileManager.js

@@ -35,7 +35,7 @@ const async = require('async')
 const Errors = require('./Errors')
 const CommandRunner = require('./CommandRunner')
 
-const getCompileName = function(project_id, user_id) {
+const getCompileName = function (project_id, user_id) {
   if (user_id != null) {
     return `${project_id}-${user_id}`
   } else {
@@ -49,22 +49,19 @@ const getCompileDir = (project_id, user_id) =>
 module.exports = CompileManager = {
   doCompileWithLock(request, callback) {
     if (callback == null) {
-      callback = function(error, outputFiles) {}
+      callback = function (error, outputFiles) {}
     }
     const compileDir = getCompileDir(request.project_id, request.user_id)
     const lockFile = Path.join(compileDir, '.project-lock')
-    // create local home and tmp directories in the compile dir
-    const homeDir = Path.join(compileDir, '.project-home')
-    const tmpDir = Path.join(compileDir, '.project-tmp')
     // use a .project-lock file in the compile directory to prevent
     // simultaneous compiles
-    async.each([compileDir, homeDir, tmpDir], fse.ensureDir, function (error) {
+    return fse.ensureDir(compileDir, function (error) {
       if (error != null) {
         return callback(error)
       }
       return LockManager.runWithLock(
         lockFile,
-        releaseLock => CompileManager.doCompile(request, releaseLock),
+        (releaseLock) => CompileManager.doCompile(request, releaseLock),
         callback
       )
     })
@@ -72,7 +69,7 @@ module.exports = CompileManager = {
 
   doCompile(request, callback) {
     if (callback == null) {
-      callback = function(error, outputFiles) {}
+      callback = function (error, outputFiles) {}
     }
     const compileDir = getCompileDir(request.project_id, request.user_id)
     let timer = new Metrics.Timer('write-to-disk')
@@ -80,7 +77,7 @@ module.exports = CompileManager = {
       { project_id: request.project_id, user_id: request.user_id },
       'syncing resources to disk'
     )
-    return ResourceWriter.syncResourcesToDisk(request, compileDir, function(
+    return ResourceWriter.syncResourcesToDisk(request, compileDir, function (
       error,
       resourceList
     ) {
@@ -112,7 +109,7 @@ module.exports = CompileManager = {
       )
       timer.done()
 
-      const injectDraftModeIfRequired = function(callback) {
+      const injectDraftModeIfRequired = function (callback) {
         if (request.draft) {
           return DraftModeManager.injectDraftMode(
             Path.join(compileDir, request.rootResourcePath),
@@ -123,12 +120,12 @@ module.exports = CompileManager = {
         }
       }
 
-      const createTikzFileIfRequired = callback =>
+      const createTikzFileIfRequired = (callback) =>
         TikzManager.checkMainFile(
           compileDir,
           request.rootResourcePath,
           resourceList,
-          function(error, needsMainFile) {
+          function (error, needsMainFile) {
             if (error != null) {
               return callback(error)
             }
@@ -145,6 +142,10 @@ module.exports = CompileManager = {
         )
       // set up environment variables for chktex
       const env = {}
+      if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') {
+        // override default texlive openout_any environment variable
+        env.openout_any = Settings.texliveOpenoutAny
+      }
       // only run chktex on LaTeX files (not knitr .Rtex files or any others)
       const isLaTeXFile =
         request.rootResourcePath != null
@@ -164,7 +165,7 @@ module.exports = CompileManager = {
       // apply a series of file modifications/creations for draft mode and tikz
       return async.series(
         [injectDraftModeIfRequired, createTikzFileIfRequired],
-        function(error) {
+        function (error) {
           if (error != null) {
             return callback(error)
           }
@@ -176,9 +177,9 @@ module.exports = CompileManager = {
                 request.imageName != null
                   ? request.imageName.match(/:(.*)/)
                   : undefined,
-                x1 => x1[1]
+                (x1) => x1[1]
               ),
-              x => x.replace(/\./g, '-')
+              (x) => x.replace(/\./g, '-')
             ) || 'default'
           if (!request.project_id.match(/^[0-9a-f]{24}$/)) {
             tag = 'other'
@@ -198,15 +199,14 @@ module.exports = CompileManager = {
               timeout: request.timeout,
               image: request.imageName,
               flags: request.flags,
-              environment: env
+              environment: env,
+              compileGroup: request.compileGroup
             },
-            function(error, output, stats, timings) {
+            function (error, output, stats, timings) {
               // request was for validation only
               let metric_key, metric_value
               if (request.check === 'validate') {
-                const result = (error != null
-                ? error.code
-                : undefined)
+                const result = (error != null ? error.code : undefined)
                   ? 'fail'
                   : 'pass'
                 error = new Error('validation')
@@ -229,7 +229,7 @@ module.exports = CompileManager = {
                 OutputFileFinder.findOutputFiles(
                   resourceList,
                   compileDir,
-                  function(err, outputFiles) {
+                  function (err, outputFiles) {
                     if (err != null) {
                       return callback(err)
                     }
@@ -287,7 +287,7 @@ module.exports = CompileManager = {
               return OutputFileFinder.findOutputFiles(
                 resourceList,
                 compileDir,
-                function(error, outputFiles) {
+                function (error, outputFiles) {
                   if (error != null) {
                     return callback(error)
                   }
@@ -307,7 +307,7 @@ module.exports = CompileManager = {
 
   stopCompile(project_id, user_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const compileName = getCompileName(project_id, user_id)
     return LatexRunner.killLatex(compileName, callback)
@@ -315,16 +315,16 @@ module.exports = CompileManager = {
 
   clearProject(project_id, user_id, _callback) {
     if (_callback == null) {
-      _callback = function(error) {}
+      _callback = function (error) {}
     }
-    const callback = function(error) {
+    const callback = function (error) {
       _callback(error)
-      return (_callback = function() {})
+      return (_callback = function () {})
     }
 
     const compileDir = getCompileDir(project_id, user_id)
 
-    return CompileManager._checkDirectory(compileDir, function(err, exists) {
+    return CompileManager._checkDirectory(compileDir, function (err, exists) {
       if (err != null) {
         return callback(err)
       }
@@ -337,9 +337,9 @@ module.exports = CompileManager = {
       proc.on('error', callback)
 
       let stderr = ''
-      proc.stderr.on('data', chunk => (stderr += chunk.toString()))
+      proc.stderr.setEncoding('utf8').on('data', (chunk) => (stderr += chunk))
 
-      return proc.on('close', function(code) {
+      return proc.on('close', function (code) {
         if (code === 0) {
           return callback(null)
         } else {
@@ -351,26 +351,26 @@ module.exports = CompileManager = {
 
   _findAllDirs(callback) {
     if (callback == null) {
-      callback = function(error, allDirs) {}
+      callback = function (error, allDirs) {}
     }
     const root = Settings.path.compilesDir
-    return fs.readdir(root, function(err, files) {
+    return fs.readdir(root, function (err, files) {
       if (err != null) {
         return callback(err)
       }
-      const allDirs = Array.from(files).map(file => Path.join(root, file))
+      const allDirs = Array.from(files).map((file) => Path.join(root, file))
       return callback(null, allDirs)
     })
   },
 
   clearExpiredProjects(max_cache_age_ms, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const now = Date.now()
     // action for each directory
     const expireIfNeeded = (checkDir, cb) =>
-      fs.stat(checkDir, function(err, stats) {
+      fs.stat(checkDir, function (err, stats) {
         if (err != null) {
           return cb()
         } // ignore errors checking directory
@@ -383,7 +383,7 @@ module.exports = CompileManager = {
         }
       })
     // iterate over all project directories
-    return CompileManager._findAllDirs(function(error, allDirs) {
+    return CompileManager._findAllDirs(function (error, allDirs) {
       if (error != null) {
         return callback()
       }
@@ -393,9 +393,9 @@ module.exports = CompileManager = {
 
   _checkDirectory(compileDir, callback) {
     if (callback == null) {
-      callback = function(error, exists) {}
+      callback = function (error, exists) {}
     }
-    return fs.lstat(compileDir, function(err, stats) {
+    return fs.lstat(compileDir, function (err, stats) {
       if ((err != null ? err.code : undefined) === 'ENOENT') {
         return callback(null, false) //  directory does not exist
       } else if (err != null) {
@@ -421,7 +421,7 @@ module.exports = CompileManager = {
     // might not match the file path on the host. The .synctex.gz file however, will be accessed
     // wherever it is on the host.
     if (callback == null) {
-      callback = function(error, pdfPositions) {}
+      callback = function (error, pdfPositions) {}
     }
     const compileName = getCompileName(project_id, user_id)
     const base_dir = Settings.path.synctexBaseDir(compileName)
@@ -429,15 +429,7 @@ module.exports = CompileManager = {
     const compileDir = getCompileDir(project_id, user_id)
     const synctex_path = `${base_dir}/output.pdf`
     const command = ['code', synctex_path, file_path, line, column]
-    return fse.ensureDir(compileDir, function(error) {
-      if (error != null) {
-        logger.err(
-          { error, project_id, user_id, file_name },
-          'error ensuring dir for sync from code'
-        )
-        return callback(error)
-      }
-      return CompileManager._runSynctex(project_id, user_id, command, function(
+    CompileManager._runSynctex(project_id, user_id, command, function (
       error,
       stdout
     ) {
@@ -448,32 +440,20 @@ module.exports = CompileManager = {
         { project_id, user_id, file_name, line, column, command, stdout },
         'synctex code output'
       )
-        return callback(
-          null,
-          CompileManager._parseSynctexFromCodeOutput(stdout)
-        )
-      })
+      return callback(null, CompileManager._parseSynctexFromCodeOutput(stdout))
     })
   },
 
   syncFromPdf(project_id, user_id, page, h, v, callback) {
     if (callback == null) {
-      callback = function(error, filePositions) {}
+      callback = function (error, filePositions) {}
     }
     const compileName = getCompileName(project_id, user_id)
     const compileDir = getCompileDir(project_id, user_id)
     const base_dir = Settings.path.synctexBaseDir(compileName)
     const synctex_path = `${base_dir}/output.pdf`
     const command = ['pdf', synctex_path, page, h, v]
-    return fse.ensureDir(compileDir, function(error) {
-      if (error != null) {
-        logger.err(
-          { error, project_id, user_id, file_name },
-          'error ensuring dir for sync to code'
-        )
-        return callback(error)
-      }
-      return CompileManager._runSynctex(project_id, user_id, command, function(
+    CompileManager._runSynctex(project_id, user_id, command, function (
       error,
       stdout
     ) {
@@ -489,29 +469,23 @@ module.exports = CompileManager = {
         CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
       )
     })
-    })
   },
 
-  _checkFileExists(path, callback) {
+  _checkFileExists(dir, filename, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    const synctexDir = Path.dirname(path)
-    const synctexFile = Path.join(synctexDir, 'output.synctex.gz')
-    return fs.stat(synctexDir, function(error, stats) {
+    const file = Path.join(dir, filename)
+    return fs.stat(dir, function (error, stats) {
       if ((error != null ? error.code : undefined) === 'ENOENT') {
-        return callback(
-          new Errors.NotFoundError('called synctex with no output directory')
-        )
+        return callback(new Errors.NotFoundError('no output directory'))
       }
       if (error != null) {
         return callback(error)
       }
-      return fs.stat(synctexFile, function(error, stats) {
+      return fs.stat(file, function (error, stats) {
         if ((error != null ? error.code : undefined) === 'ENOENT') {
-          return callback(
-            new Errors.NotFoundError('called synctex with no output file')
-          )
+          return callback(new Errors.NotFoundError('no output file'))
         }
         if (error != null) {
           return callback(error)
@@ -526,7 +500,7 @@ module.exports = CompileManager = {
 
   _runSynctex(project_id, user_id, command, callback) {
     if (callback == null) {
-      callback = function(error, stdout) {}
+      callback = function (error, stdout) {}
     }
     const seconds = 1000
 
@@ -535,14 +509,22 @@ module.exports = CompileManager = {
     const directory = getCompileDir(project_id, user_id)
     const timeout = 60 * 1000 // increased to allow for large projects
     const compileName = getCompileName(project_id, user_id)
+    const compileGroup = 'synctex'
+    CompileManager._checkFileExists(directory, 'output.synctex.gz', (error) => {
+      if (error) {
+        return callback(error)
+      }
       return CommandRunner.run(
         compileName,
         command,
         directory,
-      Settings.clsi != null ? Settings.clsi.docker.image : undefined,
+        Settings.clsi && Settings.clsi.docker
+          ? Settings.clsi.docker.image
+          : undefined,
         timeout,
         {},
-      function(error, output) {
+        compileGroup,
+        function (error, output) {
           if (error != null) {
             logger.err(
               { err: error, command, project_id, user_id },
@@ -553,6 +535,7 @@ module.exports = CompileManager = {
           return callback(null, output.stdout)
         }
       )
+    })
   },
 
   _parseSynctexFromCodeOutput(output) {
@@ -591,7 +574,7 @@ module.exports = CompileManager = {
 
   wordcount(project_id, user_id, file_name, image, callback) {
     if (callback == null) {
-      callback = function(error, pdfPositions) {}
+      callback = function (error, pdfPositions) {}
     }
     logger.log({ project_id, user_id, file_name, image }, 'running wordcount')
     const file_path = `$COMPILE_DIR/${file_name}`
@@ -605,7 +588,8 @@ module.exports = CompileManager = {
     const compileDir = getCompileDir(project_id, user_id)
     const timeout = 60 * 1000
     const compileName = getCompileName(project_id, user_id)
-    return fse.ensureDir(compileDir, function(error) {
+    const compileGroup = 'wordcount'
+    return fse.ensureDir(compileDir, function (error) {
       if (error != null) {
         logger.err(
           { error, project_id, user_id, file_name },
@@ -620,14 +604,15 @@ module.exports = CompileManager = {
         image,
         timeout,
         {},
-        function(error) {
+        compileGroup,
+        function (error) {
           if (error != null) {
             return callback(error)
           }
           return fs.readFile(
             compileDir + '/' + file_name + '.wc',
             'utf-8',
-            function(err, stdout) {
+            function (err, stdout) {
               if (err != null) {
                 // call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
                 logger.err(

app/js/DockerLockManager.js

@@ -23,7 +23,7 @@ module.exports = LockManager = {
   tryLock(key, callback) {
     let lockValue
     if (callback == null) {
-      callback = function(err, gotLock) {}
+      callback = function (err, gotLock) {}
     }
     const existingLock = LockState[key]
     if (existingLock != null) {
@@ -46,11 +46,11 @@ module.exports = LockManager = {
   getLock(key, callback) {
     let attempt
     if (callback == null) {
-      callback = function(error, lockValue) {}
+      callback = function (error, lockValue) {}
     }
     const startTime = Date.now()
     return (attempt = () =>
-      LockManager.tryLock(key, function(error, gotLock, lockValue) {
+      LockManager.tryLock(key, function (error, gotLock, lockValue) {
         if (error != null) {
           return callback(error)
         }
@@ -68,7 +68,7 @@ module.exports = LockManager = {
 
   releaseLock(key, lockValue, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const existingLock = LockState[key]
     if (existingLock === lockValue) {
@@ -93,14 +93,14 @@ module.exports = LockManager = {
 
   runWithLock(key, runner, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return LockManager.getLock(key, function(error, lockValue) {
+    return LockManager.getLock(key, function (error, lockValue) {
       if (error != null) {
         return callback(error)
       }
       return runner((error1, ...args) =>
-        LockManager.releaseLock(key, lockValue, function(error2) {
+        LockManager.releaseLock(key, lockValue, function (error2) {
           error = error1 || error2
           if (error != null) {
             return callback(error)

app/js/DockerRunner.js

@@ -1,21 +1,3 @@
-/* eslint-disable
-    camelcase,
-    handle-callback-err,
-    no-return-assign,
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS101: Remove unnecessary use of Array.from
- * DS102: Remove unnecessary code created because of implicit returns
- * DS103: Rewrite code to no longer use __guard__
- * DS205: Consider reworking code to avoid use of IIFEs
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-let DockerRunner, oneHour
 const Settings = require('settings-sharelatex')
 const logger = require('logger-sharelatex')
 const Docker = require('dockerode')
@@ -25,30 +7,33 @@ const async = require('async')
 const LockManager = require('./DockerLockManager')
 const fs = require('fs')
 const Path = require('path')
-const _ = require('underscore')
+const _ = require('lodash')
 
+const ONE_HOUR_IN_MS = 60 * 60 * 1000
 logger.info('using docker runner')
 
-const usingSiblingContainers = () =>
-  __guard__(
-    Settings != null ? Settings.path : undefined,
-    x => x.sandboxedCompilesHostDir
-  ) != null
+function usingSiblingContainers() {
+  return (
+    Settings != null &&
+    Settings.path != null &&
+    Settings.path.sandboxedCompilesHostDir != null
+  )
+}
 
 let containerMonitorTimeout
 let containerMonitorInterval
 
-module.exports = DockerRunner = {
-  ERR_NOT_DIRECTORY: new Error('not a directory'),
-  ERR_TERMINATED: new Error('terminated'),
-  ERR_EXITED: new Error('exited'),
-  ERR_TIMED_OUT: new Error('container timed out'),
-
-  run(project_id, command, directory, image, timeout, environment, callback) {
-    let name
-    if (callback == null) {
-      callback = function(error, output) {}
-    }
+const DockerRunner = {
+  run(
+    projectId,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    callback
+  ) {
     if (usingSiblingContainers()) {
       const _newPath = Settings.path.sandboxedCompilesHostDir
       logger.log(
@@ -65,16 +50,20 @@ module.exports = DockerRunner = {
       )
     }
 
-    const volumes = {}
-    volumes[directory] = '/compile'
+    const volumes = { [directory]: '/compile' }
 
-    command = Array.from(command).map(arg =>
-      __guardMethod__(arg.toString(), 'replace', o =>
-        o.replace('$COMPILE_DIR', '/compile')
-      )
+    command = command.map((arg) =>
+      arg.toString().replace('$COMPILE_DIR', '/compile')
     )
     if (image == null) {
-      ;({ image } = Settings.clsi.docker)
+      image = Settings.clsi.docker.image
+    }
+
+    if (
+      Settings.clsi.docker.allowedImages &&
+      !Settings.clsi.docker.allowedImages.includes(image)
+    ) {
+      return callback(new Error('image not allowed'))
     }
 
     if (Settings.texliveImageNameOveride != null) {
@@ -87,28 +76,31 @@ module.exports = DockerRunner = {
       image,
       volumes,
       timeout,
-      environment
+      environment,
+      compileGroup
     )
     const fingerprint = DockerRunner._fingerprintContainer(options)
-    options.name = name = `project-${project_id}-${fingerprint}`
+    const name = `project-${projectId}-${fingerprint}`
+    options.name = name
 
     // logOptions = _.clone(options)
     // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
-    logger.log({ project_id }, 'running docker container')
-    DockerRunner._runAndWaitForContainer(options, volumes, timeout, function(
-      error,
-      output
-    ) {
+    logger.log({ projectId }, 'running docker container')
+    DockerRunner._runAndWaitForContainer(
+      options,
+      volumes,
+      timeout,
+      (error, output) => {
         if (error && error.statusCode === 500) {
           logger.log(
-          { err: error, project_id },
+            { err: error, projectId },
             'error running container so destroying and retrying'
           )
-        return DockerRunner.destroyContainer(name, null, true, function(error) {
+          DockerRunner.destroyContainer(name, null, true, (error) => {
             if (error != null) {
               return callback(error)
             }
-          return DockerRunner._runAndWaitForContainer(
+            DockerRunner._runAndWaitForContainer(
               options,
               volumes,
               timeout,
@@ -116,121 +108,111 @@ module.exports = DockerRunner = {
             )
           })
         } else {
-        return callback(error, output)
+          callback(error, output)
         }
-    })
+      }
+    )
 
+    // pass back the container name to allow it to be killed
     return name
-  }, // pass back the container name to allow it to be killed
+  },
 
-  kill(container_id, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    logger.log({ container_id }, 'sending kill signal to container')
-    const container = dockerode.getContainer(container_id)
-    return container.kill(function(error) {
+  kill(containerId, callback) {
+    logger.log({ containerId }, 'sending kill signal to container')
+    const container = dockerode.getContainer(containerId)
+    container.kill((error) => {
       if (
         error != null &&
-        __guardMethod__(error != null ? error.message : undefined, 'match', o =>
-          o.match(/Cannot kill container .* is not running/)
-        )
+        error.message != null &&
+        error.message.match(/Cannot kill container .* is not running/)
       ) {
         logger.warn(
-          { err: error, container_id },
+          { err: error, containerId },
           'container not running, continuing'
         )
         error = null
       }
       if (error != null) {
-        logger.error({ err: error, container_id }, 'error killing container')
-        return callback(error)
+        logger.error({ err: error, containerId }, 'error killing container')
+        callback(error)
       } else {
-        return callback()
+        callback()
       }
     })
   },
 
   _runAndWaitForContainer(options, volumes, timeout, _callback) {
-    if (_callback == null) {
-      _callback = function(error, output) {}
-    }
-    const callback = function(...args) {
-      _callback(...Array.from(args || []))
-      // Only call the callback once
-      return (_callback = function() {})
-    }
-
+    const callback = _.once(_callback)
     const { name } = options
 
     let streamEnded = false
     let containerReturned = false
     let output = {}
 
-    const callbackIfFinished = function() {
+    function callbackIfFinished() {
       if (streamEnded && containerReturned) {
-        return callback(null, output)
+        callback(null, output)
       }
     }
 
-    const attachStreamHandler = function(error, _output) {
+    function attachStreamHandler(error, _output) {
       if (error != null) {
         return callback(error)
       }
       output = _output
       streamEnded = true
-      return callbackIfFinished()
+      callbackIfFinished()
     }
 
-    return DockerRunner.startContainer(
+    DockerRunner.startContainer(
       options,
       volumes,
       attachStreamHandler,
-      function(error, containerId) {
+      (error, containerId) => {
         if (error != null) {
           return callback(error)
         }
 
-        return DockerRunner.waitForContainer(name, timeout, function(
-          error,
-          exitCode
-        ) {
-          let err
+        DockerRunner.waitForContainer(name, timeout, (error, exitCode) => {
           if (error != null) {
             return callback(error)
           }
           if (exitCode === 137) {
             // exit status from kill -9
-            err = DockerRunner.ERR_TERMINATED
+            const err = new Error('terminated')
             err.terminated = true
             return callback(err)
           }
           if (exitCode === 1) {
             // exit status from chktex
-            err = DockerRunner.ERR_EXITED
+            const err = new Error('exited')
             err.code = exitCode
             return callback(err)
           }
           containerReturned = true
-          __guard__(
-            options != null ? options.HostConfig : undefined,
-            x => (x.SecurityOpt = null)
-          ) // small log line
-          logger.log({ err, exitCode, options }, 'docker container has exited')
-          return callbackIfFinished()
+          if (options != null && options.HostConfig != null) {
+            options.HostConfig.SecurityOpt = null
+          }
+          logger.log({ exitCode, options }, 'docker container has exited')
+          callbackIfFinished()
         })
       }
     )
   },
 
-  _getContainerOptions(command, image, volumes, timeout, environment) {
-    let m, year
-    let key, value, hostVol, dockerVol
+  _getContainerOptions(
+    command,
+    image,
+    volumes,
+    timeout,
+    environment,
+    compileGroup
+  ) {
     const timeoutInSeconds = timeout / 1000
 
     const dockerVolumes = {}
-    for (hostVol in volumes) {
-      dockerVol = volumes[hostVol]
+    for (const hostVol in volumes) {
+      const dockerVol = volumes[hostVol]
       dockerVolumes[dockerVol] = {}
 
       if (volumes[hostVol].slice(-3).indexOf(':r') === -1) {
@@ -241,17 +223,14 @@ module.exports = DockerRunner = {
     // merge settings and environment parameter
     const env = {}
     for (const src of [Settings.clsi.docker.env, environment || {}]) {
-      for (key in src) {
-        value = src[key]
+      for (const key in src) {
+        const value = src[key]
         env[key] = value
       }
     }
     // set the path based on the image year
-    if ((m = image.match(/:([0-9]+)\.[0-9]+/))) {
-      year = m[1]
-    } else {
-      year = '2014'
-    }
+    const match = image.match(/:([0-9]+)\.[0-9]+/)
+    const year = match ? match[1] : '2014'
     env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`
     const options = {
       Cmd: command,
@@ -261,23 +240,11 @@ module.exports = DockerRunner = {
       NetworkDisabled: true,
       Memory: 1024 * 1024 * 1024 * 1024, // 1 Gb
       User: Settings.clsi.docker.user,
-      Env: (() => {
-        const result = []
-        for (key in env) {
-          value = env[key]
-          result.push(`${key}=${value}`)
-        }
-        return result
-      })(), // convert the environment hash to an array
+      Env: Object.entries(env).map(([key, value]) => `${key}=${value}`),
       HostConfig: {
-        Binds: (() => {
-          const result1 = []
-          for (hostVol in volumes) {
-            dockerVol = volumes[hostVol]
-            result1.push(`${hostVol}:${dockerVol}`)
-          }
-          return result1
-        })(),
+        Binds: Object.entries(volumes).map(
+          ([hostVol, dockerVol]) => `${hostVol}:${dockerVol}`
+        ),
         LogConfig: { Type: 'none', Config: {} },
         Ulimits: [
           {
@@ -291,10 +258,7 @@ module.exports = DockerRunner = {
       }
     }
 
-    if (
-      (Settings.path != null ? Settings.path.synctexBinHostPath : undefined) !=
-      null
-    ) {
+    if (Settings.path != null && Settings.path.synctexBinHostPath != null) {
       options.HostConfig.Binds.push(
         `${Settings.path.synctexBinHostPath}:/opt/synctex:ro`
       )
@@ -310,31 +274,45 @@ module.exports = DockerRunner = {
       options.HostConfig.Runtime = Settings.clsi.docker.runtime
     }
 
+    if (Settings.clsi.docker.Readonly) {
+      options.HostConfig.ReadonlyRootfs = true
+      options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' }
+      options.Volumes['/home/tex'] = {}
+    }
+
+    // Allow per-compile group overriding of individual settings
+    if (
+      Settings.clsi.docker.compileGroupConfig &&
+      Settings.clsi.docker.compileGroupConfig[compileGroup]
+    ) {
+      const override = Settings.clsi.docker.compileGroupConfig[compileGroup]
+      for (const key in override) {
+        _.set(options, key, override[key])
+      }
+    }
+
     return options
   },
 
   _fingerprintContainer(containerOptions) {
     // Yay, Hashing!
     const json = JSON.stringify(containerOptions)
-    return crypto
-      .createHash('md5')
-      .update(json)
-      .digest('hex')
+    return crypto.createHash('md5').update(json).digest('hex')
   },
 
   startContainer(options, volumes, attachStreamHandler, callback) {
-    return LockManager.runWithLock(
+    LockManager.runWithLock(
       options.name,
-      releaseLock =>
+      (releaseLock) =>
         // Check that volumes exist before starting the container.
         // When a container is started with volume pointing to a
         // non-existent directory then docker creates the directory but
         // with root ownership.
-        DockerRunner._checkVolumes(options, volumes, function(err) {
+        DockerRunner._checkVolumes(options, volumes, (err) => {
           if (err != null) {
             return releaseLock(err)
           }
-          return DockerRunner._startContainer(
+          DockerRunner._startContainer(
             options,
             volumes,
             attachStreamHandler,
@@ -348,93 +326,85 @@ module.exports = DockerRunner = {
 
   // Check that volumes exist and are directories
   _checkVolumes(options, volumes, callback) {
-    if (callback == null) {
-      callback = function(error, containerName) {}
-    }
     if (usingSiblingContainers()) {
       // Server Pro, with sibling-containers active, skip checks
       return callback(null)
     }
 
     const checkVolume = (path, cb) =>
-      fs.stat(path, function(err, stats) {
+      fs.stat(path, (err, stats) => {
         if (err != null) {
           return cb(err)
         }
-        if (!(stats != null ? stats.isDirectory() : undefined)) {
-          return cb(DockerRunner.ERR_NOT_DIRECTORY)
+        if (!stats.isDirectory()) {
+          return cb(new Error('not a directory'))
         }
-        return cb()
+        cb()
       })
     const jobs = []
     for (const vol in volumes) {
-      ;(vol => jobs.push(cb => checkVolume(vol, cb)))(vol)
+      jobs.push((cb) => checkVolume(vol, cb))
     }
-    return async.series(jobs, callback)
+    async.series(jobs, callback)
   },
 
   _startContainer(options, volumes, attachStreamHandler, callback) {
-    if (callback == null) {
-      callback = function(error, output) {}
-    }
     callback = _.once(callback)
     const { name } = options
 
     logger.log({ container_name: name }, 'starting container')
     const container = dockerode.getContainer(name)
 
-    const createAndStartContainer = () =>
-      dockerode.createContainer(options, function(error, container) {
+    function createAndStartContainer() {
+      dockerode.createContainer(options, (error, container) => {
         if (error != null) {
           return callback(error)
         }
-        return startExistingContainer()
+        startExistingContainer()
       })
-    var startExistingContainer = () =>
+    }
+
+    function startExistingContainer() {
       DockerRunner.attachToContainer(
         options.name,
         attachStreamHandler,
-        function(error) {
+        (error) => {
           if (error != null) {
             return callback(error)
           }
-          return container.start(function(error) {
-            if (
-              error != null &&
-              (error != null ? error.statusCode : undefined) !== 304
-            ) {
-              // already running
-              return callback(error)
+          container.start((error) => {
+            if (error != null && error.statusCode !== 304) {
+              callback(error)
             } else {
-              return callback()
+              // already running
+              callback()
             }
           })
         }
       )
-    return container.inspect(function(error, stats) {
-      if ((error != null ? error.statusCode : undefined) === 404) {
-        return createAndStartContainer()
+    }
+
+    container.inspect((error, stats) => {
+      if (error != null && error.statusCode === 404) {
+        createAndStartContainer()
       } else if (error != null) {
         logger.err(
           { container_name: name, error },
           'unable to inspect container to start'
         )
-        return callback(error)
+        callback(error)
       } else {
-        return startExistingContainer()
+        startExistingContainer()
       }
     })
   },
 
   attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
     const container = dockerode.getContainer(containerId)
-    return container.attach({ stdout: 1, stderr: 1, stream: 1 }, function(
-      error,
-      stream
-    ) {
+    container.attach({ stdout: 1, stderr: 1, stream: 1 }, (error, stream) => {
       if (error != null) {
         logger.error(
-          { err: error, container_id: containerId },
+          { err: error, containerId },
           'error attaching to container'
         )
         return attachStartCallback(error)
@@ -442,10 +412,10 @@ module.exports = DockerRunner = {
         attachStartCallback()
       }
 
-      logger.log({ container_id: containerId }, 'attached to container')
+      logger.log({ containerId }, 'attached to container')
 
       const MAX_OUTPUT = 1024 * 1024 // limit output to 1MB
-      const createStringOutputStream = function(name) {
+      function createStringOutputStream(name) {
         return {
           data: '',
           overflowed: false,
@@ -454,18 +424,18 @@ module.exports = DockerRunner = {
               return
             }
             if (this.data.length < MAX_OUTPUT) {
-              return (this.data += data)
+              this.data += data
             } else {
               logger.error(
                 {
-                  container_id: containerId,
+                  containerId,
                   length: this.data.length,
                   maxLen: MAX_OUTPUT
                 },
                 `${name} exceeds max size`
               )
               this.data += `(...truncated at ${MAX_OUTPUT} chars...)`
-              return (this.overflowed = true)
+              this.overflowed = true
             }
           }
           // kill container if too much output
@@ -478,63 +448,52 @@ module.exports = DockerRunner = {
 
       container.modem.demuxStream(stream, stdout, stderr)
 
-      stream.on('error', err =>
+      stream.on('error', (err) =>
         logger.error(
-          { err, container_id: containerId },
+          { err, containerId },
           'error reading from container stream'
         )
       )
 
-      return stream.on('end', () =>
+      stream.on('end', () =>
         attachStreamHandler(null, { stdout: stdout.data, stderr: stderr.data })
       )
     })
   },
 
   waitForContainer(containerId, timeout, _callback) {
-    if (_callback == null) {
-      _callback = function(error, exitCode) {}
-    }
-    const callback = function(...args) {
-      _callback(...Array.from(args || []))
-      // Only call the callback once
-      return (_callback = function() {})
-    }
+    const callback = _.once(_callback)
 
     const container = dockerode.getContainer(containerId)
 
     let timedOut = false
-    const timeoutId = setTimeout(function() {
+    const timeoutId = setTimeout(() => {
       timedOut = true
-      logger.log(
-        { container_id: containerId },
-        'timeout reached, killing container'
-      )
-      return container.kill(function() {})
+      logger.log({ containerId }, 'timeout reached, killing container')
+      container.kill((err) => {
+        logger.warn({ err, containerId }, 'failed to kill container')
+      })
     }, timeout)
 
-    logger.log({ container_id: containerId }, 'waiting for docker container')
-    return container.wait(function(error, res) {
+    logger.log({ containerId }, 'waiting for docker container')
+    container.wait((error, res) => {
       if (error != null) {
         clearTimeout(timeoutId)
-        logger.error(
-          { err: error, container_id: containerId },
-          'error waiting for container'
-        )
+        logger.error({ err: error, containerId }, 'error waiting for container')
         return callback(error)
       }
       if (timedOut) {
         logger.log({ containerId }, 'docker container timed out')
-        error = DockerRunner.ERR_TIMED_OUT
+        error = new Error('container timed out')
         error.timedout = true
-        return callback(error)
+        callback(error)
       } else {
         clearTimeout(timeoutId)
         logger.log(
-          { container_id: containerId, exitCode: res.StatusCode },
+          { containerId, exitCode: res.StatusCode },
           'docker container returned'
         )
-        return callback(null, res.StatusCode)
+        callback(null, res.StatusCode)
       }
     })
   },
@@ -546,12 +505,9 @@ module.exports = DockerRunner = {
     // async exception, but if you delete by id it just does a normal
     // error callback. We fall back to deleting by name if no id is
     // supplied.
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    return LockManager.runWithLock(
+    LockManager.runWithLock(
       containerName,
-      releaseLock =>
+      (releaseLock) =>
         DockerRunner._destroyContainer(
           containerId || containerName,
           shouldForce,
@@ -562,46 +518,31 @@ module.exports = DockerRunner = {
   },
 
   _destroyContainer(containerId, shouldForce, callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    logger.log({ container_id: containerId }, 'destroying docker container')
+    logger.log({ containerId }, 'destroying docker container')
     const container = dockerode.getContainer(containerId)
-    return container.remove({ force: shouldForce === true }, function(error) {
-      if (
-        error != null &&
-        (error != null ? error.statusCode : undefined) === 404
-      ) {
+    container.remove({ force: shouldForce === true, v: true }, (error) => {
+      if (error != null && error.statusCode === 404) {
         logger.warn(
-          { err: error, container_id: containerId },
+          { err: error, containerId },
           'container not found, continuing'
         )
         error = null
       }
       if (error != null) {
-        logger.error(
-          { err: error, container_id: containerId },
-          'error destroying container'
-        )
+        logger.error({ err: error, containerId }, 'error destroying container')
       } else {
-        logger.log({ container_id: containerId }, 'destroyed container')
+        logger.log({ containerId }, 'destroyed container')
       }
-      return callback(error)
+      callback(error)
     })
   },
 
   // handle expiry of docker containers
 
-  MAX_CONTAINER_AGE:
-    Settings.clsi.docker.maxContainerAge || (oneHour = 60 * 60 * 1000),
+  MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || ONE_HOUR_IN_MS,
 
   examineOldContainer(container, callback) {
-    if (callback == null) {
-      callback = function(error, name, id, ttl) {}
-    }
-    const name =
-      container.Name ||
-      (container.Names != null ? container.Names[0] : undefined)
+    const name = container.Name || (container.Names && container.Names[0])
     const created = container.Created * 1000 // creation time is returned in seconds
     const now = Date.now()
     const age = now - created
@@ -611,36 +552,29 @@ module.exports = DockerRunner = {
       { containerName: name, created, now, age, maxAge, ttl },
       'checking whether to destroy container'
     )
-    return callback(null, name, container.Id, ttl)
+    return { name, id: container.Id, ttl }
   },
 
   destroyOldContainers(callback) {
-    if (callback == null) {
-      callback = function(error) {}
-    }
-    return dockerode.listContainers({ all: true }, function(error, containers) {
+    dockerode.listContainers({ all: true }, (error, containers) => {
       if (error != null) {
         return callback(error)
       }
       const jobs = []
-      for (const container of Array.from(containers || [])) {
-        ;(container =>
-          DockerRunner.examineOldContainer(container, function(
-            err,
-            name,
-            id,
-            ttl
-          ) {
+      for (const container of containers) {
+        const { name, id, ttl } = DockerRunner.examineOldContainer(container)
         if (name.slice(0, 9) === '/project-' && ttl <= 0) {
-              return jobs.push(cb =>
-                DockerRunner.destroyContainer(name, id, false, () => cb())
+          // strip the / prefix
+          // the LockManager uses the plain container name
+          const plainName = name.slice(1)
+          jobs.push((cb) =>
+            DockerRunner.destroyContainer(plainName, id, false, () => cb())
           )
         }
-          }))(container)
       }
       // Ignore errors because some containers get stuck but
       // will be destroyed next time
-      return async.series(jobs, callback)
+      async.series(jobs, callback)
     })
   },
 
@@ -657,8 +591,13 @@ module.exports = DockerRunner = {
     const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
     containerMonitorTimeout = setTimeout(() => {
       containerMonitorInterval = setInterval(
-        () => DockerRunner.destroyOldContainers(),
-        (oneHour = 60 * 60 * 1000)
+        () =>
+          DockerRunner.destroyOldContainers((err) => {
+            if (err) {
+              logger.error({ err }, 'failed to destroy old containers')
+            }
+          }),
+        ONE_HOUR_IN_MS
       )
     }, randomDelay)
   },
@@ -669,27 +608,12 @@ module.exports = DockerRunner = {
       containerMonitorTimeout = undefined
     }
     if (containerMonitorInterval) {
-      clearInterval(containerMonitorTimeout)
-      containerMonitorTimeout = undefined
+      clearInterval(containerMonitorInterval)
+      containerMonitorInterval = undefined
     }
   }
 }
 
 DockerRunner.startContainerMonitor()
 
-function __guard__(value, transform) {
-  return typeof value !== 'undefined' && value !== null
-    ? transform(value)
-    : undefined
-}
-function __guardMethod__(obj, methodName, transform) {
-  if (
-    typeof obj !== 'undefined' &&
-    obj !== null &&
-    typeof obj[methodName] === 'function'
-  ) {
-    return transform(obj, methodName)
-  } else {
-    return undefined
-  }
-}
+module.exports = DockerRunner

app/js/DraftModeManager.js

@@ -18,9 +18,9 @@ const logger = require('logger-sharelatex')
 module.exports = DraftModeManager = {
   injectDraftMode(filename, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return fs.readFile(filename, 'utf8', function(error, content) {
+    return fs.readFile(filename, 'utf8', function (error, content) {
       if (error != null) {
         return callback(error)
       }

app/js/Errors.js

@@ -5,7 +5,7 @@
 // TODO: This file was created by bulk-decaffeinate.
 // Fix any style issues and re-enable lint.
 let Errors
-var NotFoundError = function(message) {
+var NotFoundError = function (message) {
   const error = new Error(message)
   error.name = 'NotFoundError'
   error.__proto__ = NotFoundError.prototype
@@ -13,7 +13,7 @@ var NotFoundError = function(message) {
 }
 NotFoundError.prototype.__proto__ = Error.prototype
 
-var FilesOutOfSyncError = function(message) {
+var FilesOutOfSyncError = function (message) {
   const error = new Error(message)
   error.name = 'FilesOutOfSyncError'
   error.__proto__ = FilesOutOfSyncError.prototype
@@ -21,7 +21,7 @@ var FilesOutOfSyncError = function(message) {
 }
 FilesOutOfSyncError.prototype.__proto__ = Error.prototype
 
-var AlreadyCompilingError = function(message) {
+var AlreadyCompilingError = function (message) {
   const error = new Error(message)
   error.name = 'AlreadyCompilingError'
   error.__proto__ = AlreadyCompilingError.prototype

app/js/LatexRunner.js

@@ -19,6 +19,7 @@ const Settings = require('settings-sharelatex')
 const logger = require('logger-sharelatex')
 const Metrics = require('./Metrics')
 const CommandRunner = require('./CommandRunner')
+const fs = require('fs')
 
 const ProcessTable = {} // table of currently running jobs (pids or docker container names)
 
@@ -26,7 +27,7 @@ module.exports = LatexRunner = {
   runLatex(project_id, options, callback) {
     let command
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     let {
       directory,
@@ -35,7 +36,8 @@ module.exports = LatexRunner = {
       timeout,
       image,
       environment,
-      flags
+      flags,
+      compileGroup
     } = options
     if (!compiler) {
       compiler = 'pdflatex'
@@ -45,7 +47,15 @@ module.exports = LatexRunner = {
     } // milliseconds
 
     logger.log(
-      { directory, compiler, timeout, mainFile, environment, flags },
+      {
+        directory,
+        compiler,
+        timeout,
+        mainFile,
+        environment,
+        flags,
+        compileGroup
+      },
       'starting compile'
     )
 
@@ -78,20 +88,21 @@ module.exports = LatexRunner = {
       image,
       timeout,
       environment,
-      function(error, output) {
+      compileGroup,
+      function (error, output) {
         delete ProcessTable[id]
         if (error != null) {
           return callback(error)
         }
         const runs =
           __guard__(
-            __guard__(output != null ? output.stderr : undefined, x1 =>
+            __guard__(output != null ? output.stderr : undefined, (x1) =>
               x1.match(/^Run number \d+ of .*latex/gm)
             ),
-            x => x.length
+            (x) => x.length
           ) || 0
         const failed =
-          __guard__(output != null ? output.stdout : undefined, x2 =>
+          __guard__(output != null ? output.stdout : undefined, (x2) =>
             x2.match(/^Latexmk: Errors/m)
           ) != null
             ? 1
@@ -111,30 +122,58 @@ module.exports = LatexRunner = {
             stderr != null
               ? stderr.match(/Percent of CPU this job got: (\d+)/m)
               : undefined,
-            x3 => x3[1]
+            (x3) => x3[1]
           ) || 0
         timings['cpu-time'] =
           __guard__(
             stderr != null
               ? stderr.match(/User time.*: (\d+.\d+)/m)
               : undefined,
-            x4 => x4[1]
+            (x4) => x4[1]
           ) || 0
         timings['sys-time'] =
           __guard__(
             stderr != null
               ? stderr.match(/System time.*: (\d+.\d+)/m)
               : undefined,
-            x5 => x5[1]
+            (x5) => x5[1]
           ) || 0
+        // record output files
+        LatexRunner.writeLogOutput(project_id, directory, output, () => {
           return callback(error, output, stats, timings)
+        })
       }
     ))
   },
 
+  writeLogOutput(project_id, directory, output, callback) {
+    if (!output) {
+      return callback()
+    }
+    // internal method for writing non-empty log files
+    function _writeFile(file, content, cb) {
+      if (content && content.length > 0) {
+        fs.writeFile(file, content, (err) => {
+          if (err) {
+            logger.error({ project_id, file }, 'error writing log file') // don't fail on error
+          }
+          cb()
+        })
+      } else {
+        cb()
+      }
+    }
+    // write stdout and stderr, ignoring errors
+    _writeFile(Path.join(directory, 'output.stdout'), output.stdout, () => {
+      _writeFile(Path.join(directory, 'output.stderr'), output.stderr, () => {
+        callback()
+      })
+    })
+  },
+
   killLatex(project_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const id = `${project_id}`
     logger.log({ id }, 'killing running compile')
@@ -163,7 +202,7 @@ module.exports = LatexRunner = {
     return (
       __guard__(
         Settings != null ? Settings.clsi : undefined,
-        x => x.latexmkCommandPrefix
+        (x) => x.latexmkCommandPrefix
       ) || []
     ).concat(args)
   },

app/js/LocalCommandRunner.js

@@ -15,17 +15,29 @@
  */
 let CommandRunner
 const { spawn } = require('child_process')
+const _ = require('underscore')
 const logger = require('logger-sharelatex')
 
 logger.info('using standard command runner')
 
 module.exports = CommandRunner = {
-  run(project_id, command, directory, image, timeout, environment, callback) {
+  run(
+    project_id,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    callback
+  ) {
     let key, value
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
+    } else {
+      callback = _.once(callback)
     }
-    command = Array.from(command).map(arg =>
+    command = Array.from(command).map((arg) =>
       arg.toString().replace('$COMPILE_DIR', directory)
     )
     logger.log({ project_id, command, directory }, 'running command')
@@ -46,9 +58,9 @@ module.exports = CommandRunner = {
     const proc = spawn(command[0], command.slice(1), { cwd: directory, env })
 
     let stdout = ''
-    proc.stdout.on('data', data => (stdout += data))
+    proc.stdout.setEncoding('utf8').on('data', (data) => (stdout += data))
 
-    proc.on('error', function(err) {
+    proc.on('error', function (err) {
       logger.err(
         { err, project_id, command, directory },
         'error running command'
@@ -56,7 +68,7 @@ module.exports = CommandRunner = {
       return callback(err)
     })
 
-    proc.on('close', function(code, signal) {
+    proc.on('close', function (code, signal) {
       let err
       logger.info({ code, signal, project_id }, 'command exited')
       if (signal === 'SIGTERM') {
@@ -79,7 +91,7 @@ module.exports = CommandRunner = {
 
   kill(pid, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     try {
       process.kill(-pid) // kill all processes in group

app/js/LockManager.js

@@ -25,20 +25,20 @@ module.exports = LockManager = {
 
   runWithLock(path, runner, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const lockOpts = {
       wait: this.MAX_LOCK_WAIT_TIME,
       pollPeriod: this.LOCK_TEST_INTERVAL,
       stale: this.LOCK_STALE
     }
-    return Lockfile.lock(path, lockOpts, function(error) {
+    return Lockfile.lock(path, lockOpts, function (error) {
       if ((error != null ? error.code : undefined) === 'EEXIST') {
         return callback(new Errors.AlreadyCompilingError('compile in progress'))
       } else if (error != null) {
         return fs.lstat(path, (statLockErr, statLock) =>
           fs.lstat(Path.dirname(path), (statDirErr, statDir) =>
-            fs.readdir(Path.dirname(path), function(readdirErr, readdirDir) {
+            fs.readdir(Path.dirname(path), function (readdirErr, readdirDir) {
               logger.err(
                 {
                   error,
@@ -58,7 +58,7 @@ module.exports = LockManager = {
         )
       } else {
         return runner((error1, ...args) =>
-          Lockfile.unlock(path, function(error2) {
+          Lockfile.unlock(path, function (error2) {
             error = error1 || error2
             if (error != null) {
               return callback(error)

app/js/OutputCacheManager.js

@@ -19,7 +19,7 @@ const fs = require('fs')
 const fse = require('fs-extra')
 const Path = require('path')
 const logger = require('logger-sharelatex')
-const _ = require('underscore')
+const _ = require('lodash')
 const Settings = require('settings-sharelatex')
 const crypto = require('crypto')
 
@@ -47,9 +47,9 @@ module.exports = OutputCacheManager = {
   generateBuildId(callback) {
     // generate a secure build id from Date.now() and 8 random bytes in hex
     if (callback == null) {
-      callback = function(error, buildId) {}
+      callback = function (error, buildId) {}
     }
-    return crypto.randomBytes(8, function(err, buf) {
+    return crypto.randomBytes(8, function (err, buf) {
       if (err != null) {
         return callback(err)
       }
@@ -61,9 +61,9 @@ module.exports = OutputCacheManager = {
 
   saveOutputFiles(outputFiles, compileDir, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return OutputCacheManager.generateBuildId(function(err, buildId) {
+    return OutputCacheManager.generateBuildId(function (err, buildId) {
       if (err != null) {
         return callback(err)
       }
@@ -80,7 +80,7 @@ module.exports = OutputCacheManager = {
     // make a compileDir/CACHE_SUBDIR/build_id directory and
     // copy all the output files into it
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
     // Put the files into a new cache subdirectory
@@ -99,17 +99,20 @@ module.exports = OutputCacheManager = {
       (Settings.clsi != null ? Settings.clsi.archive_logs : undefined) ||
       (Settings.clsi != null ? Settings.clsi.strace : undefined)
     ) {
-      OutputCacheManager.archiveLogs(outputFiles, compileDir, buildId, function(
-        err
-      ) {
+      OutputCacheManager.archiveLogs(
+        outputFiles,
+        compileDir,
+        buildId,
+        function (err) {
           if (err != null) {
             return logger.warn({ err }, 'erroring archiving log files')
           }
-      })
+        }
+      )
     }
 
     // make the new cache directory
-    return fse.ensureDir(cacheDir, function(err) {
+    return fse.ensureDir(cacheDir, function (err) {
       if (err != null) {
         logger.error(
           { err, directory: cacheDir },
@@ -121,7 +124,7 @@ module.exports = OutputCacheManager = {
         const results = []
         return async.mapSeries(
           outputFiles,
-          function(file, cb) {
+          function (file, cb) {
             // don't send dot files as output, express doesn't serve them
             if (OutputCacheManager._fileIsHidden(file.path)) {
               logger.debug(
@@ -136,7 +139,7 @@ module.exports = OutputCacheManager = {
               Path.join(compileDir, file.path),
               Path.join(cacheDir, file.path)
             ])
-            return OutputCacheManager._checkFileIsSafe(src, function(
+            return OutputCacheManager._checkFileIsSafe(src, function (
               err,
               isSafe
             ) {
@@ -146,7 +149,7 @@ module.exports = OutputCacheManager = {
               if (!isSafe) {
                 return cb()
               }
-              return OutputCacheManager._checkIfShouldCopy(src, function(
+              return OutputCacheManager._checkIfShouldCopy(src, function (
                 err,
                 shouldCopy
               ) {
@@ -156,7 +159,7 @@ module.exports = OutputCacheManager = {
                 if (!shouldCopy) {
                   return cb()
                 }
-                return OutputCacheManager._copyFile(src, dst, function(err) {
+                return OutputCacheManager._copyFile(src, dst, function (err) {
                   if (err != null) {
                     return cb(err)
                   }
@@ -167,12 +170,12 @@ module.exports = OutputCacheManager = {
               })
             })
           },
-          function(err) {
+          function (err) {
             if (err != null) {
               // pass back the original files if we encountered *any* error
               callback(err, outputFiles)
               // clean up the directory we just created
-              return fse.remove(cacheDir, function(err) {
+              return fse.remove(cacheDir, function (err) {
                 if (err != null) {
                   return logger.error(
                     { err, dir: cacheDir },
@@ -197,7 +200,7 @@ module.exports = OutputCacheManager = {
 
   archiveLogs(outputFiles, compileDir, buildId, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const archiveDir = Path.join(
       compileDir,
@@ -205,18 +208,18 @@ module.exports = OutputCacheManager = {
       buildId
     )
     logger.log({ dir: archiveDir }, 'archiving log files for project')
-    return fse.ensureDir(archiveDir, function(err) {
+    return fse.ensureDir(archiveDir, function (err) {
       if (err != null) {
         return callback(err)
       }
       return async.mapSeries(
         outputFiles,
-        function(file, cb) {
+        function (file, cb) {
           const [src, dst] = Array.from([
             Path.join(compileDir, file.path),
             Path.join(archiveDir, file.path)
           ])
-          return OutputCacheManager._checkFileIsSafe(src, function(
+          return OutputCacheManager._checkFileIsSafe(src, function (
             err,
             isSafe
           ) {
@@ -226,7 +229,7 @@ module.exports = OutputCacheManager = {
             if (!isSafe) {
               return cb()
             }
-            return OutputCacheManager._checkIfShouldArchive(src, function(
+            return OutputCacheManager._checkIfShouldArchive(src, function (
               err,
               shouldArchive
             ) {
@@ -248,9 +251,9 @@ module.exports = OutputCacheManager = {
   expireOutputFiles(cacheRoot, options, callback) {
     // look in compileDir for build dirs and delete if > N or age of mod time > T
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return fs.readdir(cacheRoot, function(err, results) {
+    return fs.readdir(cacheRoot, function (err, results) {
       if (err != null) {
         if (err.code === 'ENOENT') {
           return callback(null)
@@ -262,7 +265,7 @@ module.exports = OutputCacheManager = {
       const dirs = results.sort().reverse()
       const currentTime = Date.now()
 
-      const isExpired = function(dir, index) {
+      const isExpired = function (dir, index) {
         if ((options != null ? options.keep : undefined) === dir) {
           return false
         }
@@ -280,7 +283,7 @@ module.exports = OutputCacheManager = {
         // we can get the build time from the first part of the directory name DDDD-RRRR
         // DDDD is date and RRRR is random bytes
         const dirTime = parseInt(
-          __guard__(dir.split('-'), x => x[0]),
+          __guard__(dir.split('-'), (x) => x[0]),
           16
         )
         const age = currentTime - dirTime
@@ -290,7 +293,7 @@ module.exports = OutputCacheManager = {
       const toRemove = _.filter(dirs, isExpired)
 
       const removeDir = (dir, cb) =>
-        fse.remove(Path.join(cacheRoot, dir), function(err, result) {
+        fse.remove(Path.join(cacheRoot, dir), function (err, result) {
           logger.log({ cache: cacheRoot, dir }, 'removed expired cache dir')
           if (err != null) {
             logger.error({ err, dir }, 'cache remove error')
@@ -312,9 +315,9 @@ module.exports = OutputCacheManager = {
   _checkFileIsSafe(src, callback) {
     // check if we have a valid file to copy into the cache
     if (callback == null) {
-      callback = function(error, isSafe) {}
+      callback = function (error, isSafe) {}
     }
-    return fs.stat(src, function(err, stats) {
+    return fs.stat(src, function (err, stats) {
       if ((err != null ? err.code : undefined) === 'ENOENT') {
         logger.warn(
           { err, file: src },
@@ -341,7 +344,7 @@ module.exports = OutputCacheManager = {
 
   _copyFile(src, dst, callback) {
     // copy output file into the cache
-    return fse.copy(src, dst, function(err) {
+    return fse.copy(src, dst, function (err) {
       if ((err != null ? err.code : undefined) === 'ENOENT') {
         logger.warn(
           { err, file: src },
@@ -368,7 +371,7 @@ module.exports = OutputCacheManager = {
 
   _checkIfShouldCopy(src, callback) {
     if (callback == null) {
-      callback = function(err, shouldCopy) {}
+      callback = function (err, shouldCopy) {}
     }
     return callback(null, !Path.basename(src).match(/^strace/))
   },
@@ -376,7 +379,7 @@ module.exports = OutputCacheManager = {
   _checkIfShouldArchive(src, callback) {
     let needle
     if (callback == null) {
-      callback = function(err, shouldCopy) {}
+      callback = function (err, shouldCopy) {}
     }
     if (Path.basename(src).match(/^strace/)) {
       return callback(null, true)

app/js/OutputFileFinder.js

@@ -24,14 +24,14 @@ const logger = require('logger-sharelatex')
 module.exports = OutputFileFinder = {
   findOutputFiles(resources, directory, callback) {
     if (callback == null) {
-      callback = function(error, outputFiles, allFiles) {}
+      callback = function (error, outputFiles, allFiles) {}
     }
     const incomingResources = {}
     for (const resource of Array.from(resources)) {
       incomingResources[resource.path] = true
     }
 
-    return OutputFileFinder._getAllFiles(directory, function(error, allFiles) {
+    return OutputFileFinder._getAllFiles(directory, function (error, allFiles) {
       if (allFiles == null) {
         allFiles = []
       }
@@ -44,7 +44,7 @@ module.exports = OutputFileFinder = {
         if (!incomingResources[file]) {
           outputFiles.push({
             path: file,
-            type: __guard__(file.match(/\.([^\.]+)$/), x => x[1])
+            type: __guard__(file.match(/\.([^\.]+)$/), (x) => x[1])
           })
         }
       }
@@ -54,11 +54,11 @@ module.exports = OutputFileFinder = {
 
   _getAllFiles(directory, _callback) {
     if (_callback == null) {
-      _callback = function(error, fileList) {}
+      _callback = function (error, fileList) {}
     }
-    const callback = function(error, fileList) {
+    const callback = function (error, fileList) {
       _callback(error, fileList)
-      return (_callback = function() {})
+      return (_callback = function () {})
     }
 
     // don't include clsi-specific files/directories in the output list
@@ -87,9 +87,9 @@ module.exports = OutputFileFinder = {
 
     const proc = spawn('find', args)
     let stdout = ''
-    proc.stdout.on('data', chunk => (stdout += chunk.toString()))
+    proc.stdout.setEncoding('utf8').on('data', (chunk) => (stdout += chunk))
     proc.on('error', callback)
-    return proc.on('close', function(code) {
+    return proc.on('close', function (code) {
       if (code !== 0) {
         logger.warn(
           { directory, code },
@@ -98,7 +98,7 @@ module.exports = OutputFileFinder = {
         return callback(null, [])
       }
       let fileList = stdout.trim().split('\n')
-      fileList = fileList.map(function(file) {
+      fileList = fileList.map(function (file) {
         // Strip leading directory
         let path
         return (path = Path.relative(directory, file))

app/js/OutputFileOptimiser.js

@@ -19,17 +19,17 @@ const Path = require('path')
 const { spawn } = require('child_process')
 const logger = require('logger-sharelatex')
 const Metrics = require('./Metrics')
-const _ = require('underscore')
+const _ = require('lodash')
 
 module.exports = OutputFileOptimiser = {
   optimiseFile(src, dst, callback) {
     // check output file (src) and see if we can optimise it, storing
     // the result in the build directory (dst)
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     if (src.match(/\/output\.pdf$/)) {
-      return OutputFileOptimiser.checkIfPDFIsOptimised(src, function(
+      return OutputFileOptimiser.checkIfPDFIsOptimised(src, function (
         err,
         isOptimised
       ) {
@@ -45,14 +45,13 @@ module.exports = OutputFileOptimiser = {
 
   checkIfPDFIsOptimised(file, callback) {
     const SIZE = 16 * 1024 // check the header of the pdf
-    const result = new Buffer(SIZE)
-    result.fill(0) // prevent leakage of uninitialised buffer
-    return fs.open(file, 'r', function(err, fd) {
+    const result = Buffer.alloc(SIZE) // fills with zeroes by default
+    return fs.open(file, 'r', function (err, fd) {
       if (err != null) {
         return callback(err)
       }
       return fs.read(fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) =>
-        fs.close(fd, function(errClose) {
+        fs.close(fd, function (errClose) {
           if (errRead != null) {
             return callback(errRead)
           }
@@ -69,7 +68,7 @@ module.exports = OutputFileOptimiser = {
 
   optimisePDF(src, dst, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const tmpOutput = dst + '.opt'
     const args = ['--linearize', src, tmpOutput]
@@ -78,19 +77,19 @@ module.exports = OutputFileOptimiser = {
     const timer = new Metrics.Timer('qpdf')
     const proc = spawn('qpdf', args)
     let stdout = ''
-    proc.stdout.on('data', chunk => (stdout += chunk.toString()))
+    proc.stdout.setEncoding('utf8').on('data', (chunk) => (stdout += chunk))
     callback = _.once(callback) // avoid double call back for error and close event
-    proc.on('error', function(err) {
+    proc.on('error', function (err) {
       logger.warn({ err, args }, 'qpdf failed')
       return callback(null)
     }) // ignore the error
-    return proc.on('close', function(code) {
+    return proc.on('close', function (code) {
       timer.done()
       if (code !== 0) {
         logger.warn({ code, args }, 'qpdf returned error')
         return callback(null) // ignore the error
       }
-      return fs.rename(tmpOutput, dst, function(err) {
+      return fs.rename(tmpOutput, dst, function (err) {
         if (err != null) {
           logger.warn(
             { tmpOutput, dst },

app/js/ProjectPersistenceManager.js

@@ -20,15 +20,37 @@ const async = require('async')
 const logger = require('logger-sharelatex')
 const oneDay = 24 * 60 * 60 * 1000
 const Settings = require('settings-sharelatex')
+const diskusage = require('diskusage')
 
 module.exports = ProjectPersistenceManager = {
   EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
 
+  refreshExpiryTimeout(callback) {
+    if (callback == null) {
+      callback = function (error) {}
+    }
+    diskusage.check('/', function (err, stats) {
+      if (err) {
+        logger.err({ err: err }, 'error getting disk usage')
+        return callback(err)
+      }
+      const lowDisk = stats.available / stats.total < 0.1
+      const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
+      if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
+        logger.warn(
+          { stats: stats },
+          'disk running low on space, modifying EXPIRY_TIMEOUT'
+        )
+        ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
+      }
+      callback()
+    })
+  },
   markProjectAsJustAccessed(project_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    const job = cb =>
+    const job = (cb) =>
       db.Project.findOrCreate({ where: { project_id } })
         .spread((project, created) =>
           project
@@ -42,9 +64,9 @@ module.exports = ProjectPersistenceManager = {
 
   clearExpiredProjects(callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return ProjectPersistenceManager._findExpiredProjectIds(function(
+    return ProjectPersistenceManager._findExpiredProjectIds(function (
       error,
       project_ids
     ) {
@@ -52,9 +74,9 @@ module.exports = ProjectPersistenceManager = {
         return callback(error)
       }
       logger.log({ project_ids }, 'clearing expired projects')
-      const jobs = Array.from(project_ids || []).map(project_id =>
-        (project_id => callback =>
-          ProjectPersistenceManager.clearProjectFromCache(project_id, function(
+      const jobs = Array.from(project_ids || []).map((project_id) =>
+        ((project_id) => (callback) =>
+          ProjectPersistenceManager.clearProjectFromCache(project_id, function (
             err
           ) {
             if (err != null) {
@@ -63,13 +85,13 @@ module.exports = ProjectPersistenceManager = {
             return callback()
           }))(project_id)
       )
-      return async.series(jobs, function(error) {
+      return async.series(jobs, function (error) {
         if (error != null) {
           return callback(error)
         }
         return CompileManager.clearExpiredProjects(
           ProjectPersistenceManager.EXPIRY_TIMEOUT,
-          error => callback()
+          (error) => callback()
         )
       })
     })
@@ -77,16 +99,16 @@ module.exports = ProjectPersistenceManager = {
 
   clearProject(project_id, user_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     logger.log({ project_id, user_id }, 'clearing project for user')
-    return CompileManager.clearProject(project_id, user_id, function(error) {
+    return CompileManager.clearProject(project_id, user_id, function (error) {
       if (error != null) {
         return callback(error)
       }
       return ProjectPersistenceManager.clearProjectFromCache(
         project_id,
-        function(error) {
+        function (error) {
           if (error != null) {
             return callback(error)
           }
@@ -98,17 +120,17 @@ module.exports = ProjectPersistenceManager = {
 
   clearProjectFromCache(project_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     logger.log({ project_id }, 'clearing project from cache')
-    return UrlCache.clearProject(project_id, function(error) {
+    return UrlCache.clearProject(project_id, function (error) {
       if (error != null) {
         logger.err({ error, project_id }, 'error clearing project from cache')
         return callback(error)
       }
       return ProjectPersistenceManager._clearProjectFromDatabase(
         project_id,
-        function(error) {
+        function (error) {
           if (error != null) {
             logger.err(
               { error, project_id },
@@ -123,10 +145,10 @@ module.exports = ProjectPersistenceManager = {
 
   _clearProjectFromDatabase(project_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     logger.log({ project_id }, 'clearing project from database')
-    const job = cb =>
+    const job = (cb) =>
       db.Project.destroy({ where: { project_id } })
         .then(() => cb())
         .error(cb)
@@ -135,19 +157,19 @@ module.exports = ProjectPersistenceManager = {
 
   _findExpiredProjectIds(callback) {
     if (callback == null) {
-      callback = function(error, project_ids) {}
+      callback = function (error, project_ids) {}
     }
-    const job = function(cb) {
+    const job = function (cb) {
       const keepProjectsFrom = new Date(
         Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT
       )
       const q = {}
       q[db.op.lt] = keepProjectsFrom
       return db.Project.findAll({ where: { lastAccessed: q } })
-        .then(projects =>
+        .then((projects) =>
           cb(
             null,
-            projects.map(project => project.project_id)
+            projects.map((project) => project.project_id)
           )
         )
         .error(cb)

app/js/RequestParser.js

@@ -27,7 +27,7 @@ module.exports = RequestParser = {
   parse(body, callback) {
     let resource
     if (callback == null) {
-      callback = function(error, data) {}
+      callback = function (error, data) {}
     }
     const response = {}
 
@@ -61,7 +61,13 @@ module.exports = RequestParser = {
       response.imageName = this._parseAttribute(
         'imageName',
         compile.options.imageName,
-        { type: 'string' }
+        {
+          type: 'string',
+          validValues:
+            settings.clsi &&
+            settings.clsi.docker &&
+            settings.clsi.docker.allowedImages
+        }
       )
       response.draft = this._parseAttribute('draft', compile.options.draft, {
         default: false,
@@ -74,7 +80,17 @@ module.exports = RequestParser = {
         default: [],
         type: 'object'
       })
-
+      if (settings.allowedCompileGroups) {
+        response.compileGroup = this._parseAttribute(
+          'compileGroup',
+          compile.options.compileGroup,
+          {
+            validValues: settings.allowedCompileGroups,
+            default: '',
+            type: 'string'
+          }
+        )
+      }
       // The syncType specifies whether the request contains all
       // resources (full) or only those resources to be updated
       // in-place (incremental).

app/js/ResourceStateManager.js

@@ -41,13 +41,13 @@ module.exports = ResourceStateManager = {
 
   saveProjectState(state, resources, basePath, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     const stateFile = Path.join(basePath, this.SYNC_STATE_FILE)
     if (state == null) {
       // remove the file if no state passed in
       logger.log({ state, basePath }, 'clearing sync state')
-      return fs.unlink(stateFile, function(err) {
+      return fs.unlink(stateFile, function (err) {
         if (err != null && err.code !== 'ENOENT') {
           return callback(err)
         } else {
@@ -56,7 +56,9 @@ module.exports = ResourceStateManager = {
       })
     } else {
       logger.log({ state, basePath }, 'writing sync state')
-      const resourceList = Array.from(resources).map(resource => resource.path)
+      const resourceList = Array.from(resources).map(
+        (resource) => resource.path
+      )
       return fs.writeFile(
         stateFile,
         [...Array.from(resourceList), `stateHash:${state}`].join('\n'),
@@ -67,11 +69,11 @@ module.exports = ResourceStateManager = {
 
   checkProjectStateMatches(state, basePath, callback) {
     if (callback == null) {
-      callback = function(error, resources) {}
+      callback = function (error, resources) {}
     }
     const stateFile = Path.join(basePath, this.SYNC_STATE_FILE)
     const size = this.SYNC_STATE_MAX_SIZE
-    return SafeReader.readFile(stateFile, size, 'utf8', function(
+    return SafeReader.readFile(stateFile, size, 'utf8', function (
       err,
       result,
       bytesRead
@@ -86,7 +88,7 @@ module.exports = ResourceStateManager = {
         )
       }
       const array =
-        __guard__(result != null ? result.toString() : undefined, x =>
+        __guard__(result != null ? result.toString() : undefined, (x) =>
           x.split('\n')
         ) || []
       const adjustedLength = Math.max(array.length, 1)
@@ -102,7 +104,7 @@ module.exports = ResourceStateManager = {
           new Errors.FilesOutOfSyncError('invalid state for incremental update')
         )
       } else {
-        const resources = Array.from(resourceList).map(path => ({ path }))
+        const resources = Array.from(resourceList).map((path) => ({ path }))
         return callback(null, resources)
       }
     })
@@ -112,11 +114,11 @@ module.exports = ResourceStateManager = {
     // check the paths are all relative to current directory
     let file
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     for (file of Array.from(resources || [])) {
       for (const dir of Array.from(
-        __guard__(file != null ? file.path : undefined, x => x.split('/'))
+        __guard__(file != null ? file.path : undefined, (x) => x.split('/'))
       )) {
         if (dir === '..') {
           return callback(new Error('relative path in resource file list'))
@@ -129,8 +131,8 @@ module.exports = ResourceStateManager = {
       seenFile[file] = true
     }
     const missingFiles = Array.from(resources)
-      .filter(resource => !seenFile[resource.path])
-      .map(resource => resource.path)
+      .filter((resource) => !seenFile[resource.path])
+      .map((resource) => resource.path)
     if ((missingFiles != null ? missingFiles.length : undefined) > 0) {
       logger.err(
         { missingFiles, basePath, allFiles, resources },

app/js/ResourceWriter.js

@@ -30,7 +30,7 @@ const parallelFileDownloads = settings.parallelFileDownloads || 1
 module.exports = ResourceWriter = {
   syncResourcesToDisk(request, basePath, callback) {
     if (callback == null) {
-      callback = function(error, resourceList) {}
+      callback = function (error, resourceList) {}
     }
     if (request.syncType === 'incremental') {
       logger.log(
@@ -40,14 +40,14 @@ module.exports = ResourceWriter = {
       return ResourceStateManager.checkProjectStateMatches(
         request.syncState,
         basePath,
-        function(error, resourceList) {
+        function (error, resourceList) {
           if (error != null) {
             return callback(error)
           }
           return ResourceWriter._removeExtraneousFiles(
             resourceList,
             basePath,
-            function(error, outputFiles, allFiles) {
+            function (error, outputFiles, allFiles) {
               if (error != null) {
                 return callback(error)
               }
@@ -55,7 +55,7 @@ module.exports = ResourceWriter = {
                 resourceList,
                 allFiles,
                 basePath,
-                function(error) {
+                function (error) {
                   if (error != null) {
                     return callback(error)
                   }
@@ -63,7 +63,7 @@ module.exports = ResourceWriter = {
                     request.project_id,
                     request.resources,
                     basePath,
-                    function(error) {
+                    function (error) {
                       if (error != null) {
                         return callback(error)
                       }
@@ -85,7 +85,7 @@ module.exports = ResourceWriter = {
         request.project_id,
         request.resources,
         basePath,
-        function(error) {
+        function (error) {
           if (error != null) {
             return callback(error)
           }
@@ -93,7 +93,7 @@ module.exports = ResourceWriter = {
             request.syncState,
             request.resources,
             basePath,
-            function(error) {
+            function (error) {
               if (error != null) {
                 return callback(error)
               }
@@ -107,15 +107,15 @@ module.exports = ResourceWriter = {
 
   saveIncrementalResourcesToDisk(project_id, resources, basePath, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return this._createDirectory(basePath, error => {
+    return this._createDirectory(basePath, (error) => {
       if (error != null) {
         return callback(error)
       }
-      const jobs = Array.from(resources).map(resource =>
-        (resource => {
-          return callback =>
+      const jobs = Array.from(resources).map((resource) =>
+        ((resource) => {
+          return (callback) =>
             this._writeResourceToDisk(project_id, resource, basePath, callback)
         })(resource)
       )
@@ -125,19 +125,19 @@ module.exports = ResourceWriter = {
 
   saveAllResourcesToDisk(project_id, resources, basePath, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return this._createDirectory(basePath, error => {
+    return this._createDirectory(basePath, (error) => {
       if (error != null) {
         return callback(error)
       }
-      return this._removeExtraneousFiles(resources, basePath, error => {
+      return this._removeExtraneousFiles(resources, basePath, (error) => {
         if (error != null) {
           return callback(error)
         }
-        const jobs = Array.from(resources).map(resource =>
-          (resource => {
-            return callback =>
+        const jobs = Array.from(resources).map((resource) =>
+          ((resource) => {
+            return (callback) =>
               this._writeResourceToDisk(
                 project_id,
                 resource,
@@ -153,9 +153,9 @@ module.exports = ResourceWriter = {
 
   _createDirectory(basePath, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return fs.mkdir(basePath, function(err) {
+    return fs.mkdir(basePath, function (err) {
       if (err != null) {
         if (err.code === 'EEXIST') {
           return callback()
@@ -171,15 +171,15 @@ module.exports = ResourceWriter = {
 
   _removeExtraneousFiles(resources, basePath, _callback) {
     if (_callback == null) {
-      _callback = function(error, outputFiles, allFiles) {}
+      _callback = function (error, outputFiles, allFiles) {}
     }
     const timer = new Metrics.Timer('unlink-output-files')
-    const callback = function(error, ...result) {
+    const callback = function (error, ...result) {
       timer.done()
       return _callback(error, ...Array.from(result))
     }
 
-    return OutputFileFinder.findOutputFiles(resources, basePath, function(
+    return OutputFileFinder.findOutputFiles(resources, basePath, function (
       error,
       outputFiles,
       allFiles
@@ -190,7 +190,7 @@ module.exports = ResourceWriter = {
 
       const jobs = []
       for (const file of Array.from(outputFiles || [])) {
-        ;(function(file) {
+        ;(function (file) {
           const { path } = file
           let should_delete = true
           if (
@@ -231,7 +231,9 @@ module.exports = ResourceWriter = {
             path === 'output.pdf' ||
             path === 'output.dvi' ||
             path === 'output.log' ||
-            path === 'output.xdv'
+            path === 'output.xdv' ||
+            path === 'output.stdout' ||
+            path === 'output.stderr'
           ) {
             should_delete = true
           }
@@ -240,7 +242,7 @@ module.exports = ResourceWriter = {
             should_delete = true
           }
           if (should_delete) {
-            return jobs.push(callback =>
+            return jobs.push((callback) =>
               ResourceWriter._deleteFileIfNotDirectory(
                 Path.join(basePath, path),
                 callback
@@ -250,7 +252,7 @@ module.exports = ResourceWriter = {
         })(file)
       }
 
-      return async.series(jobs, function(error) {
+      return async.series(jobs, function (error) {
         if (error != null) {
           return callback(error)
         }
@@ -261,9 +263,9 @@ module.exports = ResourceWriter = {
 
   _deleteFileIfNotDirectory(path, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return fs.stat(path, function(error, stat) {
+    return fs.stat(path, function (error, stat) {
       if (error != null && error.code === 'ENOENT') {
         return callback()
       } else if (error != null) {
@@ -273,7 +275,7 @@ module.exports = ResourceWriter = {
         )
         return callback(error)
       } else if (stat.isFile()) {
-        return fs.unlink(path, function(error) {
+        return fs.unlink(path, function (error) {
           if (error != null) {
             logger.err(
               { err: error, path },
@@ -292,16 +294,18 @@ module.exports = ResourceWriter = {
 
   _writeResourceToDisk(project_id, resource, basePath, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return ResourceWriter.checkPath(basePath, resource.path, function(
+    return ResourceWriter.checkPath(basePath, resource.path, function (
       error,
       path
     ) {
       if (error != null) {
         return callback(error)
       }
-      return fs.mkdir(Path.dirname(path), { recursive: true }, function(error) {
+      return fs.mkdir(Path.dirname(path), { recursive: true }, function (
+        error
+      ) {
         if (error != null) {
           return callback(error)
         }
@@ -312,7 +316,7 @@ module.exports = ResourceWriter = {
             resource.url,
             path,
             resource.modified,
-            function(err) {
+            function (err) {
               if (err != null) {
                 logger.err(
                   {

app/js/SafeReader.js

@@ -22,9 +22,9 @@ module.exports = SafeReader = {
 
   readFile(file, size, encoding, callback) {
     if (callback == null) {
-      callback = function(error, result) {}
+      callback = function (error, result) {}
     }
-    return fs.open(file, 'r', function(err, fd) {
+    return fs.open(file, 'r', function (err, fd) {
       if (err != null && err.code === 'ENOENT') {
         return callback()
       }
@@ -34,7 +34,7 @@ module.exports = SafeReader = {
 
       // safely return always closing the file
       const callbackWithClose = (err, ...result) =>
-        fs.close(fd, function(err1) {
+        fs.close(fd, function (err1) {
           if (err != null) {
             return callback(err)
           }
@@ -43,8 +43,8 @@ module.exports = SafeReader = {
           }
           return callback(null, ...Array.from(result))
         })
-      const buff = new Buffer(size, 0) // fill with zeros
-      return fs.read(fd, buff, 0, buff.length, 0, function(
+      const buff = Buffer.alloc(size) // fills with zeroes by default
+      return fs.read(fd, buff, 0, buff.length, 0, function (
         err,
         bytesRead,
         buffer

app/js/StaticServerForbidSymlinks.js

@@ -21,12 +21,12 @@ const Settings = require('settings-sharelatex')
 const logger = require('logger-sharelatex')
 const url = require('url')
 
-module.exports = ForbidSymlinks = function(staticFn, root, options) {
+module.exports = ForbidSymlinks = function (staticFn, root, options) {
   const expressStatic = staticFn(root, options)
   const basePath = Path.resolve(root)
-  return function(req, res, next) {
+  return function (req, res, next) {
     let file, project_id, result
-    const path = __guard__(url.parse(req.url), x => x.pathname)
+    const path = __guard__(url.parse(req.url), (x) => x.pathname)
     // check that the path is of the form /project_id_or_name/path/to/file.log
     if ((result = path.match(/^\/?([a-zA-Z0-9_-]+)\/(.*)/))) {
       project_id = result[1]
@@ -52,7 +52,7 @@ module.exports = ForbidSymlinks = function(staticFn, root, options) {
       return res.sendStatus(404)
     }
     // check that the requested path is not a symlink
-    return fs.realpath(requestedFsPath, function(err, realFsPath) {
+    return fs.realpath(requestedFsPath, function (err, realFsPath) {
       if (err != null) {
         if (err.code === 'ENOENT') {
           return res.sendStatus(404)

app/js/TikzManager.js

@@ -26,7 +26,7 @@ module.exports = TikzManager = {
   checkMainFile(compileDir, mainFile, resources, callback) {
     // if there's already an output.tex file, we don't want to touch it
     if (callback == null) {
-      callback = function(error, needsMainFile) {}
+      callback = function (error, needsMainFile) {}
     }
     for (const resource of Array.from(resources)) {
       if (resource.path === 'output.tex') {
@@ -35,14 +35,17 @@ module.exports = TikzManager = {
       }
     }
     // if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
-    return ResourceWriter.checkPath(compileDir, mainFile, function(
+    return ResourceWriter.checkPath(compileDir, mainFile, function (
       error,
       path
     ) {
       if (error != null) {
         return callback(error)
       }
-      return SafeReader.readFile(path, 65536, 'utf8', function(error, content) {
+      return SafeReader.readFile(path, 65536, 'utf8', function (
+        error,
+        content
+      ) {
         if (error != null) {
           return callback(error)
         }
@@ -64,16 +67,16 @@ module.exports = TikzManager = {
 
   injectOutputFile(compileDir, mainFile, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return ResourceWriter.checkPath(compileDir, mainFile, function(
+    return ResourceWriter.checkPath(compileDir, mainFile, function (
       error,
       path
     ) {
       if (error != null) {
         return callback(error)
       }
-      return fs.readFile(path, 'utf8', function(error, content) {
+      return fs.readFile(path, 'utf8', function (error, content) {
         if (error != null) {
           return callback(error)
         }

app/js/UrlCache.js

@@ -25,7 +25,7 @@ const async = require('async')
 module.exports = UrlCache = {
   downloadUrlToFile(project_id, url, destPath, lastModified, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     return UrlCache._ensureUrlIsInCache(
       project_id,
@@ -35,7 +35,7 @@ module.exports = UrlCache = {
         if (error != null) {
           return callback(error)
         }
-        return UrlCache._copyFile(pathToCachedUrl, destPath, function(error) {
+        return UrlCache._copyFile(pathToCachedUrl, destPath, function (error) {
           if (error != null) {
             return UrlCache._clearUrlDetails(project_id, url, () =>
               callback(error)
@@ -50,9 +50,9 @@ module.exports = UrlCache = {
 
   clearProject(project_id, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return UrlCache._findAllUrlsInProject(project_id, function(error, urls) {
+    return UrlCache._findAllUrlsInProject(project_id, function (error, urls) {
       logger.log(
         { project_id, url_count: urls.length },
         'clearing project URLs'
@@ -60,9 +60,9 @@ module.exports = UrlCache = {
       if (error != null) {
         return callback(error)
       }
-      const jobs = Array.from(urls || []).map(url =>
-        (url => callback =>
-          UrlCache._clearUrlFromCache(project_id, url, function(error) {
+      const jobs = Array.from(urls || []).map((url) =>
+        ((url) => (callback) =>
+          UrlCache._clearUrlFromCache(project_id, url, function (error) {
             if (error != null) {
               logger.error(
                 { err: error, project_id, url },
@@ -78,7 +78,7 @@ module.exports = UrlCache = {
 
   _ensureUrlIsInCache(project_id, url, lastModified, callback) {
     if (callback == null) {
-      callback = function(error, pathOnDisk) {}
+      callback = function (error, pathOnDisk) {}
     }
     if (lastModified != null) {
       // MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
@@ -95,10 +95,10 @@ module.exports = UrlCache = {
         }
         if (needsDownloading) {
           logger.log({ url, lastModified }, 'downloading URL')
-          return UrlFetcher.pipeUrlToFile(
+          return UrlFetcher.pipeUrlToFileWithRetry(
             url,
             UrlCache._cacheFilePathForUrl(project_id, url),
-            error => {
+            (error) => {
               if (error != null) {
                 return callback(error)
               }
@@ -106,7 +106,7 @@ module.exports = UrlCache = {
                 project_id,
                 url,
                 lastModified,
-                error => {
+                (error) => {
                   if (error != null) {
                     return callback(error)
                   }
@@ -128,12 +128,12 @@ module.exports = UrlCache = {
 
   _doesUrlNeedDownloading(project_id, url, lastModified, callback) {
     if (callback == null) {
-      callback = function(error, needsDownloading) {}
+      callback = function (error, needsDownloading) {}
     }
     if (lastModified == null) {
       return callback(null, true)
     }
-    return UrlCache._findUrlDetails(project_id, url, function(
+    return UrlCache._findUrlDetails(project_id, url, function (
       error,
       urlDetails
     ) {
@@ -153,14 +153,7 @@ module.exports = UrlCache = {
   },
 
   _cacheFileNameForUrl(project_id, url) {
-    return (
-      project_id +
-      ':' +
-      crypto
-        .createHash('md5')
-        .update(url)
-        .digest('hex')
-    )
+    return project_id + ':' + crypto.createHash('md5').update(url).digest('hex')
   },
 
   _cacheFilePathForUrl(project_id, url) {
@@ -172,14 +165,14 @@ module.exports = UrlCache = {
 
   _copyFile(from, to, _callback) {
     if (_callback == null) {
-      _callback = function(error) {}
+      _callback = function (error) {}
     }
-    const callbackOnce = function(error) {
+    const callbackOnce = function (error) {
       if (error != null) {
         logger.error({ err: error, from, to }, 'error copying file from cache')
       }
       _callback(error)
-      return (_callback = function() {})
+      return (_callback = function () {})
     }
     const writeStream = fs.createWriteStream(to)
     const readStream = fs.createReadStream(from)
@@ -191,13 +184,15 @@ module.exports = UrlCache = {
 
   _clearUrlFromCache(project_id, url, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return UrlCache._clearUrlDetails(project_id, url, function(error) {
+    return UrlCache._clearUrlDetails(project_id, url, function (error) {
       if (error != null) {
         return callback(error)
       }
-      return UrlCache._deleteUrlCacheFromDisk(project_id, url, function(error) {
+      return UrlCache._deleteUrlCacheFromDisk(project_id, url, function (
+        error
+      ) {
         if (error != null) {
           return callback(error)
         }
@@ -208,9 +203,9 @@ module.exports = UrlCache = {
 
   _deleteUrlCacheFromDisk(project_id, url, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function(
+    return fs.unlink(UrlCache._cacheFilePathForUrl(project_id, url), function (
       error
     ) {
       if (error != null && error.code !== 'ENOENT') {
@@ -224,20 +219,20 @@ module.exports = UrlCache = {
 
   _findUrlDetails(project_id, url, callback) {
     if (callback == null) {
-      callback = function(error, urlDetails) {}
+      callback = function (error, urlDetails) {}
     }
-    const job = cb =>
+    const job = (cb) =>
       db.UrlCache.findOne({ where: { url, project_id } })
-        .then(urlDetails => cb(null, urlDetails))
+        .then((urlDetails) => cb(null, urlDetails))
         .error(cb)
     return dbQueue.queue.push(job, callback)
   },
 
   _updateOrCreateUrlDetails(project_id, url, lastModified, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    const job = cb =>
+    const job = (cb) =>
       db.UrlCache.findOrCreate({ where: { url, project_id } })
         .spread((urlDetails, created) =>
           urlDetails
@@ -251,9 +246,9 @@ module.exports = UrlCache = {
 
   _clearUrlDetails(project_id, url, callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
-    const job = cb =>
+    const job = (cb) =>
       db.UrlCache.destroy({ where: { url, project_id } })
         .then(() => cb(null))
         .error(cb)
@@ -262,14 +257,14 @@ module.exports = UrlCache = {
 
   _findAllUrlsInProject(project_id, callback) {
     if (callback == null) {
-      callback = function(error, urls) {}
+      callback = function (error, urls) {}
     }
-    const job = cb =>
+    const job = (cb) =>
       db.UrlCache.findAll({ where: { project_id } })
-        .then(urlEntries =>
+        .then((urlEntries) =>
           cb(
             null,
-            urlEntries.map(entry => entry.url)
+            urlEntries.map((entry) => entry.url)
           )
         )
         .error(cb)

app/js/UrlFetcher.js

@@ -18,20 +18,28 @@ const fs = require('fs')
 const logger = require('logger-sharelatex')
 const settings = require('settings-sharelatex')
 const URL = require('url')
+const async = require('async')
 
 const oneMinute = 60 * 1000
 
 module.exports = UrlFetcher = {
+  pipeUrlToFileWithRetry(url, filePath, callback) {
+    const doDownload = function (cb) {
+      UrlFetcher.pipeUrlToFile(url, filePath, cb)
+    }
+    async.retry(3, doDownload, callback)
+  },
+
   pipeUrlToFile(url, filePath, _callback) {
     if (_callback == null) {
-      _callback = function(error) {}
+      _callback = function (error) {}
     }
-    const callbackOnce = function(error) {
+    const callbackOnce = function (error) {
       if (timeoutHandler != null) {
         clearTimeout(timeoutHandler)
       }
       _callback(error)
-      return (_callback = function() {})
+      return (_callback = function () {})
     }
 
     if (settings.filestoreDomainOveride != null) {
@@ -39,7 +47,7 @@ module.exports = UrlFetcher = {
       url = `${settings.filestoreDomainOveride}${p}`
     }
     var timeoutHandler = setTimeout(
-      function() {
+      function () {
         timeoutHandler = null
         logger.error({ url, filePath }, 'Timed out downloading file to cache')
         return callbackOnce(
@@ -55,7 +63,7 @@ module.exports = UrlFetcher = {
     urlStream.pause() // stop data flowing until we are ready
 
     // attach handlers before setting up pipes
-    urlStream.on('error', function(error) {
+    urlStream.on('error', function (error) {
       logger.error({ err: error, url, filePath }, 'error downloading url')
       return callbackOnce(
         error || new Error(`Something went wrong downloading the URL ${url}`)
@@ -66,17 +74,17 @@ module.exports = UrlFetcher = {
       logger.log({ url, filePath }, 'finished downloading file into cache')
     )
 
-    return urlStream.on('response', function(res) {
+    return urlStream.on('response', function (res) {
       if (res.statusCode >= 200 && res.statusCode < 300) {
         const fileStream = fs.createWriteStream(filePath)
 
         // attach handlers before setting up pipes
-        fileStream.on('error', function(error) {
+        fileStream.on('error', function (error) {
           logger.error(
             { err: error, url, filePath },
             'error writing file into cache'
           )
-          return fs.unlink(filePath, function(err) {
+          return fs.unlink(filePath, function (err) {
             if (err != null) {
               logger.err({ err, filePath }, 'error deleting file from cache')
             }
@@ -84,7 +92,7 @@ module.exports = UrlFetcher = {
           })
         })
 
-        fileStream.on('finish', function() {
+        fileStream.on('finish', function () {
           logger.log({ url, filePath }, 'finished writing file into cache')
           return callbackOnce()
         })

app/js/db.js

@@ -10,7 +10,7 @@
  */
 const Sequelize = require('sequelize')
 const Settings = require('settings-sharelatex')
-const _ = require('underscore')
+const _ = require('lodash')
 const logger = require('logger-sharelatex')
 
 const options = _.extend({ logging: false }, Settings.mysql.clsi)
@@ -62,6 +62,6 @@ module.exports = {
     return sequelize
       .sync()
       .then(() => logger.log('db sync complete'))
-      .catch(err => console.log(err, 'error syncing'))
+      .catch((err) => console.log(err, 'error syncing'))
   }
 }

buildscript.txt

@@ -1,11 +1,9 @@
 clsi
---acceptance-creds=None
 --data-dirs=cache,compiles,db
 --dependencies=
 --docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=TEXLIVE_IMAGE
---language=es
---node-version=10.19.0
+--node-version=10.22.1
 --public-repo=True
---script-version=2.1.0
+--script-version=3.3.3

config/settings.defaults.js

@@ -9,7 +9,7 @@ module.exports = {
       username: 'clsi',
       dialect: 'sqlite',
       storage:
-        process.env.SQLITE_PATH || Path.resolve(__dirname + '/../db/db.sqlite'),
+        process.env.SQLITE_PATH || Path.resolve(__dirname, '../db/db.sqlite'),
       pool: {
         max: 1,
         min: 1
@@ -26,10 +26,10 @@ module.exports = {
     parseInt(process.env.PROCESS_LIFE_SPAN_LIMIT_MS) || 60 * 60 * 24 * 1000 * 2,
 
   path: {
-    compilesDir: Path.resolve(__dirname + '/../compiles'),
-    clsiCacheDir: Path.resolve(__dirname + '/../cache'),
-    synctexBaseDir(project_id) {
-      return Path.join(this.compilesDir, project_id)
+    compilesDir: Path.resolve(__dirname, '../compiles'),
+    clsiCacheDir: Path.resolve(__dirname, '../cache'),
+    synctexBaseDir(projectId) {
+      return Path.join(this.compilesDir, projectId)
     }
   },
 
@@ -57,13 +57,25 @@ module.exports = {
   parallelSqlQueryLimit: process.env.FILESTORE_PARALLEL_SQL_QUERY_LIMIT || 1,
   filestoreDomainOveride: process.env.FILESTORE_DOMAIN_OVERRIDE,
   texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
+  texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
   sentry: {
     dsn: process.env.SENTRY_DSN
   }
 }
 
+if (process.env.ALLOWED_COMPILE_GROUPS) {
+  try {
+    module.exports.allowedCompileGroups = process.env.ALLOWED_COMPILE_GROUPS.split(
+      ' '
+    )
+  } catch (error) {
+    console.error(error, 'could not apply allowed compile group setting')
+    process.exit(1)
+  }
+}
+
 if (process.env.DOCKER_RUNNER) {
-  let seccomp_profile_path
+  let seccompProfilePath
   module.exports.clsi = {
     dockerRunner: process.env.DOCKER_RUNNER === 'true',
     docker: {
@@ -71,28 +83,61 @@ if (process.env.DOCKER_RUNNER) {
       image:
         process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1',
       env: {
-        HOME: process.env.TEXLIVE_HOME || '/tmp',
-        TMPDIR: process.env.TEXLIVE_TMPDIR || '/tmp'
+        HOME: '/tmp'
       },
       socketPath: '/var/run/docker.sock',
       user: process.env.TEXLIVE_IMAGE_USER || 'tex'
     },
+    optimiseInDocker: true,
     expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
     checkProjectsIntervalMs: 10 * 60 * 1000
   }
 
   try {
-    seccomp_profile_path = Path.resolve(
-      __dirname + '/../seccomp/clsi-profile.json'
+    // Override individual docker settings using path-based keys, e.g.:
+    // compileGroupDockerConfigs = {
+    //    priority: { 'HostConfig.CpuShares': 100 }
+    //    beta: { 'dotted.path.here', 'value'}
+    // }
+    const compileGroupConfig = JSON.parse(
+      process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}'
     )
-    module.exports.clsi.docker.seccomp_profile = JSON.stringify(
-      JSON.parse(require('fs').readFileSync(seccomp_profile_path))
+    // Automatically clean up wordcount and synctex containers
+    const defaultCompileGroupConfig = {
+      wordcount: { 'HostConfig.AutoRemove': true },
+      synctex: { 'HostConfig.AutoRemove': true }
+    }
+    module.exports.clsi.docker.compileGroupConfig = Object.assign(
+      defaultCompileGroupConfig,
+      compileGroupConfig
     )
   } catch (error) {
-    console.log(
-      error,
-      `could not load seccom profile from ${seccomp_profile_path}`
+    console.error(error, 'could not apply compile group docker configs')
+    process.exit(1)
+  }
+
+  try {
+    seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
+    module.exports.clsi.docker.seccomp_profile = JSON.stringify(
+      JSON.parse(require('fs').readFileSync(seccompProfilePath))
     )
+  } catch (error) {
+    console.error(
+      error,
+      `could not load seccomp profile from ${seccompProfilePath}`
+    )
+    process.exit(1)
+  }
+
+  if (process.env.ALLOWED_IMAGES) {
+    try {
+      module.exports.clsi.docker.allowedImages = process.env.ALLOWED_IMAGES.split(
+        ' '
+      )
+    } catch (error) {
+      console.error(error, 'could not apply allowed images setting')
+      process.exit(1)
+    }
   }
 
   module.exports.path.synctexBaseDir = () => '/compile'

docker-compose-config.yml

@@ -3,6 +3,7 @@ version: "2.3"
 services:
   dev:
     environment:
+      ALLOWED_IMAGES: "quay.io/sharelatex/texlive-full:2017.1"
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
       TEXLIVE_IMAGE_USER: "tex"
       SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
@@ -18,6 +19,7 @@ services:
       
   ci:
     environment:
+      ALLOWED_IMAGES: ${TEXLIVE_IMAGE}
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
       TEXLIVE_IMAGE_USER: "tex"
       SHARELATEX_CONFIG: /app/config/settings.defaults.coffee

docker-compose.ci.yml

@@ -10,6 +10,7 @@ services:
     command: npm run test:unit:_run
     environment:
       NODE_ENV: test
+      NODE_OPTIONS: "--unhandled-rejections=strict"
 
 
   test_acceptance:
@@ -25,6 +26,7 @@ services:
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: test
+      NODE_OPTIONS: "--unhandled-rejections=strict"
       TEXLIVE_IMAGE:
     command: npm run test:acceptance:_run
 

docker-compose.yml

@@ -15,7 +15,8 @@ services:
     environment:
       MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: test
-    command: npm run test:unit
+      NODE_OPTIONS: "--unhandled-rejections=strict"
+    command: npm run --silent test:unit
 
   test_acceptance:
     build:
@@ -35,5 +36,6 @@ services:
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ERROR
       NODE_ENV: test
-    command: npm run test:acceptance
+      NODE_OPTIONS: "--unhandled-rejections=strict"
+    command: npm run --silent test:acceptance
 

nodemon.json

@@ -8,7 +8,6 @@
   "execMap": {
     "js": "npm run start"
   },
-
   "watch": [
     "app/js/",
     "app.js",

package.json

@@ -13,7 +13,7 @@
     "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
     "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
     "nodemon": "nodemon --config nodemon.json",
-    "lint": "node_modules/.bin/eslint .",
+    "lint": "node_modules/.bin/eslint --max-warnings 0 .",
     "format": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --list-different",
     "format:fix": "node_modules/.bin/prettier-eslint $PWD'/**/*.js' --write"
   },
@@ -21,21 +21,22 @@
   "dependencies": {
     "async": "3.2.0",
     "body-parser": "^1.19.0",
+    "diskusage": "^1.1.3",
     "dockerode": "^3.1.0",
     "express": "^4.17.1",
     "fs-extra": "^8.1.0",
     "heapdump": "^0.3.15",
     "lockfile": "^1.0.4",
-    "logger-sharelatex": "^1.9.1",
+    "lodash": "^4.17.20",
+    "logger-sharelatex": "^2.2.0",
     "lynx": "0.2.0",
     "metrics-sharelatex": "^2.6.0",
     "mysql": "^2.18.1",
     "request": "^2.88.2",
     "sequelize": "^5.21.5",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0",
-    "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
     "sqlite3": "^4.1.1",
-    "underscore": "^1.9.2",
+    "underscore": "^1.11.0",
     "v8-profiler-node8": "^6.1.1",
     "wrench": "~1.5.9"
   },
@@ -59,7 +60,7 @@
     "eslint-plugin-react": "^7.19.0",
     "eslint-plugin-standard": "^4.0.1",
     "mocha": "^7.1.0",
-    "prettier": "^1.19.1",
+    "prettier": "^2.0.0",
     "prettier-eslint-cli": "^5.0.0",
     "sandboxed-module": "^2.0.3",
     "sinon": "~9.0.1",

test/acceptance/js/AllowedImageNames.js

@@ -0,0 +1,102 @@
+const Client = require('./helpers/Client')
+const ClsiApp = require('./helpers/ClsiApp')
+const { expect } = require('chai')
+
+describe('AllowedImageNames', function () {
+  beforeEach(function (done) {
+    this.project_id = Client.randomId()
+    this.request = {
+      options: {
+        imageName: undefined
+      },
+      resources: [
+        {
+          path: 'main.tex',
+          content: `\
+\\documentclass{article}
+\\begin{document}
+Hello world
+\\end{document}\
+`
+        }
+      ]
+    }
+    ClsiApp.ensureRunning(done)
+  })
+
+  describe('with a valid name', function () {
+    beforeEach(function (done) {
+      this.request.options.imageName = process.env.TEXLIVE_IMAGE
+
+      Client.compile(this.project_id, this.request, (error, res, body) => {
+        this.error = error
+        this.res = res
+        this.body = body
+        done(error)
+      })
+    })
+    it('should return success', function () {
+      expect(this.res.statusCode).to.equal(200)
+    })
+
+    it('should return a PDF', function () {
+      let pdf
+      try {
+        pdf = Client.getOutputFile(this.body, 'pdf')
+      } catch (e) {}
+      expect(pdf).to.exist
+    })
+  })
+
+  describe('with an invalid name', function () {
+    beforeEach(function (done) {
+      this.request.options.imageName = 'something/evil:1337'
+      Client.compile(this.project_id, this.request, (error, res, body) => {
+        this.error = error
+        this.res = res
+        this.body = body
+        done(error)
+      })
+    })
+    it('should return non success', function () {
+      expect(this.res.statusCode).to.not.equal(200)
+    })
+
+    it('should not return a PDF', function () {
+      let pdf
+      try {
+        pdf = Client.getOutputFile(this.body, 'pdf')
+      } catch (e) {}
+      expect(pdf).to.not.exist
+    })
+  })
+
+  describe('wordcount', function () {
+    beforeEach(function (done) {
+      Client.compile(this.project_id, this.request, done)
+    })
+    it('should error out with an invalid imageName', function () {
+      Client.wordcountWithImage(
+        this.project_id,
+        'main.tex',
+        'something/evil:1337',
+        (error, result) => {
+          expect(String(error)).to.include('statusCode=400')
+        }
+      )
+    })
+
+    it('should produce a texcout a valid imageName', function () {
+      Client.wordcountWithImage(
+        this.project_id,
+        'main.tex',
+        process.env.TEXLIVE_IMAGE,
+        (error, result) => {
+          expect(error).to.not.exist
+          expect(result).to.exist
+          expect(result.texcount).to.exist
+        }
+      )
+    })
+  })
+})

test/acceptance/js/BrokenLatexFileTests.js

@@ -13,8 +13,8 @@ const request = require('request')
 require('chai').should()
 const ClsiApp = require('./helpers/ClsiApp')
 
-describe('Broken LaTeX file', function() {
-  before(function(done) {
+describe('Broken LaTeX file', function () {
+  before(function (done) {
     this.broken_request = {
       resources: [
         {
@@ -44,8 +44,8 @@ Hello world
     return ClsiApp.ensureRunning(done)
   })
 
-  describe('on first run', function() {
-    before(function(done) {
+  describe('on first run', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       return Client.compile(
         this.project_id,
@@ -59,13 +59,13 @@ Hello world
       )
     })
 
-    return it('should return a failure status', function() {
+    return it('should return a failure status', function () {
       return this.body.compile.status.should.equal('failure')
     })
   })
 
-  return describe('on second run', function() {
-    before(function(done) {
+  return describe('on second run', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       return Client.compile(this.project_id, this.correct_request, () => {
         return Client.compile(
@@ -81,7 +81,7 @@ Hello world
       })
     })
 
-    return it('should return a failure status', function() {
+    return it('should return a failure status', function () {
       return this.body.compile.status.should.equal('failure')
     })
   })

test/acceptance/js/DeleteOldFilesTest.js

@@ -13,8 +13,8 @@ const request = require('request')
 require('chai').should()
 const ClsiApp = require('./helpers/ClsiApp')
 
-describe('Deleting Old Files', function() {
-  before(function(done) {
+describe('Deleting Old Files', function () {
+  before(function (done) {
     this.request = {
       resources: [
         {
@@ -31,8 +31,8 @@ Hello world
     return ClsiApp.ensureRunning(done)
   })
 
-  return describe('on first run', function() {
-    before(function(done) {
+  return describe('on first run', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       return Client.compile(
         this.project_id,
@@ -46,12 +46,12 @@ Hello world
       )
     })
 
-    it('should return a success status', function() {
+    it('should return a success status', function () {
       return this.body.compile.status.should.equal('success')
     })
 
-    return describe('after file has been deleted', function() {
-      before(function(done) {
+    return describe('after file has been deleted', function () {
+      before(function (done) {
         this.request.resources = []
         return Client.compile(
           this.project_id,
@@ -65,7 +65,7 @@ Hello world
         )
       })
 
-      return it('should return a failure status', function() {
+      return it('should return a failure status', function () {
         return this.body.compile.status.should.equal('failure')
       })
     })

test/acceptance/js/ExampleDocumentTests.js

@@ -24,7 +24,7 @@ const ChildProcess = require('child_process')
 const ClsiApp = require('./helpers/ClsiApp')
 const logger = require('logger-sharelatex')
 const Path = require('path')
-const fixturePath = path => {
+const fixturePath = (path) => {
   if (path.slice(0, 3) === 'tmp') {
     return '/tmp/clsi_acceptance_tests' + path.slice(3)
   }
@@ -41,23 +41,23 @@ console.log(
 
 const MOCHA_LATEX_TIMEOUT = 60 * 1000
 
-const convertToPng = function(pdfPath, pngPath, callback) {
+const convertToPng = function (pdfPath, pngPath, callback) {
   if (callback == null) {
-    callback = function(error) {}
+    callback = function (error) {}
   }
   const command = `convert ${fixturePath(pdfPath)} ${fixturePath(pngPath)}`
   console.log('COMMAND')
   console.log(command)
   const convert = ChildProcess.exec(command)
   const stdout = ''
-  convert.stdout.on('data', chunk => console.log('STDOUT', chunk.toString()))
-  convert.stderr.on('data', chunk => console.log('STDERR', chunk.toString()))
+  convert.stdout.on('data', (chunk) => console.log('STDOUT', chunk.toString()))
+  convert.stderr.on('data', (chunk) => console.log('STDERR', chunk.toString()))
   return convert.on('exit', () => callback())
 }
 
-const compare = function(originalPath, generatedPath, callback) {
+const compare = function (originalPath, generatedPath, callback) {
   if (callback == null) {
-    callback = function(error, same) {}
+    callback = function (error, same) {}
   }
   const diff_file = `${fixturePath(generatedPath)}-diff.png`
   const proc = ChildProcess.exec(
@@ -66,11 +66,11 @@ const compare = function(originalPath, generatedPath, callback) {
     )} ${diff_file}`
   )
   let stderr = ''
-  proc.stderr.on('data', chunk => (stderr += chunk))
+  proc.stderr.on('data', (chunk) => (stderr += chunk))
   return proc.on('exit', () => {
     if (stderr.trim() === '0 (0)') {
       // remove output diff if test matches expected image
-      fs.unlink(diff_file, err => {
+      fs.unlink(diff_file, (err) => {
         if (err) {
           throw err
         }
@@ -83,14 +83,14 @@ const compare = function(originalPath, generatedPath, callback) {
   })
 }
 
-const checkPdfInfo = function(pdfPath, callback) {
+const checkPdfInfo = function (pdfPath, callback) {
   if (callback == null) {
-    callback = function(error, output) {}
+    callback = function (error, output) {}
   }
   const proc = ChildProcess.exec(`pdfinfo ${fixturePath(pdfPath)}`)
   let stdout = ''
-  proc.stdout.on('data', chunk => (stdout += chunk))
-  proc.stderr.on('data', chunk => console.log('STDERR', chunk.toString()))
+  proc.stdout.on('data', (chunk) => (stdout += chunk))
+  proc.stderr.on('data', (chunk) => console.log('STDERR', chunk.toString()))
   return proc.on('exit', () => {
     if (stdout.match(/Optimized:\s+yes/)) {
       return callback(null, true)
@@ -100,11 +100,11 @@ const checkPdfInfo = function(pdfPath, callback) {
   })
 }
 
-const compareMultiplePages = function(project_id, callback) {
+const compareMultiplePages = function (project_id, callback) {
   if (callback == null) {
-    callback = function(error) {}
+    callback = function (error) {}
   }
-  var compareNext = function(page_no, callback) {
+  var compareNext = function (page_no, callback) {
     const path = `tmp/${project_id}-source-${page_no}.png`
     return fs.stat(fixturePath(path), (error, stat) => {
       if (error != null) {
@@ -127,23 +127,23 @@ const compareMultiplePages = function(project_id, callback) {
   return compareNext(0, callback)
 }
 
-const comparePdf = function(project_id, example_dir, callback) {
+const comparePdf = function (project_id, example_dir, callback) {
   if (callback == null) {
-    callback = function(error) {}
+    callback = function (error) {}
   }
   console.log('CONVERT')
   console.log(`tmp/${project_id}.pdf`, `tmp/${project_id}-generated.png`)
   return convertToPng(
     `tmp/${project_id}.pdf`,
     `tmp/${project_id}-generated.png`,
-    error => {
+    (error) => {
       if (error != null) {
         throw error
       }
       return convertToPng(
         `examples/${example_dir}/output.pdf`,
         `tmp/${project_id}-source.png`,
-        error => {
+        (error) => {
           if (error != null) {
             throw error
           }
@@ -163,7 +163,7 @@ const comparePdf = function(project_id, example_dir, callback) {
                   }
                 )
               } else {
-                return compareMultiplePages(project_id, error => {
+                return compareMultiplePages(project_id, (error) => {
                   if (error != null) {
                     throw error
                   }
@@ -178,9 +178,14 @@ const comparePdf = function(project_id, example_dir, callback) {
   )
 }
 
-const downloadAndComparePdf = function(project_id, example_dir, url, callback) {
+const downloadAndComparePdf = function (
+  project_id,
+  example_dir,
+  url,
+  callback
+) {
   if (callback == null) {
-    callback = function(error) {}
+    callback = function (error) {}
   }
   const writeStream = fs.createWriteStream(fixturePath(`tmp/${project_id}.pdf`))
   request.get(url).pipe(writeStream)
@@ -198,28 +203,29 @@ const downloadAndComparePdf = function(project_id, example_dir, url, callback) {
 
 Client.runServer(4242, fixturePath('examples'))
 
-describe('Example Documents', function() {
-  before(function(done) {
+describe('Example Documents', function () {
+  before(function (done) {
     ClsiApp.ensureRunning(done)
   })
-  before(function(done) {
+  before(function (done) {
     fsExtra.remove(fixturePath('tmp'), done)
   })
-  before(function(done) {
+  before(function (done) {
     fs.mkdir(fixturePath('tmp'), done)
   })
-  after(function(done) {
+  after(function (done) {
     fsExtra.remove(fixturePath('tmp'), done)
   })
 
-  return Array.from(fs.readdirSync(fixturePath('examples'))).map(example_dir =>
-    (example_dir =>
-      describe(example_dir, function() {
-        before(function() {
+  return Array.from(fs.readdirSync(fixturePath('examples'))).map(
+    (example_dir) =>
+      ((example_dir) =>
+        describe(example_dir, function () {
+          before(function () {
             return (this.project_id = Client.randomId() + '_' + example_dir)
           })
 
-        it('should generate the correct pdf', function(done) {
+          it('should generate the correct pdf', function (done) {
             this.timeout(MOCHA_LATEX_TIMEOUT)
             return Client.compileDirectory(
               this.project_id,
@@ -231,10 +237,16 @@ describe('Example Documents', function() {
                   error ||
                   __guard__(
                     body != null ? body.compile : undefined,
-                  x => x.status
+                    (x) => x.status
                   ) === 'failure'
                 ) {
-                console.log('DEBUG: error', error, 'body', JSON.stringify(body))
+                  console.log(
+                    'DEBUG: error',
+                    error,
+                    'body',
+                    JSON.stringify(body)
+                  )
+                  return done(new Error('Compile failed'))
                 }
                 const pdf = Client.getOutputFile(body, 'pdf')
                 return downloadAndComparePdf(
@@ -247,7 +259,7 @@ describe('Example Documents', function() {
             )
           })
 
-        return it('should generate the correct pdf on the second run as well', function(done) {
+          return it('should generate the correct pdf on the second run as well', function (done) {
             this.timeout(MOCHA_LATEX_TIMEOUT)
             return Client.compileDirectory(
               this.project_id,
@@ -259,10 +271,16 @@ describe('Example Documents', function() {
                   error ||
                   __guard__(
                     body != null ? body.compile : undefined,
-                  x => x.status
+                    (x) => x.status
                   ) === 'failure'
                 ) {
-                console.log('DEBUG: error', error, 'body', JSON.stringify(body))
+                  console.log(
+                    'DEBUG: error',
+                    error,
+                    'body',
+                    JSON.stringify(body)
+                  )
+                  return done(new Error('Compile failed'))
                 }
                 const pdf = Client.getOutputFile(body, 'pdf')
                 return downloadAndComparePdf(

test/acceptance/js/SimpleLatexFileTests.js

@@ -13,8 +13,8 @@ const request = require('request')
 require('chai').should()
 const ClsiApp = require('./helpers/ClsiApp')
 
-describe('Simple LaTeX file', function() {
-  before(function(done) {
+describe('Simple LaTeX file', function () {
+  before(function (done) {
     this.project_id = Client.randomId()
     this.request = {
       resources: [
@@ -43,17 +43,17 @@ Hello world
     })
   })
 
-  it('should return the PDF', function() {
+  it('should return the PDF', function () {
     const pdf = Client.getOutputFile(this.body, 'pdf')
     return pdf.type.should.equal('pdf')
   })
 
-  it('should return the log', function() {
+  it('should return the log', function () {
     const log = Client.getOutputFile(this.body, 'log')
     return log.type.should.equal('log')
   })
 
-  it('should provide the pdf for download', function(done) {
+  it('should provide the pdf for download', function (done) {
     const pdf = Client.getOutputFile(this.body, 'pdf')
     return request.get(pdf.url, (error, res, body) => {
       res.statusCode.should.equal(200)
@@ -61,7 +61,7 @@ Hello world
     })
   })
 
-  return it('should provide the log for download', function(done) {
+  return it('should provide the log for download', function (done) {
     const log = Client.getOutputFile(this.body, 'pdf')
     return request.get(log.url, (error, res, body) => {
       res.statusCode.should.equal(200)

test/acceptance/js/SynctexTests.js

@@ -16,8 +16,8 @@ const { expect } = require('chai')
 const ClsiApp = require('./helpers/ClsiApp')
 const crypto = require('crypto')
 
-describe('Syncing', function() {
-  before(function(done) {
+describe('Syncing', function () {
+  before(function (done) {
     const content = `\
 \\documentclass{article}
 \\begin{document}
@@ -47,8 +47,8 @@ Hello world
     })
   })
 
-  describe('from code to pdf', function() {
-    return it('should return the correct location', function(done) {
+  describe('from code to pdf', function () {
+    return it('should return the correct location', function (done) {
       return Client.syncFromCode(
         this.project_id,
         'main.tex',
@@ -69,8 +69,8 @@ Hello world
     })
   })
 
-  return describe('from pdf to code', function() {
-    return it('should return the correct location', function(done) {
+  describe('from pdf to code', function () {
+    return it('should return the correct location', function (done) {
       return Client.syncFromPdf(
         this.project_id,
         1,
@@ -88,4 +88,104 @@ Hello world
       )
     })
   })
+
+  describe('when the project directory is not available', function () {
+    before(function () {
+      this.other_project_id = Client.randomId()
+    })
+    describe('from code to pdf', function () {
+      it('should return a 404 response', function (done) {
+        return Client.syncFromCode(
+          this.other_project_id,
+          'main.tex',
+          3,
+          5,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+    describe('from pdf to code', function () {
+      it('should return a 404 response', function (done) {
+        return Client.syncFromPdf(
+          this.other_project_id,
+          1,
+          100,
+          200,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+  })
+
+  describe('when the synctex file is not available', function () {
+    before(function (done) {
+      this.broken_project_id = Client.randomId()
+      const content = 'this is not valid tex' // not a valid tex file
+      this.request = {
+        resources: [
+          {
+            path: 'main.tex',
+            content
+          }
+        ]
+      }
+      Client.compile(
+        this.broken_project_id,
+        this.request,
+        (error, res, body) => {
+          this.error = error
+          this.res = res
+          this.body = body
+          return done()
+        }
+      )
+    })
+
+    describe('from code to pdf', function () {
+      it('should return a 404 response', function (done) {
+        return Client.syncFromCode(
+          this.broken_project_id,
+          'main.tex',
+          3,
+          5,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+    describe('from pdf to code', function () {
+      it('should return a 404 response', function (done) {
+        return Client.syncFromPdf(
+          this.broken_project_id,
+          1,
+          100,
+          200,
+          (error, body) => {
+            if (error != null) {
+              throw error
+            }
+            expect(body).to.equal('Not Found')
+            return done()
+          }
+        )
+      })
+    })
+  })
 })

test/acceptance/js/TimeoutTests.js

@@ -13,8 +13,8 @@ const request = require('request')
 require('chai').should()
 const ClsiApp = require('./helpers/ClsiApp')
 
-describe('Timed out compile', function() {
-  before(function(done) {
+describe('Timed out compile', function () {
+  before(function (done) {
     this.request = {
       options: {
         timeout: 10
@@ -47,16 +47,16 @@ describe('Timed out compile', function() {
     })
   })
 
-  it('should return a timeout error', function() {
+  it('should return a timeout error', function () {
     return this.body.compile.error.should.equal('container timed out')
   })
 
-  it('should return a timedout status', function() {
+  it('should return a timedout status', function () {
     return this.body.compile.status.should.equal('timedout')
   })
 
-  return it('should return the log output file name', function() {
-    const outputFilePaths = this.body.compile.outputFiles.map(x => x.path)
+  return it('should return the log output file name', function () {
+    const outputFilePaths = this.body.compile.outputFiles.map((x) => x.path)
     return outputFilePaths.should.include('output.log')
   })
 })

test/acceptance/js/UrlCachingTests.js

@@ -11,7 +11,6 @@
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
 const Client = require('./helpers/Client')
-const request = require('request')
 require('chai').should()
 const sinon = require('sinon')
 const ClsiApp = require('./helpers/ClsiApp')
@@ -36,17 +35,15 @@ const Server = {
   getFile() {},
 
   randomId() {
-    return Math.random()
-      .toString(16)
-      .slice(2)
+    return Math.random().toString(16).slice(2)
   }
 }
 
 Server.run()
 
-describe('Url Caching', function() {
-  describe('Downloading an image for the first time', function() {
-    before(function(done) {
+describe('Url Caching', function () {
+  describe('Downloading an image for the first time', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       this.file = `${Server.randomId()}/lion.png`
       this.request = {
@@ -83,17 +80,17 @@ describe('Url Caching', function() {
       })
     })
 
-    afterEach(function() {
+    afterEach(function () {
       return Server.getFile.restore()
     })
 
-    return it('should download the image', function() {
+    return it('should download the image', function () {
       return Server.getFile.calledWith(`/${this.file}`).should.equal(true)
     })
   })
 
-  describe('When an image is in the cache and the last modified date is unchanged', function() {
-    before(function(done) {
+  describe('When an image is in the cache and the last modified date is unchanged', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       this.file = `${Server.randomId()}/lion.png`
       this.request = {
@@ -138,17 +135,17 @@ describe('Url Caching', function() {
       )
     })
 
-    after(function() {
+    after(function () {
       return Server.getFile.restore()
     })
 
-    return it('should not download the image again', function() {
+    return it('should not download the image again', function () {
       return Server.getFile.called.should.equal(false)
     })
   })
 
-  describe('When an image is in the cache and the last modified date is advanced', function() {
-    before(function(done) {
+  describe('When an image is in the cache and the last modified date is advanced', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       this.file = `${Server.randomId()}/lion.png`
       this.request = {
@@ -194,17 +191,17 @@ describe('Url Caching', function() {
       )
     })
 
-    afterEach(function() {
+    afterEach(function () {
       return Server.getFile.restore()
     })
 
-    return it('should download the image again', function() {
+    return it('should download the image again', function () {
       return Server.getFile.called.should.equal(true)
     })
   })
 
-  describe('When an image is in the cache and the last modified date is further in the past', function() {
-    before(function(done) {
+  describe('When an image is in the cache and the last modified date is further in the past', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       this.file = `${Server.randomId()}/lion.png`
       this.request = {
@@ -250,17 +247,17 @@ describe('Url Caching', function() {
       )
     })
 
-    afterEach(function() {
+    afterEach(function () {
       return Server.getFile.restore()
     })
 
-    return it('should not download the image again', function() {
+    return it('should not download the image again', function () {
       return Server.getFile.called.should.equal(false)
     })
   })
 
-  describe('When an image is in the cache and the last modified date is not specified', function() {
-    before(function(done) {
+  describe('When an image is in the cache and the last modified date is not specified', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       this.file = `${Server.randomId()}/lion.png`
       this.request = {
@@ -306,17 +303,17 @@ describe('Url Caching', function() {
       )
     })
 
-    afterEach(function() {
+    afterEach(function () {
       return Server.getFile.restore()
     })
 
-    return it('should download the image again', function() {
+    return it('should download the image again', function () {
       return Server.getFile.called.should.equal(true)
     })
   })
 
-  return describe('After clearing the cache', function() {
-    before(function(done) {
+  return describe('After clearing the cache', function () {
+    before(function (done) {
       this.project_id = Client.randomId()
       this.file = `${Server.randomId()}/lion.png`
       this.request = {
@@ -339,7 +336,7 @@ describe('Url Caching', function() {
         ]
       }
 
-      return Client.compile(this.project_id, this.request, error => {
+      return Client.compile(this.project_id, this.request, (error) => {
         if (error != null) {
           throw error
         }
@@ -362,11 +359,11 @@ describe('Url Caching', function() {
       })
     })
 
-    afterEach(function() {
+    afterEach(function () {
       return Server.getFile.restore()
     })
 
-    return it('should download the image again', function() {
+    return it('should download the image again', function () {
       return Server.getFile.called.should.equal(true)
     })
   })

test/acceptance/js/WordcountTests.js

@@ -17,8 +17,8 @@ const path = require('path')
 const fs = require('fs')
 const ClsiApp = require('./helpers/ClsiApp')
 
-describe('Syncing', function() {
-  before(function(done) {
+describe('Syncing', function () {
+  before(function (done) {
     this.request = {
       resources: [
         {
@@ -45,8 +45,8 @@ describe('Syncing', function() {
     })
   })
 
-  return describe('wordcount file', function() {
-    return it('should return wordcount info', function(done) {
+  return describe('wordcount file', function () {
+    return it('should return wordcount info', function (done) {
       return Client.wordcount(this.project_id, 'main.tex', (error, result) => {
         if (error != null) {
           throw error

test/acceptance/js/helpers/Client.js

@@ -23,14 +23,12 @@ module.exports = Client = {
   host: Settings.apis.clsi.url,
 
   randomId() {
-    return Math.random()
-      .toString(16)
-      .slice(2)
+    return Math.random().toString(16).slice(2)
   },
 
   compile(project_id, data, callback) {
     if (callback == null) {
-      callback = function(error, res, body) {}
+      callback = function (error, res, body) {}
     }
     return request.post(
       {
@@ -45,7 +43,7 @@ module.exports = Client = {
 
   clearCache(project_id, callback) {
     if (callback == null) {
-      callback = function(error, res, body) {}
+      callback = function (error, res, body) {}
     }
     return request.del(`${this.host}/project/${project_id}`, callback)
   },
@@ -64,7 +62,7 @@ module.exports = Client = {
     const app = express()
     app.use(express.static(directory))
     console.log('starting test server on', port, host)
-    return app.listen(port, host).on('error', error => {
+    return app.listen(port, host).on('error', (error) => {
       console.error('error starting server:', error.message)
       return process.exit(1)
     })
@@ -72,7 +70,7 @@ module.exports = Client = {
 
   syncFromCode(project_id, file, line, column, callback) {
     if (callback == null) {
-      callback = function(error, pdfPositions) {}
+      callback = function (error, pdfPositions) {}
     }
     return request.get(
       {
@@ -81,20 +79,21 @@ module.exports = Client = {
           file,
           line,
           column
-        }
+        },
+        json: true
       },
       (error, response, body) => {
         if (error != null) {
           return callback(error)
         }
-        return callback(null, JSON.parse(body))
+        return callback(null, body)
       }
     )
   },
 
   syncFromPdf(project_id, page, h, v, callback) {
     if (callback == null) {
-      callback = function(error, pdfPositions) {}
+      callback = function (error, pdfPositions) {}
     }
     return request.get(
       {
@@ -103,20 +102,21 @@ module.exports = Client = {
           page,
           h,
           v
-        }
+        },
+        json: true
       },
       (error, response, body) => {
         if (error != null) {
           return callback(error)
         }
-        return callback(null, JSON.parse(body))
+        return callback(null, body)
       }
     )
   },
 
   compileDirectory(project_id, baseDirectory, directory, serverPort, callback) {
     if (callback == null) {
-      callback = function(error, res, body) {}
+      callback = function (error, res, body) {}
     }
     const resources = []
     let entities = fs.readdirSync(`${baseDirectory}/${directory}`)
@@ -128,7 +128,7 @@ module.exports = Client = {
         entities = entities.concat(
           fs
             .readdirSync(`${baseDirectory}/${directory}/${entity}`)
-            .map(subEntity => {
+            .map((subEntity) => {
               if (subEntity === 'main.tex') {
                 rootResourcePath = `${entity}/${subEntity}`
               }
@@ -187,13 +187,19 @@ module.exports = Client = {
   },
 
   wordcount(project_id, file, callback) {
+    const image = undefined
+    Client.wordcountWithImage(project_id, file, image, callback)
+  },
+
+  wordcountWithImage(project_id, file, image, callback) {
     if (callback == null) {
-      callback = function(error, pdfPositions) {}
+      callback = function (error, pdfPositions) {}
     }
     return request.get(
       {
         url: `${this.host}/project/${project_id}/wordcount`,
         qs: {
+          image,
           file
         }
       },
@@ -201,6 +207,9 @@ module.exports = Client = {
         if (error != null) {
           return callback(error)
         }
+        if (response.statusCode !== 200) {
+          return callback(new Error(`statusCode=${response.statusCode}`))
+        }
         return callback(null, JSON.parse(body))
       }
     )

test/acceptance/js/helpers/ClsiApp.js

@@ -23,7 +23,7 @@ module.exports = {
   callbacks: [],
   ensureRunning(callback) {
     if (callback == null) {
-      callback = function(error) {}
+      callback = function (error) {}
     }
     if (this.running) {
       return callback()
@@ -35,10 +35,10 @@ module.exports = {
       return app.listen(
         __guard__(
           Settings.internal != null ? Settings.internal.clsi : undefined,
-          x => x.port
+          (x) => x.port
         ),
         'localhost',
-        error => {
+        (error) => {
           if (error != null) {
             throw error
           }

test/load/js/loadTest.js

@@ -13,11 +13,11 @@ const request = require('request')
 const Settings = require('settings-sharelatex')
 const async = require('async')
 const fs = require('fs')
-const _ = require('underscore')
+const _ = require('lodash')
 const concurentCompiles = 5
 const totalCompiles = 50
 
-const buildUrl = path =>
+const buildUrl = (path) =>
   `http://${Settings.internal.clsi.host}:${Settings.internal.clsi.port}/${path}`
 
 const mainTexContent = fs.readFileSync('./bulk.tex', 'utf-8')
@@ -25,12 +25,12 @@ const mainTexContent = fs.readFileSync('./bulk.tex', 'utf-8')
 const compileTimes = []
 let failedCount = 0
 
-const getAverageCompileTime = function() {
+const getAverageCompileTime = function () {
   const totalTime = _.reduce(compileTimes, (sum, time) => sum + time, 0)
   return totalTime / compileTimes.length
 }
 
-const makeRequest = function(compileNumber, callback) {
+const makeRequest = function (compileNumber, callback) {
   let bulkBodyCount = 7
   let bodyContent = ''
   while (--bulkBodyCount) {
@@ -74,12 +74,12 @@ ${bodyContent}
   )
 }
 
-const jobs = _.map(__range__(1, totalCompiles, true), i => cb =>
+const jobs = _.map(__range__(1, totalCompiles, true), (i) => (cb) =>
   makeRequest(i, cb)
 )
 
 const startTime = new Date()
-async.parallelLimit(jobs, concurentCompiles, err => {
+async.parallelLimit(jobs, concurentCompiles, (err) => {
   if (err != null) {
     console.error(err)
   }

test/smoke/js/SmokeTests.js

@@ -1,31 +1,40 @@
-/* eslint-disable
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS101: Remove unnecessary use of Array.from
- * DS102: Remove unnecessary code created because of implicit returns
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-const chai = require('chai')
-if (Object.prototype.should == null) {
-  chai.should()
-}
-const { expect } = chai
 const request = require('request')
 const Settings = require('settings-sharelatex')
 
-const buildUrl = path =>
+const buildUrl = (path) =>
   `http://${Settings.internal.clsi.host}:${Settings.internal.clsi.port}/${path}`
 
 const url = buildUrl(`project/smoketest-${process.pid}/compile`)
 
-describe('Running a compile', function() {
-  before(function(done) {
-    return request.post(
+module.exports = {
+  sendNewResult(res) {
+    this._run((error) => this._sendResponse(res, error))
+  },
+  sendLastResult(res) {
+    this._sendResponse(res, this._lastError)
+  },
+  triggerRun(cb) {
+    this._run((error) => {
+      this._lastError = error
+      cb(error)
+    })
+  },
+
+  _lastError: new Error('SmokeTestsPending'),
+  _sendResponse(res, error) {
+    let code, body
+    if (error) {
+      code = 500
+      body = error.message
+    } else {
+      code = 200
+      body = 'OK'
+    }
+    res.contentType('text/plain')
+    res.status(code).send(body)
+  },
+  _run(done) {
+    request.post(
       {
         url,
         json: {
@@ -72,29 +81,22 @@ describe('Running a compile', function() {
         }
       },
       (error, response, body) => {
-        this.error = error
-        this.response = response
-        this.body = body
-        return done()
+        if (error) return done(error)
+        if (!body || !body.compile || !body.compile.outputFiles) {
+          return done(new Error('response payload incomplete'))
+        }
+
+        let pdfFound = false
+        let logFound = false
+        for (const file of body.compile.outputFiles) {
+          if (file.type === 'pdf') pdfFound = true
+          if (file.type === 'log') logFound = true
+        }
+
+        if (!pdfFound) return done(new Error('no pdf returned'))
+        if (!logFound) return done(new Error('no log returned'))
+        done()
       }
     )
-  })
-
-  it('should return the pdf', function() {
-    for (const file of Array.from(this.body.compile.outputFiles)) {
-      if (file.type === 'pdf') {
-        return
   }
-    }
-    throw new Error('no pdf returned')
-  })
-
-  return it('should return the log', function() {
-    for (const file of Array.from(this.body.compile.outputFiles)) {
-      if (file.type === 'log') {
-        return
-      }
-    }
-    throw new Error('no log returned')
-  })
-})
+}

test/unit/js/CompileControllerTests.js

@@ -12,14 +12,15 @@
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 require('chai').should()
+const { expect } = require('chai')
 const modulePath = require('path').join(
   __dirname,
   '../../../app/js/CompileController'
 )
 const tk = require('timekeeper')
 
-describe('CompileController', function() {
-  beforeEach(function() {
+describe('CompileController', function () {
+  beforeEach(function () {
     this.CompileController = SandboxedModule.require(modulePath, {
       requires: {
         './CompileManager': (this.CompileManager = {}),
@@ -46,8 +47,8 @@ describe('CompileController', function() {
     return (this.next = sinon.stub())
   })
 
-  describe('compile', function() {
-    beforeEach(function() {
+  describe('compile', function () {
+    beforeEach(function () {
       this.req.body = {
         compile: 'mock-body'
       }
@@ -81,40 +82,40 @@ describe('CompileController', function() {
       return (this.res.send = sinon.stub())
     })
 
-    describe('successfully', function() {
-      beforeEach(function() {
+    describe('successfully', function () {
+      beforeEach(function () {
         this.CompileManager.doCompileWithLock = sinon
           .stub()
           .callsArgWith(1, null, this.output_files)
         return this.CompileController.compile(this.req, this.res)
       })
 
-      it('should parse the request', function() {
+      it('should parse the request', function () {
         return this.RequestParser.parse
           .calledWith(this.req.body)
           .should.equal(true)
       })
 
-      it('should run the compile for the specified project', function() {
+      it('should run the compile for the specified project', function () {
         return this.CompileManager.doCompileWithLock
           .calledWith(this.request_with_project_id)
           .should.equal(true)
       })
 
-      it('should mark the project as accessed', function() {
+      it('should mark the project as accessed', function () {
         return this.ProjectPersistenceManager.markProjectAsJustAccessed
           .calledWith(this.project_id)
           .should.equal(true)
       })
 
-      return it('should return the JSON response', function() {
+      return it('should return the JSON response', function () {
         this.res.status.calledWith(200).should.equal(true)
         return this.res.send
           .calledWith({
             compile: {
               status: 'success',
               error: null,
-              outputFiles: this.output_files.map(file => {
+              outputFiles: this.output_files.map((file) => {
                 return {
                   url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
                   path: file.path,
@@ -128,15 +129,15 @@ describe('CompileController', function() {
       })
     })
 
-    describe('with an error', function() {
-      beforeEach(function() {
+    describe('with an error', function () {
+      beforeEach(function () {
         this.CompileManager.doCompileWithLock = sinon
           .stub()
           .callsArgWith(1, new Error((this.message = 'error message')), null)
         return this.CompileController.compile(this.req, this.res)
       })
 
-      return it('should return the JSON response with the error', function() {
+      return it('should return the JSON response with the error', function () {
         this.res.status.calledWith(500).should.equal(true)
         return this.res.send
           .calledWith({
@@ -150,8 +151,8 @@ describe('CompileController', function() {
       })
     })
 
-    describe('when the request times out', function() {
-      beforeEach(function() {
+    describe('when the request times out', function () {
+      beforeEach(function () {
         this.error = new Error((this.message = 'container timed out'))
         this.error.timedout = true
         this.CompileManager.doCompileWithLock = sinon
@@ -160,7 +161,7 @@ describe('CompileController', function() {
         return this.CompileController.compile(this.req, this.res)
       })
 
-      return it('should return the JSON response with the timeout status', function() {
+      return it('should return the JSON response with the timeout status', function () {
         this.res.status.calledWith(200).should.equal(true)
         return this.res.send
           .calledWith({
@@ -174,15 +175,15 @@ describe('CompileController', function() {
       })
     })
 
-    return describe('when the request returns no output files', function() {
-      beforeEach(function() {
+    return describe('when the request returns no output files', function () {
+      beforeEach(function () {
         this.CompileManager.doCompileWithLock = sinon
           .stub()
           .callsArgWith(1, null, [])
         return this.CompileController.compile(this.req, this.res)
       })
 
-      return it('should return the JSON response with the failure status', function() {
+      return it('should return the JSON response with the failure status', function () {
         this.res.status.calledWith(200).should.equal(true)
         return this.res.send
           .calledWith({
@@ -197,8 +198,8 @@ describe('CompileController', function() {
     })
   })
 
-  describe('syncFromCode', function() {
-    beforeEach(function() {
+  describe('syncFromCode', function () {
+    beforeEach(function () {
       this.file = 'main.tex'
       this.line = 42
       this.column = 5
@@ -217,7 +218,7 @@ describe('CompileController', function() {
       return this.CompileController.syncFromCode(this.req, this.res, this.next)
     })
 
-    it('should find the corresponding location in the PDF', function() {
+    it('should find the corresponding location in the PDF', function () {
       return this.CompileManager.syncFromCode
         .calledWith(
           this.project_id,
@@ -229,7 +230,7 @@ describe('CompileController', function() {
         .should.equal(true)
     })
 
-    return it('should return the positions', function() {
+    return it('should return the positions', function () {
       return this.res.json
         .calledWith({
           pdf: this.pdfPositions
@@ -238,8 +239,8 @@ describe('CompileController', function() {
     })
   })
 
-  describe('syncFromPdf', function() {
-    beforeEach(function() {
+  describe('syncFromPdf', function () {
+    beforeEach(function () {
       this.page = 5
       this.h = 100.23
       this.v = 45.67
@@ -258,13 +259,13 @@ describe('CompileController', function() {
       return this.CompileController.syncFromPdf(this.req, this.res, this.next)
     })
 
-    it('should find the corresponding location in the code', function() {
+    it('should find the corresponding location in the code', function () {
       return this.CompileManager.syncFromPdf
         .calledWith(this.project_id, undefined, this.page, this.h, this.v)
         .should.equal(true)
     })
 
-    return it('should return the positions', function() {
+    return it('should return the positions', function () {
       return this.res.json
         .calledWith({
           code: this.codePositions
@@ -273,8 +274,8 @@ describe('CompileController', function() {
     })
   })
 
-  return describe('wordcount', function() {
-    beforeEach(function() {
+  return describe('wordcount', function () {
+    beforeEach(function () {
       this.file = 'main.tex'
       this.project_id = 'mock-project-id'
       this.req.params = { project_id: this.project_id }
@@ -287,21 +288,60 @@ describe('CompileController', function() {
       this.CompileManager.wordcount = sinon
         .stub()
         .callsArgWith(4, null, (this.texcount = ['mock-texcount']))
-      return this.CompileController.wordcount(this.req, this.res, this.next)
     })
 
-    it('should return the word count of a file', function() {
+    it('should return the word count of a file', function () {
+      this.CompileController.wordcount(this.req, this.res, this.next)
       return this.CompileManager.wordcount
         .calledWith(this.project_id, undefined, this.file, this.image)
         .should.equal(true)
     })
 
-    return it('should return the texcount info', function() {
+    it('should return the texcount info', function () {
+      this.CompileController.wordcount(this.req, this.res, this.next)
       return this.res.json
         .calledWith({
           texcount: this.texcount
         })
         .should.equal(true)
     })
+
+    describe('when allowedImages is set', function () {
+      beforeEach(function () {
+        this.Settings.clsi = { docker: {} }
+        this.Settings.clsi.docker.allowedImages = [
+          'repo/image:tag1',
+          'repo/image:tag2'
+        ]
+        this.res.send = sinon.stub()
+        this.res.status = sinon.stub().returns({ send: this.res.send })
+      })
+
+      describe('with an invalid image', function () {
+        beforeEach(function () {
+          this.req.query.image = 'something/evil:1337'
+          this.CompileController.wordcount(this.req, this.res, this.next)
+        })
+        it('should return a 400', function () {
+          expect(this.res.status.calledWith(400)).to.equal(true)
+        })
+        it('should not run the query', function () {
+          expect(this.CompileManager.wordcount.called).to.equal(false)
+        })
+      })
+
+      describe('with a valid image', function () {
+        beforeEach(function () {
+          this.req.query.image = 'repo/image:tag1'
+          this.CompileController.wordcount(this.req, this.res, this.next)
+        })
+        it('should not return a 400', function () {
+          expect(this.res.status.calledWith(400)).to.equal(false)
+        })
+        it('should run the query', function () {
+          expect(this.CompileManager.wordcount.called).to.equal(true)
+        })
+      })
+    })
   })
 })

test/unit/js/CompileManagerTests.js

@@ -24,8 +24,8 @@ const tk = require('timekeeper')
 const { EventEmitter } = require('events')
 const Path = require('path')
 
-describe('CompileManager', function() {
-  beforeEach(function() {
+describe('CompileManager', function () {
+  beforeEach(function () {
     this.CompileManager = SandboxedModule.require(modulePath, {
       requires: {
         './LatexRunner': (this.LatexRunner = {}),
@@ -60,8 +60,8 @@ describe('CompileManager', function() {
     this.project_id = 'project-id-123'
     return (this.user_id = '1234')
   })
-  describe('doCompileWithLock', function() {
-    beforeEach(function() {
+  describe('doCompileWithLock', function () {
+    beforeEach(function () {
       this.request = {
         resources: (this.resources = 'mock-resources'),
         project_id: this.project_id,
@@ -77,33 +77,33 @@ describe('CompileManager', function() {
         runner((err, ...result) => callback(err, ...Array.from(result))))
     })
 
-    describe('when the project is not locked', function() {
-      beforeEach(function() {
+    describe('when the project is not locked', function () {
+      beforeEach(function () {
         return this.CompileManager.doCompileWithLock(
           this.request,
           this.callback
         )
       })
 
-      it('should ensure that the compile directory exists', function() {
+      it('should ensure that the compile directory exists', function () {
         return this.fse.ensureDir.calledWith(this.compileDir).should.equal(true)
       })
 
-      it('should call doCompile with the request', function() {
+      it('should call doCompile with the request', function () {
         return this.CompileManager.doCompile
           .calledWith(this.request)
           .should.equal(true)
       })
 
-      return it('should call the callback with the output files', function() {
+      return it('should call the callback with the output files', function () {
         return this.callback
           .calledWithExactly(null, this.output_files)
           .should.equal(true)
       })
     })
 
-    return describe('when the project is locked', function() {
-      beforeEach(function() {
+    return describe('when the project is locked', function () {
+      beforeEach(function () {
         this.error = new Error('locked')
         this.LockManager.runWithLock = (lockFile, runner, callback) => {
           return callback(this.error)
@@ -114,22 +114,22 @@ describe('CompileManager', function() {
         )
       })
 
-      it('should ensure that the compile directory exists', function() {
+      it('should ensure that the compile directory exists', function () {
         return this.fse.ensureDir.calledWith(this.compileDir).should.equal(true)
       })
 
-      it('should not call doCompile with the request', function() {
+      it('should not call doCompile with the request', function () {
         return this.CompileManager.doCompile.called.should.equal(false)
       })
 
-      return it('should call the callback with the error', function() {
+      return it('should call the callback with the error', function () {
         return this.callback.calledWithExactly(this.error).should.equal(true)
       })
     })
   })
 
-  describe('doCompile', function() {
-    beforeEach(function() {
+  describe('doCompile', function () {
+    beforeEach(function () {
       this.output_files = [
         {
           path: 'output.log',
@@ -160,7 +160,8 @@ describe('CompileManager', function() {
         compiler: (this.compiler = 'pdflatex'),
         timeout: (this.timeout = 42000),
         imageName: (this.image = 'example.com/image'),
-        flags: (this.flags = ['-file-line-error'])
+        flags: (this.flags = ['-file-line-error']),
+        compileGroup: (this.compileGroup = 'compile-group')
       }
       this.env = {}
       this.Settings.compileDir = 'compiles'
@@ -179,18 +180,18 @@ describe('CompileManager', function() {
       return (this.TikzManager.checkMainFile = sinon.stub().callsArg(3, false))
     })
 
-    describe('normally', function() {
-      beforeEach(function() {
+    describe('normally', function () {
+      beforeEach(function () {
         return this.CompileManager.doCompile(this.request, this.callback)
       })
 
-      it('should write the resources to disk', function() {
+      it('should write the resources to disk', function () {
         return this.ResourceWriter.syncResourcesToDisk
           .calledWith(this.request, this.compileDir)
           .should.equal(true)
       })
 
-      it('should run LaTeX', function() {
+      it('should run LaTeX', function () {
         return this.LatexRunner.runLatex
           .calledWith(`${this.project_id}-${this.user_id}`, {
             directory: this.compileDir,
@@ -199,48 +200,49 @@ describe('CompileManager', function() {
             timeout: this.timeout,
             image: this.image,
             flags: this.flags,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
 
-      it('should find the output files', function() {
+      it('should find the output files', function () {
         return this.OutputFileFinder.findOutputFiles
           .calledWith(this.resources, this.compileDir)
           .should.equal(true)
       })
 
-      it('should return the output files', function() {
+      it('should return the output files', function () {
         return this.callback
           .calledWith(null, this.build_files)
           .should.equal(true)
       })
 
-      return it('should not inject draft mode by default', function() {
+      return it('should not inject draft mode by default', function () {
         return this.DraftModeManager.injectDraftMode.called.should.equal(false)
       })
     })
 
-    describe('with draft mode', function() {
-      beforeEach(function() {
+    describe('with draft mode', function () {
+      beforeEach(function () {
         this.request.draft = true
         return this.CompileManager.doCompile(this.request, this.callback)
       })
 
-      return it('should inject the draft mode header', function() {
+      return it('should inject the draft mode header', function () {
         return this.DraftModeManager.injectDraftMode
           .calledWith(this.compileDir + '/' + this.rootResourcePath)
           .should.equal(true)
       })
     })
 
-    describe('with a check option', function() {
-      beforeEach(function() {
+    describe('with a check option', function () {
+      beforeEach(function () {
         this.request.check = 'error'
         return this.CompileManager.doCompile(this.request, this.callback)
       })
 
-      return it('should run chktex', function() {
+      return it('should run chktex', function () {
         return this.LatexRunner.runLatex
           .calledWith(`${this.project_id}-${this.user_id}`, {
             directory: this.compileDir,
@@ -253,20 +255,21 @@ describe('CompileManager', function() {
               CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16',
               CHKTEX_EXIT_ON_ERROR: 1,
               CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000'
-            }
+            },
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
     })
 
-    return describe('with a knitr file and check options', function() {
-      beforeEach(function() {
+    return describe('with a knitr file and check options', function () {
+      beforeEach(function () {
         this.request.rootResourcePath = 'main.Rtex'
         this.request.check = 'error'
         return this.CompileManager.doCompile(this.request, this.callback)
       })
 
-      return it('should not run chktex', function() {
+      return it('should not run chktex', function () {
         return this.LatexRunner.runLatex
           .calledWith(`${this.project_id}-${this.user_id}`, {
             directory: this.compileDir,
@@ -275,16 +278,17 @@ describe('CompileManager', function() {
             timeout: this.timeout,
             image: this.image,
             flags: this.flags,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
     })
   })
 
-  describe('clearProject', function() {
-    describe('succesfully', function() {
-      beforeEach(function() {
+  describe('clearProject', function () {
+    describe('succesfully', function () {
+      beforeEach(function () {
         this.Settings.compileDir = 'compiles'
         this.fs.lstat = sinon.stub().callsArgWith(1, null, {
           isDirectory() {
@@ -294,6 +298,7 @@ describe('CompileManager', function() {
         this.proc = new EventEmitter()
         this.proc.stdout = new EventEmitter()
         this.proc.stderr = new EventEmitter()
+        this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr)
         this.child_process.spawn = sinon.stub().returns(this.proc)
         this.CompileManager.clearProject(
           this.project_id,
@@ -303,7 +308,7 @@ describe('CompileManager', function() {
         return this.proc.emit('close', 0)
       })
 
-      it('should remove the project directory', function() {
+      it('should remove the project directory', function () {
         return this.child_process.spawn
           .calledWith('rm', [
             '-r',
@@ -312,13 +317,13 @@ describe('CompileManager', function() {
           .should.equal(true)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.called.should.equal(true)
       })
     })
 
-    return describe('with a non-success status code', function() {
-      beforeEach(function() {
+    return describe('with a non-success status code', function () {
+      beforeEach(function () {
         this.Settings.compileDir = 'compiles'
         this.fs.lstat = sinon.stub().callsArgWith(1, null, {
           isDirectory() {
@@ -328,6 +333,7 @@ describe('CompileManager', function() {
         this.proc = new EventEmitter()
         this.proc.stdout = new EventEmitter()
         this.proc.stderr = new EventEmitter()
+        this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr)
         this.child_process.spawn = sinon.stub().returns(this.proc)
         this.CompileManager.clearProject(
           this.project_id,
@@ -338,7 +344,7 @@ describe('CompileManager', function() {
         return this.proc.emit('close', 1)
       })
 
-      it('should remove the project directory', function() {
+      it('should remove the project directory', function () {
         return this.child_process.spawn
           .calledWith('rm', [
             '-r',
@@ -347,7 +353,7 @@ describe('CompileManager', function() {
           .should.equal(true)
       })
 
-      it('should call the callback with an error from the stderr', function() {
+      it('should call the callback with an error from the stderr', function () {
         this.callback.calledWithExactly(sinon.match(Error)).should.equal(true)
 
         this.callback.args[0][0].message.should.equal(
@@ -357,8 +363,8 @@ describe('CompileManager', function() {
     })
   })
 
-  describe('syncing', function() {
-    beforeEach(function() {
+  describe('syncing', function () {
+    beforeEach(function () {
       this.page = 1
       this.h = 42.23
       this.v = 87.56
@@ -368,12 +374,12 @@ describe('CompileManager', function() {
       this.column = 3
       this.file_name = 'main.tex'
       this.child_process.execFile = sinon.stub()
-      return (this.Settings.path.synctexBaseDir = project_id =>
+      return (this.Settings.path.synctexBaseDir = (project_id) =>
         `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`)
     })
 
-    describe('syncFromCode', function() {
-      beforeEach(function() {
+    describe('syncFromCode', function () {
+      beforeEach(function () {
         this.fs.stat = sinon.stub().callsArgWith(1, null, {
           isFile() {
             return true
@@ -382,7 +388,7 @@ describe('CompileManager', function() {
         this.stdout = `NODE\t${this.page}\t${this.h}\t${this.v}\t${this.width}\t${this.height}\n`
         this.CommandRunner.run = sinon
           .stub()
-          .callsArgWith(6, null, { stdout: this.stdout })
+          .callsArgWith(7, null, { stdout: this.stdout })
         return this.CompileManager.syncFromCode(
           this.project_id,
           this.user_id,
@@ -393,7 +399,7 @@ describe('CompileManager', function() {
         )
       })
 
-      it('should execute the synctex binary', function() {
+      it('should execute the synctex binary', function () {
         const bin_path = Path.resolve(__dirname + '/../../../bin/synctex')
         const synctex_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/output.pdf`
         const file_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}`
@@ -416,7 +422,7 @@ describe('CompileManager', function() {
           .should.equal(true)
       })
 
-      return it('should call the callback with the parsed output', function() {
+      return it('should call the callback with the parsed output', function () {
         return this.callback
           .calledWith(null, [
             {
@@ -431,8 +437,8 @@ describe('CompileManager', function() {
       })
     })
 
-    return describe('syncFromPdf', function() {
-      beforeEach(function() {
+    return describe('syncFromPdf', function () {
+      beforeEach(function () {
         this.fs.stat = sinon.stub().callsArgWith(1, null, {
           isFile() {
             return true
@@ -441,7 +447,7 @@ describe('CompileManager', function() {
         this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}\t${this.line}\t${this.column}\n`
         this.CommandRunner.run = sinon
           .stub()
-          .callsArgWith(6, null, { stdout: this.stdout })
+          .callsArgWith(7, null, { stdout: this.stdout })
         return this.CompileManager.syncFromPdf(
           this.project_id,
           this.user_id,
@@ -452,7 +458,7 @@ describe('CompileManager', function() {
         )
       })
 
-      it('should execute the synctex binary', function() {
+      it('should execute the synctex binary', function () {
         const bin_path = Path.resolve(__dirname + '/../../../bin/synctex')
         const synctex_path = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/output.pdf`
         return this.CommandRunner.run
@@ -467,7 +473,7 @@ describe('CompileManager', function() {
           .should.equal(true)
       })
 
-      return it('should call the callback with the parsed output', function() {
+      return it('should call the callback with the parsed output', function () {
         return this.callback
           .calledWith(null, [
             {
@@ -481,9 +487,9 @@ describe('CompileManager', function() {
     })
   })
 
-  return describe('wordcount', function() {
-    beforeEach(function() {
-      this.CommandRunner.run = sinon.stub().callsArg(6)
+  return describe('wordcount', function () {
+    beforeEach(function () {
+      this.CommandRunner.run = sinon.stub().callsArg(7)
       this.fs.readFile = sinon
         .stub()
         .callsArgWith(
@@ -508,7 +514,7 @@ describe('CompileManager', function() {
       )
     })
 
-    it('should run the texcount command', function() {
+    it('should run the texcount command', function () {
       this.directory = `${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}`
       this.file_path = `$COMPILE_DIR/${this.file_name}`
       this.command = [
@@ -531,7 +537,7 @@ describe('CompileManager', function() {
         .should.equal(true)
     })
 
-    return it('should call the callback with the parsed output', function() {
+    return it('should call the callback with the parsed output', function () {
       return this.callback
         .calledWith(null, {
           encode: 'ascii',

test/unit/js/ContentTypeMapperTests.js

@@ -18,61 +18,61 @@ const modulePath = require('path').join(
   '../../../app/js/ContentTypeMapper'
 )
 
-describe('ContentTypeMapper', function() {
-  beforeEach(function() {
+describe('ContentTypeMapper', function () {
+  beforeEach(function () {
     return (this.ContentTypeMapper = SandboxedModule.require(modulePath))
   })
 
-  return describe('map', function() {
-    it('should map .txt to text/plain', function() {
+  return describe('map', function () {
+    it('should map .txt to text/plain', function () {
       const content_type = this.ContentTypeMapper.map('example.txt')
       return content_type.should.equal('text/plain')
     })
 
-    it('should map .csv to text/csv', function() {
+    it('should map .csv to text/csv', function () {
       const content_type = this.ContentTypeMapper.map('example.csv')
       return content_type.should.equal('text/csv')
     })
 
-    it('should map .pdf to application/pdf', function() {
+    it('should map .pdf to application/pdf', function () {
       const content_type = this.ContentTypeMapper.map('example.pdf')
       return content_type.should.equal('application/pdf')
     })
 
-    it('should fall back to octet-stream', function() {
+    it('should fall back to octet-stream', function () {
       const content_type = this.ContentTypeMapper.map('example.unknown')
       return content_type.should.equal('application/octet-stream')
     })
 
-    describe('coercing web files to plain text', function() {
-      it('should map .js to plain text', function() {
+    describe('coercing web files to plain text', function () {
+      it('should map .js to plain text', function () {
         const content_type = this.ContentTypeMapper.map('example.js')
         return content_type.should.equal('text/plain')
       })
 
-      it('should map .html to plain text', function() {
+      it('should map .html to plain text', function () {
         const content_type = this.ContentTypeMapper.map('example.html')
         return content_type.should.equal('text/plain')
       })
 
-      return it('should map .css to plain text', function() {
+      return it('should map .css to plain text', function () {
         const content_type = this.ContentTypeMapper.map('example.css')
         return content_type.should.equal('text/plain')
       })
     })
 
-    return describe('image files', function() {
-      it('should map .png to image/png', function() {
+    return describe('image files', function () {
+      it('should map .png to image/png', function () {
         const content_type = this.ContentTypeMapper.map('example.png')
         return content_type.should.equal('image/png')
       })
 
-      it('should map .jpeg to image/jpeg', function() {
+      it('should map .jpeg to image/jpeg', function () {
         const content_type = this.ContentTypeMapper.map('example.jpeg')
         return content_type.should.equal('image/jpeg')
       })
 
-      return it('should map .svg to text/plain to protect against XSS (SVG can execute JS)', function() {
+      return it('should map .svg to text/plain to protect against XSS (SVG can execute JS)', function () {
         const content_type = this.ContentTypeMapper.map('example.svg')
         return content_type.should.equal('text/plain')
       })

test/unit/js/DockerLockManagerTests.js

@@ -17,8 +17,8 @@ const modulePath = require('path').join(
   '../../../app/js/DockerLockManager'
 )
 
-describe('LockManager', function() {
-  beforeEach(function() {
+describe('LockManager', function () {
+  beforeEach(function () {
     return (this.LockManager = SandboxedModule.require(modulePath, {
       requires: {
         'settings-sharelatex': (this.Settings = { clsi: { docker: {} } }),
@@ -30,13 +30,13 @@ describe('LockManager', function() {
     }))
   })
 
-  return describe('runWithLock', function() {
-    describe('with a single lock', function() {
-      beforeEach(function(done) {
+  return describe('runWithLock', function () {
+    describe('with a single lock', function () {
+      beforeEach(function (done) {
         this.callback = sinon.stub()
         return this.LockManager.runWithLock(
           'lock-one',
-          releaseLock =>
+          (releaseLock) =>
             setTimeout(() => releaseLock(null, 'hello', 'world'), 100),
 
           (err, ...args) => {
@@ -46,20 +46,20 @@ describe('LockManager', function() {
         )
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback
           .calledWith(null, 'hello', 'world')
           .should.equal(true)
       })
     })
 
-    describe('with two locks', function() {
-      beforeEach(function(done) {
+    describe('with two locks', function () {
+      beforeEach(function (done) {
         this.callback1 = sinon.stub()
         this.callback2 = sinon.stub()
         this.LockManager.runWithLock(
           'lock-one',
-          releaseLock =>
+          (releaseLock) =>
             setTimeout(() => releaseLock(null, 'hello', 'world', 'one'), 100),
 
           (err, ...args) => {
@@ -68,7 +68,7 @@ describe('LockManager', function() {
         )
         return this.LockManager.runWithLock(
           'lock-two',
-          releaseLock =>
+          (releaseLock) =>
             setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 200),
 
           (err, ...args) => {
@@ -78,29 +78,29 @@ describe('LockManager', function() {
         )
       })
 
-      it('should call the first callback', function() {
+      it('should call the first callback', function () {
         return this.callback1
           .calledWith(null, 'hello', 'world', 'one')
           .should.equal(true)
       })
 
-      return it('should call the second callback', function() {
+      return it('should call the second callback', function () {
         return this.callback2
           .calledWith(null, 'hello', 'world', 'two')
           .should.equal(true)
       })
     })
 
-    return describe('with lock contention', function() {
-      describe('where the first lock is released quickly', function() {
-        beforeEach(function(done) {
+    return describe('with lock contention', function () {
+      describe('where the first lock is released quickly', function () {
+        beforeEach(function (done) {
           this.LockManager.MAX_LOCK_WAIT_TIME = 1000
           this.LockManager.LOCK_TEST_INTERVAL = 100
           this.callback1 = sinon.stub()
           this.callback2 = sinon.stub()
           this.LockManager.runWithLock(
             'lock',
-            releaseLock =>
+            (releaseLock) =>
               setTimeout(() => releaseLock(null, 'hello', 'world', 'one'), 100),
 
             (err, ...args) => {
@@ -109,7 +109,7 @@ describe('LockManager', function() {
           )
           return this.LockManager.runWithLock(
             'lock',
-            releaseLock =>
+            (releaseLock) =>
               setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 200),
 
             (err, ...args) => {
@@ -119,21 +119,21 @@ describe('LockManager', function() {
           )
         })
 
-        it('should call the first callback', function() {
+        it('should call the first callback', function () {
           return this.callback1
             .calledWith(null, 'hello', 'world', 'one')
             .should.equal(true)
         })
 
-        return it('should call the second callback', function() {
+        return it('should call the second callback', function () {
           return this.callback2
             .calledWith(null, 'hello', 'world', 'two')
             .should.equal(true)
         })
       })
 
-      describe('where the first lock is held longer than the waiting time', function() {
-        beforeEach(function(done) {
+      describe('where the first lock is held longer than the waiting time', function () {
+        beforeEach(function (done) {
           let doneTwo
           this.LockManager.MAX_LOCK_HOLD_TIME = 10000
           this.LockManager.MAX_LOCK_WAIT_TIME = 1000
@@ -141,7 +141,7 @@ describe('LockManager', function() {
           this.callback1 = sinon.stub()
           this.callback2 = sinon.stub()
           let doneOne = (doneTwo = false)
-          const finish = function(key) {
+          const finish = function (key) {
             if (key === 1) {
               doneOne = true
             }
@@ -154,7 +154,7 @@ describe('LockManager', function() {
           }
           this.LockManager.runWithLock(
             'lock',
-            releaseLock =>
+            (releaseLock) =>
               setTimeout(
                 () => releaseLock(null, 'hello', 'world', 'one'),
                 1100
@@ -167,7 +167,7 @@ describe('LockManager', function() {
           )
           return this.LockManager.runWithLock(
             'lock',
-            releaseLock =>
+            (releaseLock) =>
               setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 100),
 
             (err, ...args) => {
@@ -177,20 +177,20 @@ describe('LockManager', function() {
           )
         })
 
-        it('should call the first callback', function() {
+        it('should call the first callback', function () {
           return this.callback1
             .calledWith(null, 'hello', 'world', 'one')
             .should.equal(true)
         })
 
-        return it('should call the second callback with an error', function() {
+        return it('should call the second callback with an error', function () {
           const error = sinon.match.instanceOf(Error)
           return this.callback2.calledWith(error).should.equal(true)
         })
       })
 
-      return describe('where the first lock is held longer than the max holding time', function() {
-        beforeEach(function(done) {
+      return describe('where the first lock is held longer than the max holding time', function () {
+        beforeEach(function (done) {
           let doneTwo
           this.LockManager.MAX_LOCK_HOLD_TIME = 1000
           this.LockManager.MAX_LOCK_WAIT_TIME = 2000
@@ -198,7 +198,7 @@ describe('LockManager', function() {
           this.callback1 = sinon.stub()
           this.callback2 = sinon.stub()
           let doneOne = (doneTwo = false)
-          const finish = function(key) {
+          const finish = function (key) {
             if (key === 1) {
               doneOne = true
             }
@@ -211,7 +211,7 @@ describe('LockManager', function() {
           }
           this.LockManager.runWithLock(
             'lock',
-            releaseLock =>
+            (releaseLock) =>
               setTimeout(
                 () => releaseLock(null, 'hello', 'world', 'one'),
                 1500
@@ -224,7 +224,7 @@ describe('LockManager', function() {
           )
           return this.LockManager.runWithLock(
             'lock',
-            releaseLock =>
+            (releaseLock) =>
               setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 100),
 
             (err, ...args) => {
@@ -234,13 +234,13 @@ describe('LockManager', function() {
           )
         })
 
-        it('should call the first callback', function() {
+        it('should call the first callback', function () {
           return this.callback1
             .calledWith(null, 'hello', 'world', 'one')
             .should.equal(true)
         })
 
-        return it('should call the second callback', function() {
+        return it('should call the second callback', function () {
           return this.callback2
             .calledWith(null, 'hello', 'world', 'two')
             .should.equal(true)

test/unit/js/DockerRunnerTests.js

@@ -23,8 +23,8 @@ const modulePath = require('path').join(
 )
 const Path = require('path')
 
-describe('DockerRunner', function() {
-  beforeEach(function() {
+describe('DockerRunner', function () {
+  beforeEach(function () {
     let container, Docker, Timer
     this.container = container = {}
     this.DockerRunner = SandboxedModule.require(modulePath, {
@@ -39,7 +39,7 @@ describe('DockerRunner', function() {
           info: sinon.stub(),
           warn: sinon.stub()
         }),
-        dockerode: (Docker = (function() {
+        dockerode: (Docker = (function () {
           Docker = class Docker {
             static initClass() {
               this.prototype.getContainer = sinon.stub().returns(container)
@@ -69,7 +69,8 @@ describe('DockerRunner', function() {
             return runner(callback)
           }
         }
-      }
+      },
+      globals: { Math } // used by lodash
     })
     this.Docker = Docker
     this.getContainer = Docker.prototype.getContainer
@@ -85,15 +86,16 @@ describe('DockerRunner', function() {
     this.project_id = 'project-id-123'
     this.volumes = { '/local/compile/directory': '/compile' }
     this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
+    this.compileGroup = 'compile-group'
     return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
   })
 
-  afterEach(function() {
+  afterEach(function () {
     this.DockerRunner.stopContainerMonitor()
   })
 
-  describe('run', function() {
-    beforeEach(function(done) {
+  describe('run', function () {
+    beforeEach(function (done) {
       this.DockerRunner._getContainerOptions = sinon
         .stub()
         .returns((this.options = { mockoptions: 'foo' }))
@@ -109,8 +111,8 @@ describe('DockerRunner', function() {
       return done()
     })
 
-    describe('successfully', function() {
-      beforeEach(function(done) {
+    describe('successfully', function () {
+      beforeEach(function (done) {
         this.DockerRunner._runAndWaitForContainer = sinon
           .stub()
           .callsArgWith(3, null, (this.output = 'mock-output'))
@@ -121,6 +123,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           (err, output) => {
             this.callback(err, output)
             return done()
@@ -128,7 +131,7 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should generate the options for the container', function() {
+      it('should generate the options for the container', function () {
         return this.DockerRunner._getContainerOptions
           .calledWith(
             this.command_with_dir,
@@ -139,25 +142,25 @@ describe('DockerRunner', function() {
           .should.equal(true)
       })
 
-      it('should generate the fingerprint from the returned options', function() {
+      it('should generate the fingerprint from the returned options', function () {
         return this.DockerRunner._fingerprintContainer
           .calledWith(this.options)
           .should.equal(true)
       })
 
-      it('should do the run', function() {
+      it('should do the run', function () {
         return this.DockerRunner._runAndWaitForContainer
           .calledWith(this.options, this.volumes, this.timeout)
           .should.equal(true)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.calledWith(null, this.output).should.equal(true)
       })
     })
 
-    describe('when path.sandboxedCompilesHostDir is set', function() {
-      beforeEach(function() {
+    describe('when path.sandboxedCompilesHostDir is set', function () {
+      beforeEach(function () {
         this.Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles'
         this.directory = '/var/lib/sharelatex/data/compiles/xyz'
         this.DockerRunner._runAndWaitForContainer = sinon
@@ -170,11 +173,12 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
 
-      it('should re-write the bind directory', function() {
+      it('should re-write the bind directory', function () {
         const volumes = this.DockerRunner._runAndWaitForContainer.lastCall
           .args[1]
         return expect(volumes).to.deep.equal({
@@ -182,13 +186,13 @@ describe('DockerRunner', function() {
         })
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.calledWith(null, this.output).should.equal(true)
       })
     })
 
-    describe('when the run throws an error', function() {
-      beforeEach(function() {
+    describe('when the run throws an error', function () {
+      beforeEach(function () {
         let firstTime = true
         this.output = 'mock-output'
         this.DockerRunner._runAndWaitForContainer = (
@@ -198,7 +202,7 @@ describe('DockerRunner', function() {
           callback
         ) => {
           if (callback == null) {
-            callback = function(error, output) {}
+            callback = function (error, output) {}
           }
           if (firstTime) {
             firstTime = false
@@ -218,29 +222,30 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
 
-      it('should do the run twice', function() {
+      it('should do the run twice', function () {
         return this.DockerRunner._runAndWaitForContainer.calledTwice.should.equal(
           true
         )
       })
 
-      it('should destroy the container in between', function() {
+      it('should destroy the container in between', function () {
         return this.DockerRunner.destroyContainer
           .calledWith(this.name, null)
           .should.equal(true)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.calledWith(null, this.output).should.equal(true)
       })
     })
 
-    describe('with no image', function() {
-      beforeEach(function() {
+    describe('with no image', function () {
+      beforeEach(function () {
         this.DockerRunner._runAndWaitForContainer = sinon
           .stub()
           .callsArgWith(3, null, (this.output = 'mock-output'))
@@ -251,11 +256,12 @@ describe('DockerRunner', function() {
           null,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
 
-      return it('should use the default image', function() {
+      return it('should use the default image', function () {
         return this.DockerRunner._getContainerOptions
           .calledWith(
             this.command_with_dir,
@@ -267,8 +273,8 @@ describe('DockerRunner', function() {
       })
     })
 
-    return describe('with image override', function() {
-      beforeEach(function() {
+    describe('with image override', function () {
+      beforeEach(function () {
         this.Settings.texliveImageNameOveride = 'overrideimage.com/something'
         this.DockerRunner._runAndWaitForContainer = sinon
           .stub()
@@ -280,19 +286,134 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
 
-      return it('should use the override and keep the tag', function() {
+      return it('should use the override and keep the tag', function () {
         const image = this.DockerRunner._getContainerOptions.args[0][1]
         return image.should.equal('overrideimage.com/something/image:2016.2')
       })
     })
+
+    describe('with image restriction', function () {
+      beforeEach(function () {
+        this.Settings.clsi.docker.allowedImages = [
+          'repo/image:tag1',
+          'repo/image:tag2'
+        ]
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
       })
 
-  describe('_runAndWaitForContainer', function() {
-    beforeEach(function() {
+      describe('with a valid image', function () {
+        beforeEach(function () {
+          this.DockerRunner.run(
+            this.project_id,
+            this.command,
+            this.directory,
+            'repo/image:tag1',
+            this.timeout,
+            this.env,
+            this.compileGroup,
+            this.callback
+          )
+        })
+
+        it('should setup the container', function () {
+          this.DockerRunner._getContainerOptions.called.should.equal(true)
+        })
+      })
+
+      describe('with a invalid image', function () {
+        beforeEach(function () {
+          this.DockerRunner.run(
+            this.project_id,
+            this.command,
+            this.directory,
+            'something/different:evil',
+            this.timeout,
+            this.env,
+            this.compileGroup,
+            this.callback
+          )
+        })
+
+        it('should call the callback with an error', function () {
+          const err = new Error('image not allowed')
+          this.callback.called.should.equal(true)
+          this.callback.args[0][0].message.should.equal(err.message)
+        })
+
+        it('should not setup the container', function () {
+          this.DockerRunner._getContainerOptions.called.should.equal(false)
+        })
+      })
+    })
+  })
+
+  describe('run with _getOptions', function () {
+    beforeEach(function (done) {
+      // this.DockerRunner._getContainerOptions = sinon
+      //   .stub()
+      //   .returns((this.options = { mockoptions: 'foo' }))
+      this.DockerRunner._fingerprintContainer = sinon
+        .stub()
+        .returns((this.fingerprint = 'fingerprint'))
+
+      this.name = `project-${this.project_id}-${this.fingerprint}`
+
+      this.command = ['mock', 'command', '--outdir=$COMPILE_DIR']
+      this.command_with_dir = ['mock', 'command', '--outdir=/compile']
+      this.timeout = 42000
+      return done()
+    })
+
+    describe('when a compile group config is set', function () {
+      beforeEach(function () {
+        this.Settings.clsi.docker.compileGroupConfig = {
+          'compile-group': {
+            'HostConfig.newProperty': 'new-property'
+          },
+          'other-group': { otherProperty: 'other-property' }
+        }
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+        return this.DockerRunner.run(
+          this.project_id,
+          this.command,
+          this.directory,
+          this.image,
+          this.timeout,
+          this.env,
+          this.compileGroup,
+          this.callback
+        )
+      })
+
+      it('should set the docker options for the compile group', function () {
+        const options = this.DockerRunner._runAndWaitForContainer.lastCall
+          .args[0]
+        return expect(options.HostConfig).to.deep.include({
+          Binds: ['/local/compile/directory:/compile:rw'],
+          LogConfig: { Type: 'none', Config: {} },
+          CapDrop: 'ALL',
+          SecurityOpt: ['no-new-privileges'],
+          newProperty: 'new-property'
+        })
+      })
+
+      return it('should call the callback', function () {
+        return this.callback.calledWith(null, this.output).should.equal(true)
+      })
+    })
+  })
+
+  describe('_runAndWaitForContainer', function () {
+    beforeEach(function () {
       this.options = { mockoptions: 'foo', name: (this.name = 'mock-name') }
       this.DockerRunner.startContainer = (
         options,
@@ -315,25 +436,25 @@ describe('DockerRunner', function() {
       )
     })
 
-    it('should create/start the container', function() {
+    it('should create/start the container', function () {
       return this.DockerRunner.startContainer
         .calledWith(this.options, this.volumes)
         .should.equal(true)
     })
 
-    it('should wait for the container to finish', function() {
+    it('should wait for the container to finish', function () {
       return this.DockerRunner.waitForContainer
         .calledWith(this.name, this.timeout)
         .should.equal(true)
     })
 
-    return it('should call the callback with the output', function() {
+    return it('should call the callback with the output', function () {
       return this.callback.calledWith(null, this.output).should.equal(true)
     })
   })
 
-  describe('startContainer', function() {
-    beforeEach(function() {
+  describe('startContainer', function () {
+    beforeEach(function () {
       this.attachStreamHandler = sinon.stub()
       this.attachStreamHandler.cock = true
       this.options = { mockoptions: 'foo', name: 'mock-name' }
@@ -349,37 +470,37 @@ describe('DockerRunner', function() {
       return sinon.spy(this.DockerRunner, 'attachToContainer')
     })
 
-    describe('when the container exists', function() {
-      beforeEach(function() {
+    describe('when the container exists', function () {
+      beforeEach(function () {
         this.container.inspect = sinon.stub().callsArgWith(0)
         this.container.start = sinon.stub().yields()
 
         return this.DockerRunner.startContainer(
           this.options,
           this.volumes,
-          this.callback,
-          () => {}
+          () => {},
+          this.callback
         )
       })
 
-      it('should start the container with the given name', function() {
+      it('should start the container with the given name', function () {
         this.getContainer.calledWith(this.options.name).should.equal(true)
         return this.container.start.called.should.equal(true)
       })
 
-      it('should not try to create the container', function() {
+      it('should not try to create the container', function () {
         return this.createContainer.called.should.equal(false)
       })
 
-      it('should attach to the container', function() {
+      it('should attach to the container', function () {
         return this.DockerRunner.attachToContainer.called.should.equal(true)
       })
 
-      it('should call the callback', function() {
+      it('should call the callback', function () {
         return this.callback.called.should.equal(true)
       })
 
-      return it('should attach before the container starts', function() {
+      return it('should attach before the container starts', function () {
         return sinon.assert.callOrder(
           this.DockerRunner.attachToContainer,
           this.container.start
@@ -387,8 +508,8 @@ describe('DockerRunner', function() {
       })
     })
 
-    describe('when the container does not exist', function() {
-      beforeEach(function() {
+    describe('when the container does not exist', function () {
+      beforeEach(function () {
         const exists = false
         this.container.start = sinon.stub().yields()
         this.container.inspect = sinon
@@ -402,20 +523,20 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should create the container', function() {
+      it('should create the container', function () {
         return this.createContainer.calledWith(this.options).should.equal(true)
       })
 
-      it('should call the callback and stream handler', function() {
+      it('should call the callback and stream handler', function () {
         this.attachStreamHandler.called.should.equal(true)
         return this.callback.called.should.equal(true)
       })
 
-      it('should attach to the container', function() {
+      it('should attach to the container', function () {
         return this.DockerRunner.attachToContainer.called.should.equal(true)
       })
 
-      return it('should attach before the container starts', function() {
+      return it('should attach before the container starts', function () {
         return sinon.assert.callOrder(
           this.DockerRunner.attachToContainer,
           this.container.start
@@ -423,8 +544,8 @@ describe('DockerRunner', function() {
       })
     })
 
-    describe('when the container is already running', function() {
-      beforeEach(function() {
+    describe('when the container is already running', function () {
+      beforeEach(function () {
         const error = new Error(
           `HTTP code is 304 which indicates error: server error - start: Cannot start container ${this.name}: The container MOCKID is already running.`
         )
@@ -439,18 +560,18 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should not try to create the container', function() {
+      it('should not try to create the container', function () {
         return this.createContainer.called.should.equal(false)
       })
 
-      return it('should call the callback  and stream handler without an error', function() {
+      return it('should call the callback  and stream handler without an error', function () {
         this.attachStreamHandler.called.should.equal(true)
         return this.callback.called.should.equal(true)
       })
     })
 
-    describe('when a volume does not exist', function() {
-      beforeEach(function() {
+    describe('when a volume does not exist', function () {
+      beforeEach(function () {
         this.fs.stat = sinon.stub().yields(new Error('no such path'))
         return this.DockerRunner.startContainer(
           this.options,
@@ -460,17 +581,17 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should not try to create the container', function() {
+      it('should not try to create the container', function () {
         return this.createContainer.called.should.equal(false)
       })
 
-      it('should call the callback with an error', function() {
+      it('should call the callback with an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
       })
     })
 
-    describe('when a volume exists but is not a directory', function() {
-      beforeEach(function() {
+    describe('when a volume exists but is not a directory', function () {
+      beforeEach(function () {
         this.fs.stat = sinon.stub().yields(null, {
           isDirectory() {
             return false
@@ -484,17 +605,17 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should not try to create the container', function() {
+      it('should not try to create the container', function () {
         return this.createContainer.called.should.equal(false)
       })
 
-      it('should call the callback with an error', function() {
+      it('should call the callback with an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
       })
     })
 
-    describe('when a volume does not exist, but sibling-containers are used', function() {
-      beforeEach(function() {
+    describe('when a volume does not exist, but sibling-containers are used', function () {
+      beforeEach(function () {
         this.fs.stat = sinon.stub().yields(new Error('no such path'))
         this.Settings.path.sandboxedCompilesHostDir = '/some/path'
         this.container.start = sinon.stub().yields()
@@ -505,30 +626,30 @@ describe('DockerRunner', function() {
         )
       })
 
-      afterEach(function() {
+      afterEach(function () {
         return delete this.Settings.path.sandboxedCompilesHostDir
       })
 
-      it('should start the container with the given name', function() {
+      it('should start the container with the given name', function () {
         this.getContainer.calledWith(this.options.name).should.equal(true)
         return this.container.start.called.should.equal(true)
       })
 
-      it('should not try to create the container', function() {
+      it('should not try to create the container', function () {
         return this.createContainer.called.should.equal(false)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         this.callback.called.should.equal(true)
         return this.callback.calledWith(new Error()).should.equal(false)
       })
     })
 
-    return describe('when the container tries to be created, but already has been (race condition)', function() {})
+    return describe('when the container tries to be created, but already has been (race condition)', function () {})
   })
 
-  describe('waitForContainer', function() {
-    beforeEach(function() {
+  describe('waitForContainer', function () {
+    beforeEach(function () {
       this.containerId = 'container-id'
       this.timeout = 5000
       this.container.wait = sinon
@@ -537,8 +658,8 @@ describe('DockerRunner', function() {
       return (this.container.kill = sinon.stub().yields())
     })
 
-    describe('when the container returns in time', function() {
-      beforeEach(function() {
+    describe('when the container returns in time', function () {
+      beforeEach(function () {
         return this.DockerRunner.waitForContainer(
           this.containerId,
           this.timeout,
@@ -546,23 +667,23 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should wait for the container', function() {
+      it('should wait for the container', function () {
         this.getContainer.calledWith(this.containerId).should.equal(true)
         return this.container.wait.called.should.equal(true)
       })
 
-      return it('should call the callback with the exit', function() {
+      return it('should call the callback with the exit', function () {
         return this.callback
           .calledWith(null, this.statusCode)
           .should.equal(true)
       })
     })
 
-    return describe('when the container does not return before the timeout', function() {
-      beforeEach(function(done) {
-        this.container.wait = function(callback) {
+    return describe('when the container does not return before the timeout', function () {
+      beforeEach(function (done) {
+        this.container.wait = function (callback) {
           if (callback == null) {
-            callback = function(error, exitCode) {}
+            callback = function (error, exitCode) {}
           }
           return setTimeout(() => callback(null, { StatusCode: 42 }), 100)
         }
@@ -577,12 +698,12 @@ describe('DockerRunner', function() {
         )
       })
 
-      it('should call kill on the container', function() {
+      it('should call kill on the container', function () {
         this.getContainer.calledWith(this.containerId).should.equal(true)
         return this.container.kill.called.should.equal(true)
       })
 
-      it('should call the callback with an error', function() {
+      it('should call the callback with an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
 
         const errorObj = this.callback.args[0][0]
@@ -592,8 +713,8 @@ describe('DockerRunner', function() {
     })
   })
 
-  describe('destroyOldContainers', function() {
-    beforeEach(function(done) {
+  describe('destroyOldContainers', function () {
+    beforeEach(function (done) {
       const oneHourInSeconds = 60 * 60
       const oneHourInMilliseconds = oneHourInSeconds * 1000
       const nowInSeconds = Date.now() / 1000
@@ -617,42 +738,42 @@ describe('DockerRunner', function() {
       this.DockerRunner.MAX_CONTAINER_AGE = oneHourInMilliseconds
       this.listContainers.callsArgWith(1, null, this.containers)
       this.DockerRunner.destroyContainer = sinon.stub().callsArg(3)
-      return this.DockerRunner.destroyOldContainers(error => {
+      return this.DockerRunner.destroyOldContainers((error) => {
         this.callback(error)
         return done()
       })
     })
 
-    it('should list all containers', function() {
+    it('should list all containers', function () {
       return this.listContainers.calledWith({ all: true }).should.equal(true)
     })
 
-    it('should destroy old containers', function() {
+    it('should destroy old containers', function () {
       this.DockerRunner.destroyContainer.callCount.should.equal(1)
       return this.DockerRunner.destroyContainer
-        .calledWith('/project-old-container-name', 'old-container-id')
+        .calledWith('project-old-container-name', 'old-container-id')
         .should.equal(true)
     })
 
-    it('should not destroy new containers', function() {
+    it('should not destroy new containers', function () {
       return this.DockerRunner.destroyContainer
-        .calledWith('/project-new-container-name', 'new-container-id')
+        .calledWith('project-new-container-name', 'new-container-id')
         .should.equal(false)
     })
 
-    it('should not destroy non-project containers', function() {
+    it('should not destroy non-project containers', function () {
       return this.DockerRunner.destroyContainer
-        .calledWith('/totally-not-a-project-container', 'some-random-id')
+        .calledWith('totally-not-a-project-container', 'some-random-id')
         .should.equal(false)
     })
 
-    return it('should callback the callback', function() {
+    return it('should callback the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  describe('_destroyContainer', function() {
-    beforeEach(function() {
+  describe('_destroyContainer', function () {
+    beforeEach(function () {
       this.containerId = 'some_id'
       this.fakeContainer = { remove: sinon.stub().callsArgWith(1, null) }
       return (this.Docker.prototype.getContainer = sinon
@@ -660,11 +781,11 @@ describe('DockerRunner', function() {
         .returns(this.fakeContainer))
     })
 
-    it('should get the container', function(done) {
+    it('should get the container', function (done) {
       return this.DockerRunner._destroyContainer(
         this.containerId,
         false,
-        err => {
+        (err) => {
           this.Docker.prototype.getContainer.callCount.should.equal(1)
           this.Docker.prototype.getContainer
             .calledWith(this.containerId)
@@ -674,47 +795,47 @@ describe('DockerRunner', function() {
       )
     })
 
-    it('should try to force-destroy the container when shouldForce=true', function(done) {
+    it('should try to force-destroy the container when shouldForce=true', function (done) {
       return this.DockerRunner._destroyContainer(
         this.containerId,
         true,
-        err => {
+        (err) => {
           this.fakeContainer.remove.callCount.should.equal(1)
           this.fakeContainer.remove
-            .calledWith({ force: true })
+            .calledWith({ force: true, v: true })
             .should.equal(true)
           return done()
         }
       )
     })
 
-    it('should not try to force-destroy the container when shouldForce=false', function(done) {
+    it('should not try to force-destroy the container when shouldForce=false', function (done) {
       return this.DockerRunner._destroyContainer(
         this.containerId,
         false,
-        err => {
+        (err) => {
           this.fakeContainer.remove.callCount.should.equal(1)
           this.fakeContainer.remove
-            .calledWith({ force: false })
+            .calledWith({ force: false, v: true })
             .should.equal(true)
           return done()
         }
       )
     })
 
-    it('should not produce an error', function(done) {
+    it('should not produce an error', function (done) {
       return this.DockerRunner._destroyContainer(
         this.containerId,
         false,
-        err => {
+        (err) => {
           expect(err).to.equal(null)
           return done()
         }
       )
     })
 
-    describe('when the container is already gone', function() {
-      beforeEach(function() {
+    describe('when the container is already gone', function () {
+      beforeEach(function () {
         this.fakeError = new Error('woops')
         this.fakeError.statusCode = 404
         this.fakeContainer = {
@@ -725,11 +846,11 @@ describe('DockerRunner', function() {
           .returns(this.fakeContainer))
       })
 
-      return it('should not produce an error', function(done) {
+      return it('should not produce an error', function (done) {
         return this.DockerRunner._destroyContainer(
           this.containerId,
           false,
-          err => {
+          (err) => {
             expect(err).to.equal(null)
             return done()
           }
@@ -737,8 +858,8 @@ describe('DockerRunner', function() {
       })
     })
 
-    return describe('when container.destroy produces an error', function(done) {
-      beforeEach(function() {
+    return describe('when container.destroy produces an error', function (done) {
+      beforeEach(function () {
         this.fakeError = new Error('woops')
         this.fakeError.statusCode = 500
         this.fakeContainer = {
@@ -749,11 +870,11 @@ describe('DockerRunner', function() {
           .returns(this.fakeContainer))
       })
 
-      return it('should produce an error', function(done) {
+      return it('should produce an error', function (done) {
         return this.DockerRunner._destroyContainer(
           this.containerId,
           false,
-          err => {
+          (err) => {
             expect(err).to.not.equal(null)
             expect(err).to.equal(this.fakeError)
             return done()
@@ -763,8 +884,8 @@ describe('DockerRunner', function() {
     })
   })
 
-  return describe('kill', function() {
-    beforeEach(function() {
+  return describe('kill', function () {
+    beforeEach(function () {
       this.containerId = 'some_id'
       this.fakeContainer = { kill: sinon.stub().callsArgWith(0, null) }
       return (this.Docker.prototype.getContainer = sinon
@@ -772,8 +893,8 @@ describe('DockerRunner', function() {
         .returns(this.fakeContainer))
     })
 
-    it('should get the container', function(done) {
-      return this.DockerRunner.kill(this.containerId, err => {
+    it('should get the container', function (done) {
+      return this.DockerRunner.kill(this.containerId, (err) => {
         this.Docker.prototype.getContainer.callCount.should.equal(1)
         this.Docker.prototype.getContainer
           .calledWith(this.containerId)
@@ -782,22 +903,22 @@ describe('DockerRunner', function() {
       })
     })
 
-    it('should try to force-destroy the container', function(done) {
-      return this.DockerRunner.kill(this.containerId, err => {
+    it('should try to force-destroy the container', function (done) {
+      return this.DockerRunner.kill(this.containerId, (err) => {
         this.fakeContainer.kill.callCount.should.equal(1)
         return done()
       })
     })
 
-    it('should not produce an error', function(done) {
-      return this.DockerRunner.kill(this.containerId, err => {
+    it('should not produce an error', function (done) {
+      return this.DockerRunner.kill(this.containerId, (err) => {
         expect(err).to.equal(undefined)
         return done()
       })
     })
 
-    describe('when the container is not actually running', function() {
-      beforeEach(function() {
+    describe('when the container is not actually running', function () {
+      beforeEach(function () {
         this.fakeError = new Error('woops')
         this.fakeError.statusCode = 500
         this.fakeError.message =
@@ -810,16 +931,16 @@ describe('DockerRunner', function() {
           .returns(this.fakeContainer))
       })
 
-      return it('should not produce an error', function(done) {
-        return this.DockerRunner.kill(this.containerId, err => {
+      return it('should not produce an error', function (done) {
+        return this.DockerRunner.kill(this.containerId, (err) => {
           expect(err).to.equal(undefined)
           return done()
         })
       })
     })
 
-    return describe('when container.kill produces a legitimate error', function(done) {
-      beforeEach(function() {
+    return describe('when container.kill produces a legitimate error', function (done) {
+      beforeEach(function () {
         this.fakeError = new Error('woops')
         this.fakeError.statusCode = 500
         this.fakeError.message = 'Totally legitimate reason to throw an error'
@@ -831,8 +952,8 @@ describe('DockerRunner', function() {
           .returns(this.fakeContainer))
       })
 
-      return it('should produce an error', function(done) {
-        return this.DockerRunner.kill(this.containerId, err => {
+      return it('should produce an error', function (done) {
+        return this.DockerRunner.kill(this.containerId, (err) => {
           expect(err).to.not.equal(undefined)
           expect(err).to.equal(this.fakeError)
           return done()

test/unit/js/DraftModeManagerTests.js

@@ -16,8 +16,8 @@ const modulePath = require('path').join(
   '../../../app/js/DraftModeManager'
 )
 
-describe('DraftModeManager', function() {
-  beforeEach(function() {
+describe('DraftModeManager', function () {
+  beforeEach(function () {
     return (this.DraftModeManager = SandboxedModule.require(modulePath, {
       requires: {
         fs: (this.fs = {}),
@@ -26,8 +26,8 @@ describe('DraftModeManager', function() {
     }))
   })
 
-  describe('_injectDraftOption', function() {
-    it('should add draft option into documentclass with existing options', function() {
+  describe('_injectDraftOption', function () {
+    it('should add draft option into documentclass with existing options', function () {
       return this.DraftModeManager._injectDraftOption(`\
 \\documentclass[a4paper,foo=bar]{article}\
 `).should.equal(`\
@@ -35,7 +35,7 @@ describe('DraftModeManager', function() {
 `)
     })
 
-    return it('should add draft option into documentclass with no options', function() {
+    return it('should add draft option into documentclass with no options', function () {
       return this.DraftModeManager._injectDraftOption(`\
 \\documentclass{article}\
 `).should.equal(`\
@@ -44,8 +44,8 @@ describe('DraftModeManager', function() {
     })
   })
 
-  return describe('injectDraftMode', function() {
-    beforeEach(function() {
+  return describe('injectDraftMode', function () {
+    beforeEach(function () {
       this.filename = '/mock/filename.tex'
       this.callback = sinon.stub()
       const content = `\
@@ -59,13 +59,13 @@ Hello world
       return this.DraftModeManager.injectDraftMode(this.filename, this.callback)
     })
 
-    it('should read the file', function() {
+    it('should read the file', function () {
       return this.fs.readFile
         .calledWith(this.filename, 'utf8')
         .should.equal(true)
     })
 
-    it('should write the modified file', function() {
+    it('should write the modified file', function () {
       return this.fs.writeFile
         .calledWith(
           this.filename,
@@ -79,7 +79,7 @@ Hello world
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })

test/unit/js/LatexRunnerTests.js

@@ -18,8 +18,8 @@ const modulePath = require('path').join(
 )
 const Path = require('path')
 
-describe('LatexRunner', function() {
-  beforeEach(function() {
+describe('LatexRunner', function () {
+  beforeEach(function () {
     let Timer
     this.LatexRunner = SandboxedModule.require(modulePath, {
       requires: {
@@ -37,7 +37,10 @@ describe('LatexRunner', function() {
             done() {}
           })
         },
-        './CommandRunner': (this.CommandRunner = {})
+        './CommandRunner': (this.CommandRunner = {}),
+        fs: (this.fs = {
+          writeFile: sinon.stub().callsArg(2)
+        })
       }
     })
 
@@ -45,18 +48,22 @@ describe('LatexRunner', function() {
     this.mainFile = 'main-file.tex'
     this.compiler = 'pdflatex'
     this.image = 'example.com/image'
+    this.compileGroup = 'compile-group'
     this.callback = sinon.stub()
     this.project_id = 'project-id-123'
     return (this.env = { foo: '123' })
   })
 
-  return describe('runLatex', function() {
-    beforeEach(function() {
-      return (this.CommandRunner.run = sinon.stub().callsArg(6))
+  return describe('runLatex', function () {
+    beforeEach(function () {
+      return (this.CommandRunner.run = sinon.stub().callsArgWith(7, null, {
+        stdout: 'this is stdout',
+        stderr: 'this is stderr'
+      }))
     })
 
-    describe('normally', function() {
-      beforeEach(function() {
+    describe('normally', function () {
+      beforeEach(function () {
         return this.LatexRunner.runLatex(
           this.project_id,
           {
@@ -65,13 +72,14 @@ describe('LatexRunner', function() {
             compiler: this.compiler,
             timeout: (this.timeout = 42000),
             image: this.image,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           },
           this.callback
         )
       })
 
-      return it('should run the latex command', function() {
+      it('should run the latex command', function () {
         return this.CommandRunner.run
           .calledWith(
             this.project_id,
@@ -79,14 +87,24 @@ describe('LatexRunner', function() {
             this.directory,
             this.image,
             this.timeout,
-            this.env
+            this.env,
+            this.compileGroup
           )
           .should.equal(true)
       })
+
+      it('should record the stdout and stderr', function () {
+        this.fs.writeFile
+          .calledWith(this.directory + '/' + 'output.stdout', 'this is stdout')
+          .should.equal(true)
+        this.fs.writeFile
+          .calledWith(this.directory + '/' + 'output.stderr', 'this is stderr')
+          .should.equal(true)
+      })
     })
 
-    describe('with an .Rtex main file', function() {
-      beforeEach(function() {
+    describe('with an .Rtex main file', function () {
+      beforeEach(function () {
         return this.LatexRunner.runLatex(
           this.project_id,
           {
@@ -100,15 +118,15 @@ describe('LatexRunner', function() {
         )
       })
 
-      return it('should run the latex command on the equivalent .tex file', function() {
+      return it('should run the latex command on the equivalent .tex file', function () {
         const command = this.CommandRunner.run.args[0][1]
         const mainFile = command.slice(-1)[0]
         return mainFile.should.equal('$COMPILE_DIR/main-file.tex')
       })
     })
 
-    return describe('with a flags option', function() {
-      beforeEach(function() {
+    return describe('with a flags option', function () {
+      beforeEach(function () {
         return this.LatexRunner.runLatex(
           this.project_id,
           {
@@ -123,10 +141,10 @@ describe('LatexRunner', function() {
         )
       })
 
-      return it('should include the flags in the command', function() {
+      return it('should include the flags in the command', function () {
         const command = this.CommandRunner.run.args[0][1]
         const flags = command.filter(
-          arg => arg === '-file-line-error' || arg === '-halt-on-error'
+          (arg) => arg === '-file-line-error' || arg === '-halt-on-error'
         )
         flags.length.should.equal(2)
         flags[0].should.equal('-file-line-error')

test/unit/js/LockManagerTests.js

@@ -19,8 +19,8 @@ const modulePath = require('path').join(
 const Path = require('path')
 const Errors = require('../../../app/js/Errors')
 
-describe('DockerLockManager', function() {
-  beforeEach(function() {
+describe('DockerLockManager', function () {
+  beforeEach(function () {
     this.LockManager = SandboxedModule.require(modulePath, {
       requires: {
         'settings-sharelatex': {},
@@ -39,14 +39,14 @@ describe('DockerLockManager', function() {
     return (this.lockFile = '/local/compile/directory/.project-lock')
   })
 
-  return describe('runWithLock', function() {
-    beforeEach(function() {
+  return describe('runWithLock', function () {
+    beforeEach(function () {
       this.runner = sinon.stub().callsArgWith(0, null, 'foo', 'bar')
       return (this.callback = sinon.stub())
     })
 
-    describe('normally', function() {
-      beforeEach(function() {
+    describe('normally', function () {
+      beforeEach(function () {
         this.Lockfile.lock = sinon.stub().callsArgWith(2, null)
         this.Lockfile.unlock = sinon.stub().callsArgWith(1, null)
         return this.LockManager.runWithLock(
@@ -56,19 +56,19 @@ describe('DockerLockManager', function() {
         )
       })
 
-      it('should run the compile', function() {
+      it('should run the compile', function () {
         return this.runner.calledWith().should.equal(true)
       })
 
-      return it('should call the callback with the response from the compile', function() {
+      return it('should call the callback with the response from the compile', function () {
         return this.callback
           .calledWithExactly(null, 'foo', 'bar')
           .should.equal(true)
       })
     })
 
-    return describe('when the project is locked', function() {
-      beforeEach(function() {
+    return describe('when the project is locked', function () {
+      beforeEach(function () {
         this.error = new Error()
         this.error.code = 'EEXIST'
         this.Lockfile.lock = sinon.stub().callsArgWith(2, this.error)
@@ -80,11 +80,11 @@ describe('DockerLockManager', function() {
         )
       })
 
-      it('should not run the compile', function() {
+      it('should not run the compile', function () {
         return this.runner.called.should.equal(false)
       })
 
-      it('should return an error', function() {
+      it('should return an error', function () {
         this.callback
           .calledWithExactly(sinon.match(Errors.AlreadyCompilingError))
           .should.equal(true)

test/unit/js/OutputFileFinderTests.js

@@ -21,8 +21,8 @@ const path = require('path')
 const { expect } = require('chai')
 const { EventEmitter } = require('events')
 
-describe('OutputFileFinder', function() {
-  beforeEach(function() {
+describe('OutputFileFinder', function () {
+  beforeEach(function () {
     this.OutputFileFinder = SandboxedModule.require(modulePath, {
       requires: {
         fs: (this.fs = {}),
@@ -34,8 +34,8 @@ describe('OutputFileFinder', function() {
     return (this.callback = sinon.stub())
   })
 
-  describe('findOutputFiles', function() {
-    beforeEach(function() {
+  describe('findOutputFiles', function () {
+    beforeEach(function () {
       this.resource_path = 'resource/path.tex'
       this.output_paths = ['output.pdf', 'extra/file.tex']
       this.all_paths = this.output_paths.concat([this.resource_path])
@@ -52,7 +52,7 @@ describe('OutputFileFinder', function() {
       )
     })
 
-    return it('should only return the output files, not directories or resource paths', function() {
+    return it('should only return the output files, not directories or resource paths', function () {
       return expect(this.outputFiles).to.deep.equal([
         {
           path: 'output.pdf',
@@ -66,17 +66,18 @@ describe('OutputFileFinder', function() {
     })
   })
 
-  return describe('_getAllFiles', function() {
-    beforeEach(function() {
+  return describe('_getAllFiles', function () {
+    beforeEach(function () {
       this.proc = new EventEmitter()
       this.proc.stdout = new EventEmitter()
+      this.proc.stdout.setEncoding = sinon.stub().returns(this.proc.stdout)
       this.spawn.returns(this.proc)
       this.directory = '/base/dir'
       return this.OutputFileFinder._getAllFiles(this.directory, this.callback)
     })
 
-    describe('successfully', function() {
-      beforeEach(function() {
+    describe('successfully', function () {
+      beforeEach(function () {
         this.proc.stdout.emit(
           'data',
           ['/base/dir/main.tex', '/base/dir/chapters/chapter1.tex'].join('\n') +
@@ -85,19 +86,19 @@ describe('OutputFileFinder', function() {
         return this.proc.emit('close', 0)
       })
 
-      return it('should call the callback with the relative file paths', function() {
+      return it('should call the callback with the relative file paths', function () {
         return this.callback
           .calledWith(null, ['main.tex', 'chapters/chapter1.tex'])
           .should.equal(true)
       })
     })
 
-    return describe("when the directory doesn't exist", function() {
-      beforeEach(function() {
+    return describe("when the directory doesn't exist", function () {
+      beforeEach(function () {
         return this.proc.emit('close', 1)
       })
 
-      return it('should call the callback with a blank array', function() {
+      return it('should call the callback with a blank array', function () {
         return this.callback.calledWith(null, []).should.equal(true)
       })
     })

test/unit/js/OutputFileOptimiserTests.js

@@ -21,8 +21,8 @@ const path = require('path')
 const { expect } = require('chai')
 const { EventEmitter } = require('events')
 
-describe('OutputFileOptimiser', function() {
-  beforeEach(function() {
+describe('OutputFileOptimiser', function () {
+  beforeEach(function () {
     this.OutputFileOptimiser = SandboxedModule.require(modulePath, {
       requires: {
         fs: (this.fs = {}),
@@ -30,20 +30,21 @@ describe('OutputFileOptimiser', function() {
         child_process: { spawn: (this.spawn = sinon.stub()) },
         'logger-sharelatex': { log: sinon.stub(), warn: sinon.stub() },
         './Metrics': {}
-      }
+      },
+      globals: { Math } // used by lodash
     })
     this.directory = '/test/dir'
     return (this.callback = sinon.stub())
   })
 
-  describe('optimiseFile', function() {
-    beforeEach(function() {
+  describe('optimiseFile', function () {
+    beforeEach(function () {
       this.src = './output.pdf'
       return (this.dst = './output.pdf')
     })
 
-    describe('when the file is not a pdf file', function() {
-      beforeEach(function(done) {
+    describe('when the file is not a pdf file', function () {
+      beforeEach(function (done) {
         this.src = './output.log'
         this.OutputFileOptimiser.checkIfPDFIsOptimised = sinon
           .stub()
@@ -54,21 +55,21 @@ describe('OutputFileOptimiser', function() {
         return this.OutputFileOptimiser.optimiseFile(this.src, this.dst, done)
       })
 
-      it('should not check if the file is optimised', function() {
+      it('should not check if the file is optimised', function () {
         return this.OutputFileOptimiser.checkIfPDFIsOptimised
           .calledWith(this.src)
           .should.equal(false)
       })
 
-      return it('should not optimise the file', function() {
+      return it('should not optimise the file', function () {
         return this.OutputFileOptimiser.optimisePDF
           .calledWith(this.src, this.dst)
           .should.equal(false)
       })
     })
 
-    describe('when the pdf file is not optimised', function() {
-      beforeEach(function(done) {
+    describe('when the pdf file is not optimised', function () {
+      beforeEach(function (done) {
         this.OutputFileOptimiser.checkIfPDFIsOptimised = sinon
           .stub()
           .callsArgWith(1, null, false)
@@ -78,21 +79,21 @@ describe('OutputFileOptimiser', function() {
         return this.OutputFileOptimiser.optimiseFile(this.src, this.dst, done)
       })
 
-      it('should check if the pdf is optimised', function() {
+      it('should check if the pdf is optimised', function () {
         return this.OutputFileOptimiser.checkIfPDFIsOptimised
           .calledWith(this.src)
           .should.equal(true)
       })
 
-      return it('should optimise the pdf', function() {
+      return it('should optimise the pdf', function () {
         return this.OutputFileOptimiser.optimisePDF
           .calledWith(this.src, this.dst)
           .should.equal(true)
       })
     })
 
-    return describe('when the pdf file is optimised', function() {
-      beforeEach(function(done) {
+    return describe('when the pdf file is optimised', function () {
+      beforeEach(function (done) {
         this.OutputFileOptimiser.checkIfPDFIsOptimised = sinon
           .stub()
           .callsArgWith(1, null, true)
@@ -102,13 +103,13 @@ describe('OutputFileOptimiser', function() {
         return this.OutputFileOptimiser.optimiseFile(this.src, this.dst, done)
       })
 
-      it('should check if the pdf is optimised', function() {
+      it('should check if the pdf is optimised', function () {
         return this.OutputFileOptimiser.checkIfPDFIsOptimised
           .calledWith(this.src)
           .should.equal(true)
       })
 
-      return it('should not optimise the pdf', function() {
+      return it('should not optimise the pdf', function () {
         return this.OutputFileOptimiser.optimisePDF
           .calledWith(this.src, this.dst)
           .should.equal(false)
@@ -116,79 +117,76 @@ describe('OutputFileOptimiser', function() {
     })
   })
 
-  return describe('checkIfPDFISOptimised', function() {
-    beforeEach(function() {
+  return describe('checkIfPDFISOptimised', function () {
+    beforeEach(function () {
       this.callback = sinon.stub()
       this.fd = 1234
       this.fs.open = sinon.stub().yields(null, this.fd)
       this.fs.read = sinon
         .stub()
         .withArgs(this.fd)
-        .yields(null, 100, new Buffer('hello /Linearized 1'))
-      this.fs.close = sinon
-        .stub()
-        .withArgs(this.fd)
-        .yields(null)
+        .yields(null, 100, Buffer.from('hello /Linearized 1'))
+      this.fs.close = sinon.stub().withArgs(this.fd).yields(null)
       return this.OutputFileOptimiser.checkIfPDFIsOptimised(
         this.src,
         this.callback
       )
     })
 
-    describe('for a linearised file', function() {
-      beforeEach(function() {
+    describe('for a linearised file', function () {
+      beforeEach(function () {
         this.fs.read = sinon
           .stub()
           .withArgs(this.fd)
-          .yields(null, 100, new Buffer('hello /Linearized 1'))
+          .yields(null, 100, Buffer.from('hello /Linearized 1'))
         return this.OutputFileOptimiser.checkIfPDFIsOptimised(
           this.src,
           this.callback
         )
       })
 
-      it('should open the file', function() {
+      it('should open the file', function () {
         return this.fs.open.calledWith(this.src, 'r').should.equal(true)
       })
 
-      it('should read the header', function() {
+      it('should read the header', function () {
         return this.fs.read.calledWith(this.fd).should.equal(true)
       })
 
-      it('should close the file', function() {
+      it('should close the file', function () {
         return this.fs.close.calledWith(this.fd).should.equal(true)
       })
 
-      return it('should call the callback with a true result', function() {
+      return it('should call the callback with a true result', function () {
         return this.callback.calledWith(null, true).should.equal(true)
       })
     })
 
-    return describe('for an unlinearised file', function() {
-      beforeEach(function() {
+    return describe('for an unlinearised file', function () {
+      beforeEach(function () {
         this.fs.read = sinon
           .stub()
           .withArgs(this.fd)
-          .yields(null, 100, new Buffer('hello not linearized 1'))
+          .yields(null, 100, Buffer.from('hello not linearized 1'))
         return this.OutputFileOptimiser.checkIfPDFIsOptimised(
           this.src,
           this.callback
         )
       })
 
-      it('should open the file', function() {
+      it('should open the file', function () {
         return this.fs.open.calledWith(this.src, 'r').should.equal(true)
       })
 
-      it('should read the header', function() {
+      it('should read the header', function () {
         return this.fs.read.calledWith(this.fd).should.equal(true)
       })
 
-      it('should close the file', function() {
+      it('should close the file', function () {
         return this.fs.close.calledWith(this.fd).should.equal(true)
       })
 
-      return it('should call the callback with a false result', function() {
+      return it('should call the callback with a false result', function () {
         return this.callback.calledWith(null, false).should.equal(true)
       })
     })

test/unit/js/ProjectPersistenceManagerTests.js

@@ -14,19 +14,28 @@
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 require('chai').should()
+const assert = require('chai').assert
 const modulePath = require('path').join(
   __dirname,
   '../../../app/js/ProjectPersistenceManager'
 )
 const tk = require('timekeeper')
 
-describe('ProjectPersistenceManager', function() {
-  beforeEach(function() {
+describe('ProjectPersistenceManager', function () {
+  beforeEach(function () {
     this.ProjectPersistenceManager = SandboxedModule.require(modulePath, {
       requires: {
         './UrlCache': (this.UrlCache = {}),
         './CompileManager': (this.CompileManager = {}),
-        'logger-sharelatex': (this.logger = { log: sinon.stub() }),
+        diskusage: (this.diskusage = { check: sinon.stub() }),
+        'logger-sharelatex': (this.logger = {
+          log: sinon.stub(),
+          warn: sinon.stub(),
+          err: sinon.stub()
+        }),
+        'settings-sharelatex': (this.settings = {
+          project_cache_length_ms: 1000
+        }),
         './db': (this.db = {})
       }
     })
@@ -35,8 +44,59 @@ describe('ProjectPersistenceManager', function() {
     return (this.user_id = '1234')
   })
 
-  describe('clearExpiredProjects', function() {
-    beforeEach(function() {
+  describe('refreshExpiryTimeout', function () {
+    it('should leave expiry alone if plenty of disk', function (done) {
+      this.diskusage.check.callsArgWith(1, null, {
+        available: 40,
+        total: 100
+      })
+
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
+          this.settings.project_cache_length_ms
+        )
+        done()
+      })
+    })
+
+    it('should drop EXPIRY_TIMEOUT 10% if low disk usage', function (done) {
+      this.diskusage.check.callsArgWith(1, null, {
+        available: 5,
+        total: 100
+      })
+
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
+        done()
+      })
+    })
+
+    it('should not drop EXPIRY_TIMEOUT to below 50% of project_cache_length_ms', function (done) {
+      this.diskusage.check.callsArgWith(1, null, {
+        available: 5,
+        total: 100
+      })
+      this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
+        done()
+      })
+    })
+
+    it('should not modify EXPIRY_TIMEOUT if there is an error getting disk values', function (done) {
+      this.diskusage.check.callsArgWith(1, 'Error', {
+        available: 5,
+        total: 100
+      })
+      this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
+        this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(1000)
+        done()
+      })
+    })
+  })
+
+  describe('clearExpiredProjects', function () {
+    beforeEach(function () {
       this.project_ids = ['project-id-1', 'project-id-2']
       this.ProjectPersistenceManager._findExpiredProjectIds = sinon
         .stub()
@@ -48,21 +108,21 @@ describe('ProjectPersistenceManager', function() {
       return this.ProjectPersistenceManager.clearExpiredProjects(this.callback)
     })
 
-    it('should clear each expired project', function() {
-      return Array.from(this.project_ids).map(project_id =>
+    it('should clear each expired project', function () {
+      return Array.from(this.project_ids).map((project_id) =>
         this.ProjectPersistenceManager.clearProjectFromCache
           .calledWith(project_id)
           .should.equal(true)
       )
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  return describe('clearProject', function() {
-    beforeEach(function() {
+  return describe('clearProject', function () {
+    beforeEach(function () {
       this.ProjectPersistenceManager._clearProjectFromDatabase = sinon
         .stub()
         .callsArg(1)
@@ -75,25 +135,25 @@ describe('ProjectPersistenceManager', function() {
       )
     })
 
-    it('should clear the project from the database', function() {
+    it('should clear the project from the database', function () {
       return this.ProjectPersistenceManager._clearProjectFromDatabase
         .calledWith(this.project_id)
         .should.equal(true)
     })
 
-    it('should clear all the cached Urls for the project', function() {
+    it('should clear all the cached Urls for the project', function () {
       return this.UrlCache.clearProject
         .calledWith(this.project_id)
         .should.equal(true)
     })
 
-    it('should clear the project compile folder', function() {
+    it('should clear the project compile folder', function () {
       return this.CompileManager.clearProject
         .calledWith(this.project_id, this.user_id)
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })

test/unit/js/RequestParserTests.js

@@ -19,8 +19,8 @@ const modulePath = require('path').join(
 )
 const tk = require('timekeeper')
 
-describe('RequestParser', function() {
-  beforeEach(function() {
+describe('RequestParser', function () {
+  beforeEach(function () {
     tk.freeze()
     this.callback = sinon.stub()
     this.validResource = {
@@ -46,41 +46,41 @@ describe('RequestParser', function() {
     }))
   })
 
-  afterEach(function() {
+  afterEach(function () {
     return tk.reset()
   })
 
-  describe('without a top level object', function() {
-    beforeEach(function() {
+  describe('without a top level object', function () {
+    beforeEach(function () {
       return this.RequestParser.parse([], this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('top level object should have a compile attribute')
         .should.equal(true)
     })
   })
 
-  describe('without a compile attribute', function() {
-    beforeEach(function() {
+  describe('without a compile attribute', function () {
+    beforeEach(function () {
       return this.RequestParser.parse({}, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('top level object should have a compile attribute')
         .should.equal(true)
     })
   })
 
-  describe('without a valid compiler', function() {
-    beforeEach(function() {
+  describe('without a valid compiler', function () {
+    beforeEach(function () {
       this.validRequest.compile.options.compiler = 'not-a-compiler'
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith(
           'compiler attribute should be one of: pdflatex, latex, xelatex, lualatex'
@@ -89,73 +89,115 @@ describe('RequestParser', function() {
     })
   })
 
-  describe('without a compiler specified', function() {
-    beforeEach(function() {
+  describe('without a compiler specified', function () {
+    beforeEach(function () {
       delete this.validRequest.compile.options.compiler
       return this.RequestParser.parse(this.validRequest, (error, data) => {
         this.data = data
       })
     })
 
-    return it('should set the compiler to pdflatex by default', function() {
+    return it('should set the compiler to pdflatex by default', function () {
       return this.data.compiler.should.equal('pdflatex')
     })
   })
 
-  describe('with imageName set', function() {
-    beforeEach(function() {
+  describe('with imageName set', function () {
+    beforeEach(function () {
       return this.RequestParser.parse(this.validRequest, (error, data) => {
         this.data = data
       })
     })
 
-    return it('should set the imageName', function() {
+    return it('should set the imageName', function () {
       return this.data.imageName.should.equal('basicImageName/here:2017-1')
     })
   })
 
-  describe('with flags set', function() {
-    beforeEach(function() {
+  describe('when image restrictions are present', function () {
+    beforeEach(function () {
+      this.settings.clsi = { docker: {} }
+      this.settings.clsi.docker.allowedImages = [
+        'repo/name:tag1',
+        'repo/name:tag2'
+      ]
+    })
+
+    describe('with imageName set to something invalid', function () {
+      beforeEach(function () {
+        const request = this.validRequest
+        request.compile.options.imageName = 'something/different:latest'
+        this.RequestParser.parse(request, (error, data) => {
+          this.error = error
+          this.data = data
+        })
+      })
+
+      it('should throw an error for imageName', function () {
+        expect(String(this.error)).to.include(
+          'imageName attribute should be one of'
+        )
+      })
+    })
+
+    describe('with imageName set to something valid', function () {
+      beforeEach(function () {
+        const request = this.validRequest
+        request.compile.options.imageName = 'repo/name:tag1'
+        this.RequestParser.parse(request, (error, data) => {
+          this.error = error
+          this.data = data
+        })
+      })
+
+      it('should set the imageName', function () {
+        this.data.imageName.should.equal('repo/name:tag1')
+      })
+    })
+  })
+
+  describe('with flags set', function () {
+    beforeEach(function () {
       this.validRequest.compile.options.flags = ['-file-line-error']
       return this.RequestParser.parse(this.validRequest, (error, data) => {
         this.data = data
       })
     })
 
-    return it('should set the flags attribute', function() {
+    return it('should set the flags attribute', function () {
       return expect(this.data.flags).to.deep.equal(['-file-line-error'])
     })
   })
 
-  describe('with flags not specified', function() {
-    beforeEach(function() {
+  describe('with flags not specified', function () {
+    beforeEach(function () {
       return this.RequestParser.parse(this.validRequest, (error, data) => {
         this.data = data
       })
     })
 
-    return it('it should have an empty flags list', function() {
+    return it('it should have an empty flags list', function () {
       return expect(this.data.flags).to.deep.equal([])
     })
   })
 
-  describe('without a timeout specified', function() {
-    beforeEach(function() {
+  describe('without a timeout specified', function () {
+    beforeEach(function () {
       delete this.validRequest.compile.options.timeout
       return this.RequestParser.parse(this.validRequest, (error, data) => {
         this.data = data
       })
     })
 
-    return it('should set the timeout to MAX_TIMEOUT', function() {
+    return it('should set the timeout to MAX_TIMEOUT', function () {
       return this.data.timeout.should.equal(
         this.RequestParser.MAX_TIMEOUT * 1000
       )
     })
   })
 
-  describe('with a timeout larger than the maximum', function() {
-    beforeEach(function() {
+  describe('with a timeout larger than the maximum', function () {
+    beforeEach(function () {
       this.validRequest.compile.options.timeout =
         this.RequestParser.MAX_TIMEOUT + 1
       return this.RequestParser.parse(this.validRequest, (error, data) => {
@@ -163,62 +205,62 @@ describe('RequestParser', function() {
       })
     })
 
-    return it('should set the timeout to MAX_TIMEOUT', function() {
+    return it('should set the timeout to MAX_TIMEOUT', function () {
       return this.data.timeout.should.equal(
         this.RequestParser.MAX_TIMEOUT * 1000
       )
     })
   })
 
-  describe('with a timeout', function() {
-    beforeEach(function() {
+  describe('with a timeout', function () {
+    beforeEach(function () {
       return this.RequestParser.parse(this.validRequest, (error, data) => {
         this.data = data
       })
     })
 
-    return it('should set the timeout (in milliseconds)', function() {
+    return it('should set the timeout (in milliseconds)', function () {
       return this.data.timeout.should.equal(
         this.validRequest.compile.options.timeout * 1000
       )
     })
   })
 
-  describe('with a resource without a path', function() {
-    beforeEach(function() {
+  describe('with a resource without a path', function () {
+    beforeEach(function () {
       delete this.validResource.path
       this.validRequest.compile.resources.push(this.validResource)
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('all resources should have a path attribute')
         .should.equal(true)
     })
   })
 
-  describe('with a resource with a path', function() {
-    beforeEach(function() {
+  describe('with a resource with a path', function () {
+    beforeEach(function () {
       this.validResource.path = this.path = 'test.tex'
       this.validRequest.compile.resources.push(this.validResource)
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return the path in the parsed response', function() {
+    return it('should return the path in the parsed response', function () {
       return this.data.resources[0].path.should.equal(this.path)
     })
   })
 
-  describe('with a resource with a malformed modified date', function() {
-    beforeEach(function() {
+  describe('with a resource with a malformed modified date', function () {
+    beforeEach(function () {
       this.validResource.modified = 'not-a-date'
       this.validRequest.compile.resources.push(this.validResource)
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith(
           'resource modified date could not be understood: ' +
@@ -228,8 +270,8 @@ describe('RequestParser', function() {
     })
   })
 
-  describe('with a resource with a valid date', function() {
-    beforeEach(function() {
+  describe('with a resource with a valid date', function () {
+    beforeEach(function () {
       this.date = '12:00 01/02/03'
       this.validResource.modified = this.date
       this.validRequest.compile.resources.push(this.validResource)
@@ -237,7 +279,7 @@ describe('RequestParser', function() {
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return the date as a Javascript Date object', function() {
+    return it('should return the date as a Javascript Date object', function () {
       ;(this.data.resources[0].modified instanceof Date).should.equal(true)
       return this.data.resources[0].modified
         .getTime()
@@ -245,15 +287,15 @@ describe('RequestParser', function() {
     })
   })
 
-  describe('with a resource without either a content or URL attribute', function() {
-    beforeEach(function() {
+  describe('with a resource without either a content or URL attribute', function () {
+    beforeEach(function () {
       delete this.validResource.url
       delete this.validResource.content
       this.validRequest.compile.resources.push(this.validResource)
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith(
           'all resources should have either a url or content attribute'
@@ -262,99 +304,99 @@ describe('RequestParser', function() {
     })
   })
 
-  describe('with a resource where the content is not a string', function() {
-    beforeEach(function() {
+  describe('with a resource where the content is not a string', function () {
+    beforeEach(function () {
       this.validResource.content = []
       this.validRequest.compile.resources.push(this.validResource)
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('content attribute should be a string')
         .should.equal(true)
     })
   })
 
-  describe('with a resource where the url is not a string', function() {
-    beforeEach(function() {
+  describe('with a resource where the url is not a string', function () {
+    beforeEach(function () {
       this.validResource.url = []
       this.validRequest.compile.resources.push(this.validResource)
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('url attribute should be a string')
         .should.equal(true)
     })
   })
 
-  describe('with a resource with a url', function() {
-    beforeEach(function() {
+  describe('with a resource with a url', function () {
+    beforeEach(function () {
       this.validResource.url = this.url = 'www.example.com'
       this.validRequest.compile.resources.push(this.validResource)
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return the url in the parsed response', function() {
+    return it('should return the url in the parsed response', function () {
       return this.data.resources[0].url.should.equal(this.url)
     })
   })
 
-  describe('with a resource with a content attribute', function() {
-    beforeEach(function() {
+  describe('with a resource with a content attribute', function () {
+    beforeEach(function () {
       this.validResource.content = this.content = 'Hello world'
       this.validRequest.compile.resources.push(this.validResource)
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return the content in the parsed response', function() {
+    return it('should return the content in the parsed response', function () {
       return this.data.resources[0].content.should.equal(this.content)
     })
   })
 
-  describe('without a root resource path', function() {
-    beforeEach(function() {
+  describe('without a root resource path', function () {
+    beforeEach(function () {
       delete this.validRequest.compile.rootResourcePath
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it("should set the root resource path to 'main.tex' by default", function() {
+    return it("should set the root resource path to 'main.tex' by default", function () {
       return this.data.rootResourcePath.should.equal('main.tex')
     })
   })
 
-  describe('with a root resource path', function() {
-    beforeEach(function() {
+  describe('with a root resource path', function () {
+    beforeEach(function () {
       this.validRequest.compile.rootResourcePath = this.path = 'test.tex'
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return the root resource path in the parsed response', function() {
+    return it('should return the root resource path in the parsed response', function () {
       return this.data.rootResourcePath.should.equal(this.path)
     })
   })
 
-  describe('with a root resource path that is not a string', function() {
-    beforeEach(function() {
+  describe('with a root resource path that is not a string', function () {
+    beforeEach(function () {
       this.validRequest.compile.rootResourcePath = []
       return this.RequestParser.parse(this.validRequest, this.callback)
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('rootResourcePath attribute should be a string')
         .should.equal(true)
     })
   })
 
-  describe('with a root resource path that needs escaping', function() {
-    beforeEach(function() {
+  describe('with a root resource path that needs escaping', function () {
+    beforeEach(function () {
       this.badPath = '`rm -rf foo`.tex'
       this.goodPath = 'rm -rf foo.tex'
       this.validRequest.compile.rootResourcePath = this.badPath
@@ -367,51 +409,51 @@ describe('RequestParser', function() {
       return (this.data = this.callback.args[0][1])
     })
 
-    it('should return the escaped resource', function() {
+    it('should return the escaped resource', function () {
       return this.data.rootResourcePath.should.equal(this.goodPath)
     })
 
-    return it('should also escape the resource path', function() {
+    return it('should also escape the resource path', function () {
       return this.data.resources[0].path.should.equal(this.goodPath)
     })
   })
 
-  describe('with a root resource path that has a relative path', function() {
-    beforeEach(function() {
+  describe('with a root resource path that has a relative path', function () {
+    beforeEach(function () {
       this.validRequest.compile.rootResourcePath = 'foo/../../bar.tex'
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('relative path in root resource')
         .should.equal(true)
     })
   })
 
-  describe('with a root resource path that has unescaped + relative path', function() {
-    beforeEach(function() {
+  describe('with a root resource path that has unescaped + relative path', function () {
+    beforeEach(function () {
       this.validRequest.compile.rootResourcePath = 'foo/#../bar.tex'
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('relative path in root resource')
         .should.equal(true)
     })
   })
 
-  return describe('with an unknown syncType', function() {
-    beforeEach(function() {
+  return describe('with an unknown syncType', function () {
+    beforeEach(function () {
       this.validRequest.compile.options.syncType = 'unexpected'
       this.RequestParser.parse(this.validRequest, this.callback)
       return (this.data = this.callback.args[0][1])
     })
 
-    return it('should return an error', function() {
+    return it('should return an error', function () {
       return this.callback
         .calledWith('syncType attribute should be one of: full, incremental')
         .should.equal(true)

test/unit/js/ResourceStateManagerTests.js

@@ -20,8 +20,8 @@ const modulePath = require('path').join(
 const Path = require('path')
 const Errors = require('../../../app/js/Errors')
 
-describe('ResourceStateManager', function() {
-  beforeEach(function() {
+describe('ResourceStateManager', function () {
+  beforeEach(function () {
     this.ResourceStateManager = SandboxedModule.require(modulePath, {
       singleOnly: true,
       requires: {
@@ -42,13 +42,13 @@ describe('ResourceStateManager', function() {
     return (this.callback = sinon.stub())
   })
 
-  describe('saveProjectState', function() {
-    beforeEach(function() {
+  describe('saveProjectState', function () {
+    beforeEach(function () {
       return (this.fs.writeFile = sinon.stub().callsArg(2))
     })
 
-    describe('when the state is specified', function() {
-      beforeEach(function() {
+    describe('when the state is specified', function () {
+      beforeEach(function () {
         return this.ResourceStateManager.saveProjectState(
           this.state,
           this.resources,
@@ -57,19 +57,19 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      it('should write the resource list to disk', function() {
+      it('should write the resource list to disk', function () {
         return this.fs.writeFile
           .calledWith(this.resourceFileName, this.resourceFileContents)
           .should.equal(true)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.called.should.equal(true)
       })
     })
 
-    return describe('when the state is undefined', function() {
-      beforeEach(function() {
+    return describe('when the state is undefined', function () {
+      beforeEach(function () {
         this.state = undefined
         this.fs.unlink = sinon.stub().callsArg(1)
         return this.ResourceStateManager.saveProjectState(
@@ -80,25 +80,25 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      it('should unlink the resource file', function() {
+      it('should unlink the resource file', function () {
         return this.fs.unlink
           .calledWith(this.resourceFileName)
           .should.equal(true)
       })
 
-      it('should not write the resource list to disk', function() {
+      it('should not write the resource list to disk', function () {
         return this.fs.writeFile.called.should.equal(false)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.called.should.equal(true)
       })
     })
   })
 
-  describe('checkProjectStateMatches', function() {
-    describe('when the state matches', function() {
-      beforeEach(function() {
+  describe('checkProjectStateMatches', function () {
+    describe('when the state matches', function () {
+      beforeEach(function () {
         this.SafeReader.readFile = sinon
           .stub()
           .callsArgWith(3, null, this.resourceFileContents)
@@ -109,21 +109,21 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      it('should read the resource file', function() {
+      it('should read the resource file', function () {
         return this.SafeReader.readFile
           .calledWith(this.resourceFileName)
           .should.equal(true)
       })
 
-      return it('should call the callback with the results', function() {
+      return it('should call the callback with the results', function () {
         return this.callback
           .calledWithMatch(null, this.resources)
           .should.equal(true)
       })
     })
 
-    return describe('when the state does not match', function() {
-      beforeEach(function() {
+    return describe('when the state does not match', function () {
+      beforeEach(function () {
         this.SafeReader.readFile = sinon
           .stub()
           .callsArgWith(3, null, this.resourceFileContents)
@@ -134,7 +134,7 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      it('should call the callback with an error', function() {
+      it('should call the callback with an error', function () {
         this.callback
           .calledWith(sinon.match(Errors.FilesOutOfSyncError))
           .should.equal(true)
@@ -145,9 +145,9 @@ describe('ResourceStateManager', function() {
     })
   })
 
-  return describe('checkResourceFiles', function() {
-    describe('when all the files are present', function() {
-      beforeEach(function() {
+  return describe('checkResourceFiles', function () {
+    describe('when all the files are present', function () {
+      beforeEach(function () {
         this.allFiles = [
           this.resources[0].path,
           this.resources[1].path,
@@ -161,13 +161,13 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.calledWithExactly().should.equal(true)
       })
     })
 
-    describe('when there is a missing file', function() {
-      beforeEach(function() {
+    describe('when there is a missing file', function () {
+      beforeEach(function () {
         this.allFiles = [this.resources[0].path, this.resources[1].path]
         this.fs.stat = sinon.stub().callsArgWith(1, new Error())
         return this.ResourceStateManager.checkResourceFiles(
@@ -178,7 +178,7 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      it('should call the callback with an error', function() {
+      it('should call the callback with an error', function () {
         this.callback
           .calledWith(sinon.match(Errors.FilesOutOfSyncError))
           .should.equal(true)
@@ -190,8 +190,8 @@ describe('ResourceStateManager', function() {
       })
     })
 
-    return describe('when a resource contains a relative path', function() {
-      beforeEach(function() {
+    return describe('when a resource contains a relative path', function () {
+      beforeEach(function () {
         this.resources[0].path = '../foo/bar.tex'
         this.allFiles = [
           this.resources[0].path,
@@ -206,7 +206,7 @@ describe('ResourceStateManager', function() {
         )
       })
 
-      it('should call the callback with an error', function() {
+      it('should call the callback with an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
 
         const message = this.callback.args[0][0].message

test/unit/js/ResourceWriterTests.js

@@ -20,8 +20,8 @@ const modulePath = require('path').join(
 )
 const path = require('path')
 
-describe('ResourceWriter', function() {
-  beforeEach(function() {
+describe('ResourceWriter', function () {
+  beforeEach(function () {
     let Timer
     this.ResourceWriter = SandboxedModule.require(modulePath, {
       singleOnly: true,
@@ -37,7 +37,7 @@ describe('ResourceWriter', function() {
         'logger-sharelatex': { log: sinon.stub(), err: sinon.stub() },
         './Metrics': (this.Metrics = {
           inc: sinon.stub(),
-          Timer: (Timer = (function() {
+          Timer: (Timer = (function () {
             Timer = class Timer {
               static initClass() {
                 this.prototype.done = sinon.stub()
@@ -54,8 +54,8 @@ describe('ResourceWriter', function() {
     return (this.callback = sinon.stub())
   })
 
-  describe('syncResourcesToDisk on a full request', function() {
-    beforeEach(function() {
+  describe('syncResourcesToDisk on a full request', function () {
+    beforeEach(function () {
       this.resources = ['resource-1-mock', 'resource-2-mock', 'resource-3-mock']
       this.ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
       this.ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
@@ -71,33 +71,33 @@ describe('ResourceWriter', function() {
       )
     })
 
-    it('should remove old files', function() {
+    it('should remove old files', function () {
       return this.ResourceWriter._removeExtraneousFiles
         .calledWith(this.resources, this.basePath)
         .should.equal(true)
     })
 
-    it('should write each resource to disk', function() {
-      return Array.from(this.resources).map(resource =>
+    it('should write each resource to disk', function () {
+      return Array.from(this.resources).map((resource) =>
         this.ResourceWriter._writeResourceToDisk
           .calledWith(this.project_id, resource, this.basePath)
           .should.equal(true)
       )
     })
 
-    it('should store the sync state and resource list', function() {
+    it('should store the sync state and resource list', function () {
       return this.ResourceStateManager.saveProjectState
         .calledWith(this.syncState, this.resources, this.basePath)
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  describe('syncResourcesToDisk on an incremental update', function() {
-    beforeEach(function() {
+  describe('syncResourcesToDisk on an incremental update', function () {
+    beforeEach(function () {
       this.resources = ['resource-1-mock']
       this.ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
       this.ResourceWriter._removeExtraneousFiles = sinon
@@ -120,39 +120,39 @@ describe('ResourceWriter', function() {
       )
     })
 
-    it('should check the sync state matches', function() {
+    it('should check the sync state matches', function () {
       return this.ResourceStateManager.checkProjectStateMatches
         .calledWith(this.syncState, this.basePath)
         .should.equal(true)
     })
 
-    it('should remove old files', function() {
+    it('should remove old files', function () {
       return this.ResourceWriter._removeExtraneousFiles
         .calledWith(this.resources, this.basePath)
         .should.equal(true)
     })
 
-    it('should check each resource exists', function() {
+    it('should check each resource exists', function () {
       return this.ResourceStateManager.checkResourceFiles
         .calledWith(this.resources, this.allFiles, this.basePath)
         .should.equal(true)
     })
 
-    it('should write each resource to disk', function() {
-      return Array.from(this.resources).map(resource =>
+    it('should write each resource to disk', function () {
+      return Array.from(this.resources).map((resource) =>
         this.ResourceWriter._writeResourceToDisk
           .calledWith(this.project_id, resource, this.basePath)
           .should.equal(true)
       )
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  describe('syncResourcesToDisk on an incremental update when the state does not match', function() {
-    beforeEach(function() {
+  describe('syncResourcesToDisk on an incremental update when the state does not match', function () {
+    beforeEach(function () {
       this.resources = ['resource-1-mock']
       this.ResourceStateManager.checkProjectStateMatches = sinon
         .stub()
@@ -169,19 +169,19 @@ describe('ResourceWriter', function() {
       )
     })
 
-    it('should check whether the sync state matches', function() {
+    it('should check whether the sync state matches', function () {
       return this.ResourceStateManager.checkProjectStateMatches
         .calledWith(this.syncState, this.basePath)
         .should.equal(true)
     })
 
-    return it('should call the callback with an error', function() {
+    return it('should call the callback with an error', function () {
       return this.callback.calledWith(this.error).should.equal(true)
     })
   })
 
-  describe('_removeExtraneousFiles', function() {
-    beforeEach(function() {
+  describe('_removeExtraneousFiles', function () {
+    beforeEach(function () {
       this.output_files = [
         {
           path: 'output.pdf',
@@ -230,6 +230,12 @@ describe('ResourceWriter', function() {
         {
           path: '_markdown_main/30893013dec5d869a415610079774c2f.md.tex',
           type: 'tex'
+        },
+        {
+          path: 'output.stdout'
+        },
+        {
+          path: 'output.stderr'
         }
       ]
       this.resources = 'mock-resources'
@@ -244,37 +250,49 @@ describe('ResourceWriter', function() {
       )
     })
 
-    it('should find the existing output files', function() {
+    it('should find the existing output files', function () {
       return this.OutputFileFinder.findOutputFiles
         .calledWith(this.resources, this.basePath)
         .should.equal(true)
     })
 
-    it('should delete the output files', function() {
+    it('should delete the output files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'output.pdf'))
         .should.equal(true)
     })
 
-    it('should delete the extra files', function() {
+    it('should delete the stdout log file', function () {
+      return this.ResourceWriter._deleteFileIfNotDirectory
+        .calledWith(path.join(this.basePath, 'output.stdout'))
+        .should.equal(true)
+    })
+
+    it('should delete the stderr log file', function () {
+      return this.ResourceWriter._deleteFileIfNotDirectory
+        .calledWith(path.join(this.basePath, 'output.stderr'))
+        .should.equal(true)
+    })
+
+    it('should delete the extra files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'extra/file.tex'))
         .should.equal(true)
     })
 
-    it('should not delete the extra aux files', function() {
+    it('should not delete the extra aux files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'extra.aux'))
         .should.equal(false)
     })
 
-    it('should not delete the knitr cache file', function() {
+    it('should not delete the knitr cache file', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'cache/_chunk1'))
         .should.equal(false)
     })
 
-    it('should not delete the epstopdf converted files', function() {
+    it('should not delete the epstopdf converted files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(
           path.join(this.basePath, 'figures/image-eps-converted-to.pdf')
@@ -282,25 +300,25 @@ describe('ResourceWriter', function() {
         .should.equal(false)
     })
 
-    it('should not delete the tikz md5 files', function() {
+    it('should not delete the tikz md5 files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'foo/main-figure0.md5'))
         .should.equal(false)
     })
 
-    it('should not delete the tikz dpth files', function() {
+    it('should not delete the tikz dpth files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'foo/main-figure0.dpth'))
         .should.equal(false)
     })
 
-    it('should not delete the tikz pdf files', function() {
+    it('should not delete the tikz pdf files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, 'foo/main-figure0.pdf'))
         .should.equal(false)
     })
 
-    it('should not delete the minted pygstyle files', function() {
+    it('should not delete the minted pygstyle files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(
           path.join(this.basePath, '_minted-main/default-pyg-prefix.pygstyle')
@@ -308,13 +326,13 @@ describe('ResourceWriter', function() {
         .should.equal(false)
     })
 
-    it('should not delete the minted default pygstyle files', function() {
+    it('should not delete the minted default pygstyle files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(this.basePath, '_minted-main/default.pygstyle'))
         .should.equal(false)
     })
 
-    it('should not delete the minted default pygtex files', function() {
+    it('should not delete the minted default pygtex files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(
           path.join(
@@ -325,7 +343,7 @@ describe('ResourceWriter', function() {
         .should.equal(false)
     })
 
-    it('should not delete the markdown md.tex files', function() {
+    it('should not delete the markdown md.tex files', function () {
       return this.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(
           path.join(
@@ -336,18 +354,18 @@ describe('ResourceWriter', function() {
         .should.equal(false)
     })
 
-    it('should call the callback', function() {
+    it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
 
-    return it('should time the request', function() {
+    return it('should time the request', function () {
       return this.Metrics.Timer.prototype.done.called.should.equal(true)
     })
   })
 
-  describe('_writeResourceToDisk', function() {
-    describe('with a url based resource', function() {
-      beforeEach(function() {
+  describe('_writeResourceToDisk', function () {
+    describe('with a url based resource', function () {
+      beforeEach(function () {
         this.fs.mkdir = sinon.stub().callsArg(2)
         this.resource = {
           path: 'main.tex',
@@ -365,7 +383,7 @@ describe('ResourceWriter', function() {
         )
       })
 
-      it('should ensure the directory exists', function() {
+      it('should ensure the directory exists', function () {
         this.fs.mkdir
           .calledWith(
             path.dirname(path.join(this.basePath, this.resource.path))
@@ -373,7 +391,7 @@ describe('ResourceWriter', function() {
           .should.equal(true)
       })
 
-      it('should write the URL from the cache', function() {
+      it('should write the URL from the cache', function () {
         return this.UrlCache.downloadUrlToFile
           .calledWith(
             this.project_id,
@@ -384,17 +402,17 @@ describe('ResourceWriter', function() {
           .should.equal(true)
       })
 
-      it('should call the callback', function() {
+      it('should call the callback', function () {
         return this.callback.called.should.equal(true)
       })
 
-      return it('should not return an error if the resource writer errored', function() {
+      return it('should not return an error if the resource writer errored', function () {
         return should.not.exist(this.callback.args[0][0])
       })
     })
 
-    describe('with a content based resource', function() {
-      beforeEach(function() {
+    describe('with a content based resource', function () {
+      beforeEach(function () {
         this.resource = {
           path: 'main.tex',
           content: 'Hello world'
@@ -409,7 +427,7 @@ describe('ResourceWriter', function() {
         )
       })
 
-      it('should ensure the directory exists', function() {
+      it('should ensure the directory exists', function () {
         return this.fs.mkdir
           .calledWith(
             path.dirname(path.join(this.basePath, this.resource.path))
@@ -417,7 +435,7 @@ describe('ResourceWriter', function() {
           .should.equal(true)
       })
 
-      it('should write the contents to disk', function() {
+      it('should write the contents to disk', function () {
         return this.fs.writeFile
           .calledWith(
             path.join(this.basePath, this.resource.path),
@@ -426,13 +444,13 @@ describe('ResourceWriter', function() {
           .should.equal(true)
       })
 
-      return it('should call the callback', function() {
+      return it('should call the callback', function () {
         return this.callback.called.should.equal(true)
       })
     })
 
-    return describe('with a file path that breaks out of the root folder', function() {
-      beforeEach(function() {
+    return describe('with a file path that breaks out of the root folder', function () {
+      beforeEach(function () {
         this.resource = {
           path: '../../main.tex',
           content: 'Hello world'
@@ -446,11 +464,11 @@ describe('ResourceWriter', function() {
         )
       })
 
-      it('should not write to disk', function() {
+      it('should not write to disk', function () {
         return this.fs.writeFile.called.should.equal(false)
       })
 
-      it('should return an error', function() {
+      it('should return an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
 
         const message = this.callback.args[0][0].message
@@ -459,23 +477,23 @@ describe('ResourceWriter', function() {
     })
   })
 
-  return describe('checkPath', function() {
-    describe('with a valid path', function() {
-      beforeEach(function() {
+  return describe('checkPath', function () {
+    describe('with a valid path', function () {
+      beforeEach(function () {
         return this.ResourceWriter.checkPath('foo', 'bar', this.callback)
       })
 
-      return it('should return the joined path', function() {
+      return it('should return the joined path', function () {
         return this.callback.calledWith(null, 'foo/bar').should.equal(true)
       })
     })
 
-    describe('with an invalid path', function() {
-      beforeEach(function() {
+    describe('with an invalid path', function () {
+      beforeEach(function () {
         this.ResourceWriter.checkPath('foo', 'baz/../../bar', this.callback)
       })
 
-      it('should return an error', function() {
+      it('should return an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
 
         const message = this.callback.args[0][0].message
@@ -483,8 +501,8 @@ describe('ResourceWriter', function() {
       })
     })
 
-    describe('with another invalid path matching on a prefix', function() {
-      beforeEach(function() {
+    describe('with another invalid path matching on a prefix', function () {
+      beforeEach(function () {
         return this.ResourceWriter.checkPath(
           'foo',
           '../foobar/baz',
@@ -492,7 +510,7 @@ describe('ResourceWriter', function() {
         )
       })
 
-      it('should return an error', function() {
+      it('should return an error', function () {
         this.callback.calledWith(sinon.match(Error)).should.equal(true)
 
         const message = this.callback.args[0][0].message

test/unit/js/StaticServerForbidSymlinksTests.js

@@ -20,8 +20,8 @@ const modulePath = path.join(
 )
 const { expect } = require('chai')
 
-describe('StaticServerForbidSymlinks', function() {
-  beforeEach(function() {
+describe('StaticServerForbidSymlinks', function () {
+  beforeEach(function () {
     this.settings = {
       path: {
         compilesDir: '/compiles/here'
@@ -60,8 +60,8 @@ describe('StaticServerForbidSymlinks', function() {
     return (this.req.url = '/12345/output.pdf')
   })
 
-  describe('sending a normal file through', function() {
-    beforeEach(function() {
+  describe('sending a normal file through', function () {
+    beforeEach(function () {
       return (this.fs.realpath = sinon
         .stub()
         .callsArgWith(
@@ -71,8 +71,8 @@ describe('StaticServerForbidSymlinks', function() {
         ))
     })
 
-    return it('should call next', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should call next', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(200)
         return done()
       }
@@ -80,8 +80,8 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a missing file', function() {
-    beforeEach(function() {
+  describe('with a missing file', function () {
+    beforeEach(function () {
       return (this.fs.realpath = sinon
         .stub()
         .callsArgWith(
@@ -91,8 +91,8 @@ describe('StaticServerForbidSymlinks', function() {
         ))
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -100,15 +100,15 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a symlink file', function() {
-    beforeEach(function() {
+  describe('with a symlink file', function () {
+    beforeEach(function () {
       return (this.fs.realpath = sinon
         .stub()
         .callsArgWith(1, null, `/etc/${this.req.params.project_id}/output.pdf`))
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -116,13 +116,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a relative file', function() {
-    beforeEach(function() {
+  describe('with a relative file', function () {
+    beforeEach(function () {
       return (this.req.url = '/12345/../67890/output.pdf')
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -130,13 +130,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a unnormalized file containing .', function() {
-    beforeEach(function() {
+  describe('with a unnormalized file containing .', function () {
+    beforeEach(function () {
       return (this.req.url = '/12345/foo/./output.pdf')
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -144,13 +144,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a file containing an empty path', function() {
-    beforeEach(function() {
+  describe('with a file containing an empty path', function () {
+    beforeEach(function () {
       return (this.req.url = '/12345/foo//output.pdf')
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -158,13 +158,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a non-project file', function() {
-    beforeEach(function() {
+  describe('with a non-project file', function () {
+    beforeEach(function () {
       return (this.req.url = '/.foo/output.pdf')
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -172,13 +172,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a file outside the compiledir', function() {
-    beforeEach(function() {
+  describe('with a file outside the compiledir', function () {
+    beforeEach(function () {
       return (this.req.url = '/../bar/output.pdf')
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -186,13 +186,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a file with no leading /', function() {
-    beforeEach(function() {
+  describe('with a file with no leading /', function () {
+    beforeEach(function () {
       return (this.req.url = './../bar/output.pdf')
     })
 
-    return it('should send a 404', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 404', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(404)
         return done()
       }
@@ -200,8 +200,8 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  describe('with a github style path', function() {
-    beforeEach(function() {
+  describe('with a github style path', function () {
+    beforeEach(function () {
       this.req.url = '/henryoswald-latex_example/output/output.log'
       return (this.fs.realpath = sinon
         .stub()
@@ -212,8 +212,8 @@ describe('StaticServerForbidSymlinks', function() {
         ))
     })
 
-    return it('should call next', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should call next', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(200)
         return done()
       }
@@ -221,13 +221,13 @@ describe('StaticServerForbidSymlinks', function() {
     })
   })
 
-  return describe('with an error from fs.realpath', function() {
-    beforeEach(function() {
+  return describe('with an error from fs.realpath', function () {
+    beforeEach(function () {
       return (this.fs.realpath = sinon.stub().callsArgWith(1, 'error'))
     })
 
-    return it('should send a 500', function(done) {
-      this.res.sendStatus = function(resCode) {
+    return it('should send a 500', function (done) {
+      this.res.sendStatus = function (resCode) {
         resCode.should.equal(500)
         return done()
       }

test/unit/js/TikzManager.js

@@ -16,8 +16,8 @@ const modulePath = require('path').join(
   '../../../app/js/TikzManager'
 )
 
-describe('TikzManager', function() {
-  beforeEach(function() {
+describe('TikzManager', function () {
+  beforeEach(function () {
     return (this.TikzManager = SandboxedModule.require(modulePath, {
       requires: {
         './ResourceWriter': (this.ResourceWriter = {}),
@@ -28,15 +28,15 @@ describe('TikzManager', function() {
     }))
   })
 
-  describe('checkMainFile', function() {
-    beforeEach(function() {
+  describe('checkMainFile', function () {
+    beforeEach(function () {
       this.compileDir = 'compile-dir'
       this.mainFile = 'main.tex'
       return (this.callback = sinon.stub())
     })
 
-    describe('if there is already an output.tex file in the resources', function() {
-      beforeEach(function() {
+    describe('if there is already an output.tex file in the resources', function () {
+      beforeEach(function () {
         this.resources = [{ path: 'main.tex' }, { path: 'output.tex' }]
         return this.TikzManager.checkMainFile(
           this.compileDir,
@@ -46,13 +46,13 @@ describe('TikzManager', function() {
         )
       })
 
-      return it('should call the callback with false ', function() {
+      return it('should call the callback with false ', function () {
         return this.callback.calledWithExactly(null, false).should.equal(true)
       })
     })
 
-    return describe('if there is no output.tex file in the resources', function() {
-      beforeEach(function() {
+    return describe('if there is no output.tex file in the resources', function () {
+      beforeEach(function () {
         this.resources = [{ path: 'main.tex' }]
         return (this.ResourceWriter.checkPath = sinon
           .stub()
@@ -60,8 +60,8 @@ describe('TikzManager', function() {
           .callsArgWith(2, null, `${this.compileDir}/${this.mainFile}`))
       })
 
-      describe('and the main file contains tikzexternalize', function() {
-        beforeEach(function() {
+      describe('and the main file contains tikzexternalize', function () {
+        beforeEach(function () {
           this.SafeReader.readFile = sinon
             .stub()
             .withArgs(`${this.compileDir}/${this.mainFile}`)
@@ -74,19 +74,19 @@ describe('TikzManager', function() {
           )
         })
 
-        it('should look at the file on disk', function() {
+        it('should look at the file on disk', function () {
           return this.SafeReader.readFile
             .calledWith(`${this.compileDir}/${this.mainFile}`)
             .should.equal(true)
         })
 
-        return it('should call the callback with true ', function() {
+        return it('should call the callback with true ', function () {
           return this.callback.calledWithExactly(null, true).should.equal(true)
         })
       })
 
-      describe('and the main file does not contain tikzexternalize', function() {
-        beforeEach(function() {
+      describe('and the main file does not contain tikzexternalize', function () {
+        beforeEach(function () {
           this.SafeReader.readFile = sinon
             .stub()
             .withArgs(`${this.compileDir}/${this.mainFile}`)
@@ -99,19 +99,19 @@ describe('TikzManager', function() {
           )
         })
 
-        it('should look at the file on disk', function() {
+        it('should look at the file on disk', function () {
           return this.SafeReader.readFile
             .calledWith(`${this.compileDir}/${this.mainFile}`)
             .should.equal(true)
         })
 
-        return it('should call the callback with false', function() {
+        return it('should call the callback with false', function () {
           return this.callback.calledWithExactly(null, false).should.equal(true)
         })
       })
 
-      return describe('and the main file contains \\usepackage{pstool}', function() {
-        beforeEach(function() {
+      return describe('and the main file contains \\usepackage{pstool}', function () {
+        beforeEach(function () {
           this.SafeReader.readFile = sinon
             .stub()
             .withArgs(`${this.compileDir}/${this.mainFile}`)
@@ -124,21 +124,21 @@ describe('TikzManager', function() {
           )
         })
 
-        it('should look at the file on disk', function() {
+        it('should look at the file on disk', function () {
           return this.SafeReader.readFile
             .calledWith(`${this.compileDir}/${this.mainFile}`)
             .should.equal(true)
         })
 
-        return it('should call the callback with true ', function() {
+        return it('should call the callback with true ', function () {
           return this.callback.calledWithExactly(null, true).should.equal(true)
         })
       })
     })
   })
 
-  return describe('injectOutputFile', function() {
-    beforeEach(function() {
+  return describe('injectOutputFile', function () {
+    beforeEach(function () {
       this.rootDir = '/mock'
       this.filename = 'filename.tex'
       this.callback = sinon.stub()
@@ -162,25 +162,25 @@ Hello world
       )
     })
 
-    it('sould check the path', function() {
+    it('sould check the path', function () {
       return this.ResourceWriter.checkPath
         .calledWith(this.rootDir, this.filename)
         .should.equal(true)
     })
 
-    it('should read the file', function() {
+    it('should read the file', function () {
       return this.fs.readFile
         .calledWith(`${this.rootDir}/${this.filename}`, 'utf8')
         .should.equal(true)
     })
 
-    it('should write out the same file as output.tex', function() {
+    it('should write out the same file as output.tex', function () {
       return this.fs.writeFile
         .calledWith(`${this.rootDir}/output.tex`, this.content, { flag: 'wx' })
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })

test/unit/js/UrlCacheTests.js

@@ -16,8 +16,8 @@ require('chai').should()
 const modulePath = require('path').join(__dirname, '../../../app/js/UrlCache')
 const { EventEmitter } = require('events')
 
-describe('UrlCache', function() {
-  beforeEach(function() {
+describe('UrlCache', function () {
+  beforeEach(function () {
     this.callback = sinon.stub()
     this.url = 'www.example.com/file'
     this.project_id = 'project-id-123'
@@ -34,16 +34,16 @@ describe('UrlCache', function() {
     }))
   })
 
-  describe('_doesUrlNeedDownloading', function() {
-    beforeEach(function() {
+  describe('_doesUrlNeedDownloading', function () {
+    beforeEach(function () {
       this.lastModified = new Date()
       return (this.lastModifiedRoundedToSeconds = new Date(
         Math.floor(this.lastModified.getTime() / 1000) * 1000
       ))
     })
 
-    describe('when URL does not exist in cache', function() {
-      beforeEach(function() {
+    describe('when URL does not exist in cache', function () {
+      beforeEach(function () {
         this.UrlCache._findUrlDetails = sinon.stub().callsArgWith(2, null, null)
         return this.UrlCache._doesUrlNeedDownloading(
           this.project_id,
@@ -53,21 +53,21 @@ describe('UrlCache', function() {
         )
       })
 
-      return it('should return the callback with true', function() {
+      return it('should return the callback with true', function () {
         return this.callback.calledWith(null, true).should.equal(true)
       })
     })
 
-    return describe('when URL does exist in cache', function() {
-      beforeEach(function() {
+    return describe('when URL does exist in cache', function () {
+      beforeEach(function () {
         this.urlDetails = {}
         return (this.UrlCache._findUrlDetails = sinon
           .stub()
           .callsArgWith(2, null, this.urlDetails))
       })
 
-      describe('when the modified date is more recent than the cached modified date', function() {
-        beforeEach(function() {
+      describe('when the modified date is more recent than the cached modified date', function () {
+        beforeEach(function () {
           this.urlDetails.lastModified = new Date(
             this.lastModified.getTime() - 1000
           )
@@ -79,19 +79,19 @@ describe('UrlCache', function() {
           )
         })
 
-        it('should get the url details', function() {
+        it('should get the url details', function () {
           return this.UrlCache._findUrlDetails
             .calledWith(this.project_id, this.url)
             .should.equal(true)
         })
 
-        return it('should return the callback with true', function() {
+        return it('should return the callback with true', function () {
           return this.callback.calledWith(null, true).should.equal(true)
         })
       })
 
-      describe('when the cached modified date is more recent than the modified date', function() {
-        beforeEach(function() {
+      describe('when the cached modified date is more recent than the modified date', function () {
+        beforeEach(function () {
           this.urlDetails.lastModified = new Date(
             this.lastModified.getTime() + 1000
           )
@@ -103,13 +103,13 @@ describe('UrlCache', function() {
           )
         })
 
-        return it('should return the callback with false', function() {
+        return it('should return the callback with false', function () {
           return this.callback.calledWith(null, false).should.equal(true)
         })
       })
 
-      describe('when the cached modified date is equal to the modified date', function() {
-        beforeEach(function() {
+      describe('when the cached modified date is equal to the modified date', function () {
+        beforeEach(function () {
           this.urlDetails.lastModified = this.lastModified
           return this.UrlCache._doesUrlNeedDownloading(
             this.project_id,
@@ -119,13 +119,13 @@ describe('UrlCache', function() {
           )
         })
 
-        return it('should return the callback with false', function() {
+        return it('should return the callback with false', function () {
           return this.callback.calledWith(null, false).should.equal(true)
         })
       })
 
-      describe('when the provided modified date does not exist', function() {
-        beforeEach(function() {
+      describe('when the provided modified date does not exist', function () {
+        beforeEach(function () {
           this.lastModified = null
           return this.UrlCache._doesUrlNeedDownloading(
             this.project_id,
@@ -135,13 +135,13 @@ describe('UrlCache', function() {
           )
         })
 
-        return it('should return the callback with true', function() {
+        return it('should return the callback with true', function () {
           return this.callback.calledWith(null, true).should.equal(true)
         })
       })
 
-      return describe('when the URL does not have a modified date', function() {
-        beforeEach(function() {
+      return describe('when the URL does not have a modified date', function () {
+        beforeEach(function () {
           this.urlDetails.lastModified = null
           return this.UrlCache._doesUrlNeedDownloading(
             this.project_id,
@@ -151,23 +151,23 @@ describe('UrlCache', function() {
           )
         })
 
-        return it('should return the callback with true', function() {
+        return it('should return the callback with true', function () {
           return this.callback.calledWith(null, true).should.equal(true)
         })
       })
     })
   })
 
-  describe('_ensureUrlIsInCache', function() {
-    beforeEach(function() {
-      this.UrlFetcher.pipeUrlToFile = sinon.stub().callsArg(2)
+  describe('_ensureUrlIsInCache', function () {
+    beforeEach(function () {
+      this.UrlFetcher.pipeUrlToFileWithRetry = sinon.stub().callsArg(2)
       return (this.UrlCache._updateOrCreateUrlDetails = sinon
         .stub()
         .callsArg(3))
     })
 
-    describe('when the URL needs updating', function() {
-      beforeEach(function() {
+    describe('when the URL needs updating', function () {
+      beforeEach(function () {
         this.UrlCache._doesUrlNeedDownloading = sinon
           .stub()
           .callsArgWith(3, null, true)
@@ -179,7 +179,7 @@ describe('UrlCache', function() {
         )
       })
 
-      it('should check that the url needs downloading', function() {
+      it('should check that the url needs downloading', function () {
         return this.UrlCache._doesUrlNeedDownloading
           .calledWith(
             this.project_id,
@@ -189,8 +189,8 @@ describe('UrlCache', function() {
           .should.equal(true)
       })
 
-      it('should download the URL to the cache file', function() {
-        return this.UrlFetcher.pipeUrlToFile
+      it('should download the URL to the cache file', function () {
+        return this.UrlFetcher.pipeUrlToFileWithRetry
           .calledWith(
             this.url,
             this.UrlCache._cacheFilePathForUrl(this.project_id, this.url)
@@ -198,7 +198,7 @@ describe('UrlCache', function() {
           .should.equal(true)
       })
 
-      it('should update the database entry', function() {
+      it('should update the database entry', function () {
         return this.UrlCache._updateOrCreateUrlDetails
           .calledWith(
             this.project_id,
@@ -208,7 +208,7 @@ describe('UrlCache', function() {
           .should.equal(true)
       })
 
-      return it('should return the callback with the cache file path', function() {
+      return it('should return the callback with the cache file path', function () {
         return this.callback
           .calledWith(
             null,
@@ -218,8 +218,8 @@ describe('UrlCache', function() {
       })
     })
 
-    return describe('when the URL does not need updating', function() {
-      beforeEach(function() {
+    return describe('when the URL does not need updating', function () {
+      beforeEach(function () {
         this.UrlCache._doesUrlNeedDownloading = sinon
           .stub()
           .callsArgWith(3, null, false)
@@ -231,11 +231,11 @@ describe('UrlCache', function() {
         )
       })
 
-      it('should not download the URL to the cache file', function() {
-        return this.UrlFetcher.pipeUrlToFile.called.should.equal(false)
+      it('should not download the URL to the cache file', function () {
+        return this.UrlFetcher.pipeUrlToFileWithRetry.called.should.equal(false)
       })
 
-      return it('should return the callback with the cache file path', function() {
+      return it('should return the callback with the cache file path', function () {
         return this.callback
           .calledWith(
             null,
@@ -246,8 +246,8 @@ describe('UrlCache', function() {
     })
   })
 
-  describe('downloadUrlToFile', function() {
-    beforeEach(function() {
+  describe('downloadUrlToFile', function () {
+    beforeEach(function () {
       this.cachePath = 'path/to/cached/url'
       this.destPath = 'path/to/destination'
       this.UrlCache._copyFile = sinon.stub().callsArg(2)
@@ -263,25 +263,25 @@ describe('UrlCache', function() {
       )
     })
 
-    it('should ensure the URL is downloaded and updated in the cache', function() {
+    it('should ensure the URL is downloaded and updated in the cache', function () {
       return this.UrlCache._ensureUrlIsInCache
         .calledWith(this.project_id, this.url, this.lastModified)
         .should.equal(true)
     })
 
-    it('should copy the file to the new location', function() {
+    it('should copy the file to the new location', function () {
       return this.UrlCache._copyFile
         .calledWith(this.cachePath, this.destPath)
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  describe('_deleteUrlCacheFromDisk', function() {
-    beforeEach(function() {
+  describe('_deleteUrlCacheFromDisk', function () {
+    beforeEach(function () {
       this.fs.unlink = sinon.stub().callsArg(1)
       return this.UrlCache._deleteUrlCacheFromDisk(
         this.project_id,
@@ -290,7 +290,7 @@ describe('UrlCache', function() {
       )
     })
 
-    it('should delete the cache file', function() {
+    it('should delete the cache file', function () {
       return this.fs.unlink
         .calledWith(
           this.UrlCache._cacheFilePathForUrl(this.project_id, this.url)
@@ -298,13 +298,13 @@ describe('UrlCache', function() {
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  describe('_clearUrlFromCache', function() {
-    beforeEach(function() {
+  describe('_clearUrlFromCache', function () {
+    beforeEach(function () {
       this.UrlCache._deleteUrlCacheFromDisk = sinon.stub().callsArg(2)
       this.UrlCache._clearUrlDetails = sinon.stub().callsArg(2)
       return this.UrlCache._clearUrlFromCache(
@@ -314,25 +314,25 @@ describe('UrlCache', function() {
       )
     })
 
-    it('should delete the file on disk', function() {
+    it('should delete the file on disk', function () {
       return this.UrlCache._deleteUrlCacheFromDisk
         .calledWith(this.project_id, this.url)
         .should.equal(true)
     })
 
-    it('should clear the entry in the database', function() {
+    it('should clear the entry in the database', function () {
       return this.UrlCache._clearUrlDetails
         .calledWith(this.project_id, this.url)
         .should.equal(true)
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })
 
-  return describe('clearProject', function() {
-    beforeEach(function() {
+  return describe('clearProject', function () {
+    beforeEach(function () {
       this.urls = ['www.example.com/file1', 'www.example.com/file2']
       this.UrlCache._findAllUrlsInProject = sinon
         .stub()
@@ -341,15 +341,15 @@ describe('UrlCache', function() {
       return this.UrlCache.clearProject(this.project_id, this.callback)
     })
 
-    it('should clear the cache for each url in the project', function() {
-      return Array.from(this.urls).map(url =>
+    it('should clear the cache for each url in the project', function () {
+      return Array.from(this.urls).map((url) =>
         this.UrlCache._clearUrlFromCache
           .calledWith(this.project_id, url)
           .should.equal(true)
       )
     })
 
-    return it('should call the callback', function() {
+    return it('should call the callback', function () {
       return this.callback.called.should.equal(true)
     })
   })

test/unit/js/UrlFetcherTests.js

@@ -15,8 +15,8 @@ require('chai').should()
 const modulePath = require('path').join(__dirname, '../../../app/js/UrlFetcher')
 const { EventEmitter } = require('events')
 
-describe('UrlFetcher', function() {
-  beforeEach(function() {
+describe('UrlFetcher', function () {
+  beforeEach(function () {
     this.callback = sinon.stub()
     this.url = 'https://www.example.com/file/here?query=string'
     return (this.UrlFetcher = SandboxedModule.require(modulePath, {
@@ -33,13 +33,48 @@ describe('UrlFetcher', function() {
       }
     }))
   })
+  describe('pipeUrlToFileWithRetry', function () {
+    this.beforeEach(function () {
+      this.UrlFetcher.pipeUrlToFile = sinon.stub()
+    })
 
-  it('should turn off the cookie jar in request', function() {
+    it('should call pipeUrlToFile', function (done) {
+      this.UrlFetcher.pipeUrlToFile.callsArgWith(2)
+      this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, (err) => {
+        expect(err).to.equal(undefined)
+        this.UrlFetcher.pipeUrlToFile.called.should.equal(true)
+        done()
+      })
+    })
+
+    it('should call pipeUrlToFile multiple times on error', function (done) {
+      const error = new Error("couldn't download file")
+      this.UrlFetcher.pipeUrlToFile.callsArgWith(2, error)
+      this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, (err) => {
+        expect(err).to.equal(error)
+        this.UrlFetcher.pipeUrlToFile.callCount.should.equal(3)
+        done()
+      })
+    })
+
+    it('should call pipeUrlToFile twice if only 1 error', function (done) {
+      this.UrlFetcher.pipeUrlToFile.onCall(0).callsArgWith(2, 'error')
+      this.UrlFetcher.pipeUrlToFile.onCall(1).callsArgWith(2)
+      this.UrlFetcher.pipeUrlToFileWithRetry(this.url, this.path, (err) => {
+        expect(err).to.equal(undefined)
+        this.UrlFetcher.pipeUrlToFile.callCount.should.equal(2)
+        done()
+      })
+    })
+  })
+
+  describe('pipeUrlToFile', function () {
+    it('should turn off the cookie jar in request', function () {
       return this.defaults.calledWith({ jar: false }).should.equal(true)
     })
 
-  describe('rewrite url domain if filestoreDomainOveride is set', function() {
-    beforeEach(function() {
+    describe('rewrite url domain if filestoreDomainOveride is set', function () {
+      beforeEach(function () {
         this.path = '/path/to/file/on/disk'
         this.request.get = sinon
           .stub()
@@ -53,7 +88,7 @@ describe('UrlFetcher', function() {
         return (this.fs.unlink = (file, callback) => callback())
       })
 
-    it('should use the normal domain when override not set', function(done) {
+      it('should use the normal domain when override not set', function (done) {
         this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
           this.request.get.args[0][0].url.should.equal(this.url)
           return done()
@@ -64,7 +99,7 @@ describe('UrlFetcher', function() {
         return this.fileStream.emit('finish')
       })
 
-    return it('should use override domain when filestoreDomainOveride is set', function(done) {
+      return it('should use override domain when filestoreDomainOveride is set', function (done) {
         this.settings.filestoreDomainOveride = '192.11.11.11'
         this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
           this.request.get.args[0][0].url.should.equal(
@@ -79,8 +114,8 @@ describe('UrlFetcher', function() {
       })
     })
 
-  return describe('pipeUrlToFile', function() {
-    beforeEach(function(done) {
+    return describe('pipeUrlToFile', function () {
+      beforeEach(function (done) {
         this.path = '/path/to/file/on/disk'
         this.request.get = sinon
           .stub()
@@ -95,8 +130,8 @@ describe('UrlFetcher', function() {
         return done()
       })
 
-    describe('successfully', function() {
-      beforeEach(function(done) {
+      describe('successfully', function () {
+        beforeEach(function (done) {
           this.UrlFetcher.pipeUrlToFile(this.url, this.path, () => {
             this.callback()
             return done()
@@ -107,32 +142,32 @@ describe('UrlFetcher', function() {
           return this.fileStream.emit('finish')
         })
 
-      it('should request the URL', function() {
+        it('should request the URL', function () {
           return this.request.get
             .calledWith(sinon.match({ url: this.url }))
             .should.equal(true)
         })
 
-      it('should open the file for writing', function() {
+        it('should open the file for writing', function () {
           return this.fs.createWriteStream
             .calledWith(this.path)
             .should.equal(true)
         })
 
-      it('should pipe the URL to the file', function() {
+        it('should pipe the URL to the file', function () {
           return this.urlStream.pipe
             .calledWith(this.fileStream)
             .should.equal(true)
         })
 
-      return it('should call the callback', function() {
+        return it('should call the callback', function () {
           return this.callback.called.should.equal(true)
         })
       })
 
-    describe('with non success status code', function() {
-      beforeEach(function(done) {
-        this.UrlFetcher.pipeUrlToFile(this.url, this.path, err => {
+      describe('with non success status code', function () {
+        beforeEach(function (done) {
+          this.UrlFetcher.pipeUrlToFile(this.url, this.path, (err) => {
             this.callback(err)
             return done()
           })
@@ -141,17 +176,19 @@ describe('UrlFetcher', function() {
           return this.urlStream.emit('end')
         })
 
-      it('should call the callback with an error', function() {
+        it('should call the callback with an error', function () {
           this.callback.calledWith(sinon.match(Error)).should.equal(true)
 
           const message = this.callback.args[0][0].message
-        expect(message).to.include('URL returned non-success status code: 404')
+          expect(message).to.include(
+            'URL returned non-success status code: 404'
+          )
         })
       })
 
-    return describe('with error', function() {
-      beforeEach(function(done) {
-        this.UrlFetcher.pipeUrlToFile(this.url, this.path, err => {
+      return describe('with error', function () {
+        beforeEach(function (done) {
+          this.UrlFetcher.pipeUrlToFile(this.url, this.path, (err) => {
             this.callback(err)
             return done()
           })
@@ -161,14 +198,15 @@ describe('UrlFetcher', function() {
           )
         })
 
-      it('should call the callback with the error', function() {
+        it('should call the callback with the error', function () {
           return this.callback.calledWith(this.error).should.equal(true)
         })
 
-      return it('should only call the callback once, even if end is called', function() {
+        return it('should only call the callback once, even if end is called', function () {
           this.urlStream.emit('end')
           return this.callback.calledOnce.should.equal(true)
         })
       })
     })
+  })
 })