Explicitly start build immediately

This reverts commit 3ab407b91a.

.dockerignore

@@ -0,0 +1,9 @@
+node_modules/*
+gitrev
+.git
+.gitignore
+.npm
+.nvmrc
+nodemon.json
+app.js
+**/js/*

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,38 @@
+<!-- BUG REPORT TEMPLATE -->
+
+## Steps to Reproduce
+<!-- Describe the steps leading up to when / where you found the bug. -->
+<!-- Screenshots may be helpful here. -->
+
+1.
+2.
+3.
+
+## Expected Behaviour
+<!-- What should have happened when you completed the steps above? -->
+
+## Observed Behaviour
+<!-- What actually happened when you completed the steps above? -->
+<!-- Screenshots may be helpful here. -->
+
+## Context
+<!-- How has this issue affected you? What were you trying to accomplish? -->
+
+## Technical Info
+<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
+
+* URL:
+* Browser Name and version:
+* Operating System and version (desktop or mobile):
+* Signed in as:
+* Project and/or file:
+
+## Analysis
+<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
+
+## Who Needs to Know?
+<!-- If you want to bring this to the attention of particular people, @-mention them below. -->
+<!-- If a user reported this bug and should be notified when it is fixed, provide the Front conversation link. -->
+
+-
+-

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,45 @@
+<!-- Please review https://github.com/overleaf/write_latex/blob/master/.github/CONTRIBUTING.md for guidance on what is expected in each section. -->
+
+### Description
+
+
+
+#### Screenshots
+
+
+
+#### Related Issues / PRs
+
+
+
+### Review
+
+
+
+#### Potential Impact
+
+
+
+#### Manual Testing Performed
+
+- [ ]
+- [ ]
+
+#### Accessibility
+
+
+
+### Deployment
+
+
+
+#### Deployment Checklist
+
+- [ ] Update documentation not included in the PR (if any)
+- [ ]
+
+#### Metrics and Monitoring
+
+
+
+#### Who Needs to Know?

.gitignore

@@ -7,10 +7,13 @@ test/acceptance/js
 test/acceptance/fixtures/tmp
 compiles
 app.js
+**/*.map
 .DS_Store
 *~
 cache
 .vagrant
 db.sqlite
+db.sqlite-wal
+db.sqlite-shm
 config/*
-bin/synctex
+npm-debug.log

.nvmrc

@@ -0,0 +1 @@
+10.15.0

.travis.yml

@@ -1,8 +1,5 @@
 language: node_js
 
-node_js:
-    - "0.10"
-
 before_install:
     - npm install -g grunt-cli
 

.viminfo

@@ -0,0 +1,35 @@
+# This viminfo file was generated by Vim 7.4.
+# You may edit it if you're careful!
+
+# Value of 'encoding' when this file was written
+*encoding=latin1
+
+
+# hlsearch on (H) or off (h):
+~h
+# Command Line History (newest to oldest):
+:x
+
+# Search String History (newest to oldest):
+
+# Expression History (newest to oldest):
+
+# Input Line History (newest to oldest):
+
+# Input Line History (newest to oldest):
+
+# Registers:
+
+# File marks:
+'0  1  0  ~/hello
+
+# Jumplist (newest first):
+-'  1  0  ~/hello
+
+# History of marks within files (newest to oldest):
+
+> ~/hello
+	"	1	0
+	^	1	1
+	.	1	0
+	+	1	0

Dockerfile

@@ -0,0 +1,27 @@
+FROM node:10.15.0 as app
+
+WORKDIR /app
+
+#wildcard as some files may not be in all repos
+COPY package*.json npm-shrink*.json /app/
+
+RUN npm install --quiet
+
+COPY . /app
+
+
+RUN npm run compile:all
+
+FROM node:10.15.0
+
+RUN \
+   apt -y update && \
+   apt -y install moreutils
+
+COPY --from=app /app /app
+
+WORKDIR /app
+RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
+ENTRYPOINT ["/bin/bash", "entrypoint.sh"]
+
+CMD ["node", "--expose-gc", "app.js"]

Gruntfile.coffee

@@ -1,98 +0,0 @@
-spawn = require("child_process").spawn
-
-module.exports = (grunt) ->
-	grunt.initConfig
-		coffee:
-			app_src:
-				expand: true,
-				flatten: true,
-				cwd: "app"
-				src: ['coffee/*.coffee'],
-				dest: 'app/js/',
-				ext: '.js'
-
-			app:
-				src: "app.coffee"
-				dest: "app.js"
-
-			unit_tests:
-				expand: true
-				cwd:  "test/unit/coffee"
-				src: ["**/*.coffee"]
-				dest: "test/unit/js/"
-				ext:  ".js"
-
-			acceptance_tests:
-				expand: true
-				cwd:  "test/acceptance/coffee"
-				src: ["**/*.coffee"]
-				dest: "test/acceptance/js/"
-				ext:  ".js"
-
-			smoke_tests:
-				expand: true
-				cwd:  "test/smoke/coffee"
-				src: ["**/*.coffee"]
-				dest: "test/smoke/js"
-				ext:  ".js"
-
-		clean:
-			app: ["app/js/"]
-			unit_tests: ["test/unit/js"]
-			acceptance_tests: ["test/acceptance/js"]
-			smoke_tests: ["test/smoke/js"]
-
-		execute:
-			app:
-				src: "app.js"
-
-		mochaTest:
-			unit:
-				options:
-					reporter: "spec"
-					grep: grunt.option("grep")
-				src: ["test/unit/js/**/*.js"]
-			acceptance:
-				options:
-					reporter: "spec"
-					timeout: 40000
-					grep: grunt.option("grep")
-				src: ["test/acceptance/js/**/*.js"]
-			smoke:
-				options:
-					reported: "spec"
-					timeout: 10000
-				src: ["test/smoke/js/**/*.js"]
-
-	grunt.loadNpmTasks 'grunt-contrib-coffee'
-	grunt.loadNpmTasks 'grunt-contrib-clean'
-	grunt.loadNpmTasks 'grunt-mocha-test'
-	grunt.loadNpmTasks 'grunt-shell'
-	grunt.loadNpmTasks 'grunt-execute'
-	grunt.loadNpmTasks 'grunt-bunyan'
-
-	grunt.registerTask 'compile:bin', () ->	
-		callback = @async()
-		proc = spawn "cc", [
-			"-o", "bin/synctex", "-Isrc/synctex",
-			"src/synctex.c", "src/synctex/synctex_parser.c", "src/synctex/synctex_parser_utils.c", "-lz"
-		], stdio: "inherit"
-		proc.on "close", callback
-
-	grunt.registerTask 'compile:app', ['clean:app', 'coffee:app', 'coffee:app_src', 'coffee:smoke_tests', 'compile:bin']
-	grunt.registerTask 'run',         ['compile:app', 'bunyan', 'execute']
-
-	grunt.registerTask 'compile:unit_tests', ['clean:unit_tests', 'coffee:unit_tests']
-	grunt.registerTask 'test:unit',          ['compile:app', 'compile:unit_tests', 'mochaTest:unit']
-
-	grunt.registerTask 'compile:acceptance_tests', ['clean:acceptance_tests', 'coffee:acceptance_tests']
-	grunt.registerTask 'test:acceptance',          ['compile:acceptance_tests', 'mochaTest:acceptance']
-
-	grunt.registerTask 'compile:smoke_tests', ['clean:smoke_tests', 'coffee:smoke_tests']
-	grunt.registerTask 'test:smoke',          ['compile:smoke_tests', 'mochaTest:smoke']
-
-	grunt.registerTask 'install', 'compile:app'
-
-	grunt.registerTask 'default', ['run']
-
-

Jenkinsfile

@@ -0,0 +1,123 @@
+String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
+
+pipeline {
+  agent any
+
+  environment {
+    GIT_PROJECT = "clsi"
+    JENKINS_WORKFLOW = "clsi-sharelatex"
+    TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline"
+    GIT_API_URL = "https://api.github.com/repos/overleaf/${GIT_PROJECT}/statuses/$GIT_COMMIT"
+  }
+
+  triggers {
+    pollSCM('* * * * *')
+    cron(cron_string)
+  }
+
+  stages {
+    stage('Install') {
+      steps {
+        withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+          sh "curl $GIT_API_URL \
+            --data '{ \
+            \"state\" : \"pending\", \
+            \"target_url\": \"$TARGET_URL\", \
+            \"description\": \"Your build is underway\", \
+            \"context\": \"ci/jenkins\" }' \
+            -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+        }
+      }
+    }
+
+    stage('Build') {
+      steps {
+        sh 'make build'
+      }
+    }
+
+    stage('Unit Tests') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
+      }
+    }
+
+    stage('Acceptance Tests') {
+      steps {
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
+      }
+    }
+
+    stage('Package and docker push') {
+      steps {
+        sh 'echo ${BUILD_NUMBER} > build_number.txt'
+        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
+        
+        withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
+          sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
+        }
+        sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
+        sh 'docker logout https://gcr.io/overleaf-ops'
+        
+      }
+    }
+
+    stage('Publish to s3') {
+      steps {
+        sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
+        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
+            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
+        }
+        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
+            // The deployment process uses this file to figure out the latest build
+            s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
+        }
+      }
+    }
+  }
+
+  post {
+    always {
+      sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
+      sh 'make clean'
+    }
+
+    success {
+      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+        sh "curl $GIT_API_URL \
+          --data '{ \
+          \"state\" : \"success\", \
+          \"target_url\": \"$TARGET_URL\", \
+          \"description\": \"Your build succeeded!\", \
+          \"context\": \"ci/jenkins\" }' \
+          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+      }
+    }
+
+    failure {
+      mail(from: "${EMAIL_ALERT_FROM}",
+           to: "${EMAIL_ALERT_TO}",
+           subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
+           body: "Build: ${BUILD_URL}")
+      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+        sh "curl $GIT_API_URL \
+          --data '{ \
+          \"state\" : \"failure\", \
+          \"target_url\": \"$TARGET_URL\", \
+          \"description\": \"Your build failed\", \
+          \"context\": \"ci/jenkins\" }' \
+          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+      }
+    }
+  }
+
+  // The options directive is for configuration that applies to the whole job.
+  options {
+    // we'd like to make sure remove old builds, so we don't fill up our storage!
+    buildDiscarder(logRotator(numToKeepStr:'50'))
+
+    // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
+    timeout(time: 30, unit: 'MINUTES')
+  }
+}

Makefile

@@ -0,0 +1,51 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.22
+
+BUILD_NUMBER ?= local
+BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+PROJECT_NAME = clsi
+DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
+DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	docker-compose ${DOCKER_COMPOSE_FLAGS}
+
+clean:
+	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	rm -f app.js
+	rm -rf app/js
+	rm -rf test/unit/js
+	rm -rf test/acceptance/js
+
+test: test_unit test_acceptance
+
+test_unit:
+	@[ ! -d test/unit ] && echo "clsi has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
+
+test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run
+
+test_acceptance_run:
+	@[ ! -d test/acceptance ] && echo "clsi has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
+
+test_clean:
+	$(DOCKER_COMPOSE) down -v -t 0
+
+test_acceptance_pre_run:
+	@[ ! -f test/acceptance/js/scripts/pre-run ] && echo "clsi has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
+build:
+	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		.
+
+tar:
+	$(DOCKER_COMPOSE) up tar
+
+publish:
+
+	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+
+.PHONY: clean test test_unit test_acceptance test_clean build publish

README.md

@@ -1,16 +1,38 @@
-clsi-sharelatex
+overleaf/clsi
 ===============
 
 A web api for compiling LaTeX documents in the cloud
 
-[![Build Status](https://travis-ci.org/sharelatex/clsi-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/clsi-sharelatex)
+The Common LaTeX Service Interface (CLSI) provides a RESTful interface to traditional LaTeX tools (or, more generally, any command line tool for composing marked-up documents into a display format such as PDF or HTML). The CLSI listens on the following ports by default:
+
+* TCP/3009 - the RESTful interface
+* TCP/3048 - reports load information
+* TCP/3049 - HTTP interface to control the CLSI service
+
+These defaults can be modified in `config/settings.defaults.coffee`.
+
+The provided `Dockerfile` builds a docker image which has the docker command line tools installed. The configuration in `docker-compose-config.yml` mounts the docker socket, in order that the CLSI container can talk to the docker host it is running in. This allows it to spin up `sibling containers` running an image with a TeX distribution installed to perform the actual compiles.
+
+The CLSI can be configured through the following environment variables:
+
+  * `DOCKER_RUNNER` - Set to true to use sibling containers
+  * `SYNCTEX_BIN_HOST_PATH` - Path to SyncTeX binary
+  * `COMPILES_HOST_DIR` - Working directory for LaTeX compiles
+  * `SQLITE_PATH` - Path to SQLite database
+  * `TEXLIVE_IMAGE` - The TEXLIVE docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1`
+  * `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TEXLIVE image. Defaults to `tex`
+  * `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the docker image e.g. `gcr.io/overleaf-ops`
+  * `FILESTORE_DOMAIN_OVERRIDE` - The url for the filestore service e.g.`http://$FILESTORE_HOST:3009`
+  * `STATSD_HOST` - The address of the Statsd service (used by the metrics module)
+  * `LISTEN_ADDRESS` - The address for the RESTful service to listen on. Set to `0.0.0.0` to listen on all network interfaces
+  * `SMOKE_TEST` - Whether to run smoke tests
 
 Installation
 ------------
 
-The CLSI can be installed and set up as part of the entire [ShareLaTeX stack](https://github.com/sharelatex/sharelatex) (complete with front end editor and document storage), or it can be run as a standalone service. To run is as a standalone service, first checkout this repository:
+The CLSI can be installed and set up as part of the entire [Overleaf stack](https://github.com/overleaf/overleaf) (complete with front end editor and document storage), or it can be run as a standalone service. To run is as a standalone service, first checkout this repository:
 
-    $ git clone git@github.com:sharelatex/clsi-sharelatex.git
+    $ git clone git@github.com:overleaf/clsi.git
     
 Then install the require npm modules:
 
@@ -92,4 +114,4 @@ License
 
 The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the `LICENSE` file.
 
-Copyright (c) ShareLaTeX, 2014.
+Copyright (c) Overleaf, 2014-2019.

app.coffee

@@ -1,14 +1,21 @@
+Metrics = require "metrics-sharelatex"
+Metrics.initialize("clsi")
+
 CompileController = require "./app/js/CompileController"
 Settings = require "settings-sharelatex"
 logger = require "logger-sharelatex"
 logger.initialize("clsi")
+if Settings.sentry?.dsn?
+	logger.initializeErrorReporting(Settings.sentry.dsn)
+
 smokeTest = require "smoke-test-sharelatex"
+ContentTypeMapper = require "./app/js/ContentTypeMapper"
+Errors = require './app/js/Errors'
 
 Path = require "path"
 fs = require "fs"
 
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("clsi")
+
 Metrics.open_sockets.monitor(logger)
 Metrics.memory.monitor(logger)
 
@@ -21,23 +28,55 @@ express = require "express"
 bodyParser = require "body-parser"
 app = express()
 
+Metrics.injectMetricsRoute(app)
 app.use Metrics.http.monitor(logger)
 
 # Compile requests can take longer than the default two
 # minutes (including file download time), so bump up the 
 # timeout a bit.
-TIMEOUT = 6 * 60 * 1000
+TIMEOUT = 10 * 60 * 1000
 app.use (req, res, next) ->
 	req.setTimeout TIMEOUT
 	res.setTimeout TIMEOUT
+	res.removeHeader("X-Powered-By")
 	next()
 
-app.post   "/project/:project_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile
+app.param 'project_id', (req, res, next, project_id) ->
+	if project_id?.match /^[a-zA-Z0-9_-]+$/
+		next()
+	else
+		next new Error("invalid project id")
+
+app.param 'user_id', (req, res, next, user_id) ->
+	if user_id?.match /^[0-9a-f]{24}$/
+		next()
+	else
+		next new Error("invalid user id")
+
+app.param 'build_id', (req, res, next, build_id) ->
+	if build_id?.match OutputCacheManager.BUILD_REGEX
+		next()
+	else
+		next new Error("invalid build id #{build_id}")
+
+
+app.post   "/project/:project_id/compile", bodyParser.json(limit: Settings.compileSizeLimit), CompileController.compile
+app.post   "/project/:project_id/compile/stop", CompileController.stopCompile
 app.delete "/project/:project_id", CompileController.clearCache
 
 app.get  "/project/:project_id/sync/code", CompileController.syncFromCode
 app.get  "/project/:project_id/sync/pdf", CompileController.syncFromPdf
 app.get  "/project/:project_id/wordcount", CompileController.wordcount
+app.get  "/project/:project_id/status", CompileController.status
+
+# Per-user containers
+app.post   "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: Settings.compileSizeLimit), CompileController.compile
+app.post   "/project/:project_id/user/:user_id/compile/stop", CompileController.stopCompile
+app.delete "/project/:project_id/user/:user_id", CompileController.clearCache
+
+app.get  "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode
+app.get  "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf
+app.get  "/project/:project_id/user/:user_id/wordcount", CompileController.wordcount
 
 ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks"
 
@@ -46,17 +85,28 @@ ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks"
 # and serving the files
 staticServer = ForbidSymlinks express.static, Settings.path.compilesDir, setHeaders: (res, path, stat) ->
 	if Path.basename(path) == "output.pdf"
-		res.set("Content-Type", "application/pdf")
 		# Calculate an etag in the same way as nginx
 		# https://github.com/tj/send/issues/65
 		etag = (path, stat) ->
 			'"' + Math.ceil(+stat.mtime / 1000).toString(16) +
 			'-' + Number(stat.size).toString(16) + '"'
 		res.set("Etag", etag(path, stat))
-	else
-		# Force plain treatment of other file types to prevent hosting of HTTP/JS files
-		# that could be used in same-origin/XSS attacks.
-		res.set("Content-Type", "text/plain")
+	res.set("Content-Type", ContentTypeMapper.map(path))
+
+app.get "/project/:project_id/user/:user_id/build/:build_id/output/*", (req, res, next) ->
+	# for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
+	req.url = "/#{req.params.project_id}-#{req.params.user_id}/" + OutputCacheManager.path(req.params.build_id, "/#{req.params[0]}")
+	staticServer(req, res, next)
+
+app.get "/project/:project_id/build/:build_id/output/*", (req, res, next) ->
+	# for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
+	req.url = "/#{req.params.project_id}/" + OutputCacheManager.path(req.params.build_id, "/#{req.params[0]}")
+	staticServer(req, res, next)
+
+app.get "/project/:project_id/user/:user_id/output/*", (req, res, next) ->
+	# for specific user get the path to the top level file
+	req.url = "/#{req.params.project_id}-#{req.params.user_id}/#{req.params[0]}"
+	staticServer(req, res, next)
 
 app.get "/project/:project_id/output/*", (req, res, next) ->
 	if req.query?.build? && req.query.build.match(OutputCacheManager.BUILD_REGEX)
@@ -66,6 +116,11 @@ app.get "/project/:project_id/output/*", (req, res, next) ->
 		req.url = "/#{req.params.project_id}/#{req.params[0]}"
 	staticServer(req, res, next)
 
+app.get "/oops", (req, res, next) ->
+	logger.error {err: "hello"}, "test error"
+	res.send "error\n"
+
+
 app.get "/status", (req, res, next) ->
 	res.send "CLSI is alive\n"
 
@@ -82,13 +137,16 @@ if Settings.smokeTest
 	do runSmokeTest = ->
 		logger.log("running smoke tests")
 		smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))({}, resCacher)
-		setTimeout(runSmokeTest, 20 * 1000)
+		setTimeout(runSmokeTest, 30 * 1000)
 
 app.get "/health_check", (req, res)->
 	res.contentType(resCacher?.setContentType)
 	res.status(resCacher?.code).send(resCacher?.body)
 
-profiler = require "v8-profiler"
+app.get "/smoke_test_force", (req, res)->
+	smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))(req, res)
+
+profiler = require "v8-profiler-node8"
 app.get "/profile", (req, res) ->
 	time = parseInt(req.query.time || "1000")
 	profiler.startProfiling("test")
@@ -102,12 +160,85 @@ app.get "/heapdump", (req, res)->
 		res.send filename
 
 app.use (error, req, res, next) ->
-	logger.error err: error, "server error"
-	res.sendStatus(error?.statusCode || 500)
+	if error instanceof Errors.NotFoundError
+		logger.warn {err: error, url: req.url}, "not found error"
+		return res.sendStatus(404)
+	else
+		logger.error {err: error, url: req.url}, "server error"
+		res.sendStatus(error?.statusCode || 500)
 
-app.listen port = (Settings.internal?.clsi?.port or 3013), host = (Settings.internal?.clsi?.host or "localhost"), (error) ->
-	logger.info "CLSI starting up, listening on #{host}:#{port}"
+net = require "net"
+os = require "os"
+
+STATE = "up"
+
+
+loadTcpServer = net.createServer (socket) ->
+	socket.on "error", (err)->
+		if err.code == "ECONNRESET"
+			# this always comes up, we don't know why
+			return
+		logger.err err:err, "error with socket on load check"
+		socket.destroy()
+	
+	if STATE == "up" and Settings.internal.load_balancer_agent.report_load
+		currentLoad = os.loadavg()[0]
+
+		# staging clis's have 1 cpu core only
+		if os.cpus().length == 1
+			availableWorkingCpus = 1
+		else
+			availableWorkingCpus = os.cpus().length - 1
+
+		freeLoad = availableWorkingCpus - currentLoad
+		freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
+		if freeLoadPercentage <= 0
+			freeLoadPercentage = 1 # when its 0 the server is set to drain and will move projects to different servers
+		socket.write("up, #{freeLoadPercentage}%\n", "ASCII")
+		socket.end()
+	else
+		socket.write("#{STATE}\n", "ASCII")
+		socket.end()
+
+loadHttpServer = express()
+
+loadHttpServer.post "/state/up", (req, res, next) ->
+	STATE = "up"
+	logger.info "getting message to set server to down"
+	res.sendStatus 204
+
+loadHttpServer.post "/state/down", (req, res, next) ->
+	STATE = "down"
+	logger.info "getting message to set server to down"
+	res.sendStatus 204
+
+loadHttpServer.post "/state/maint", (req, res, next) ->
+	STATE = "maint"
+	logger.info "getting message to set server to maint"
+	res.sendStatus 204
+	
+
+port = (Settings.internal?.clsi?.port or 3013)
+host = (Settings.internal?.clsi?.host or "localhost")
+
+load_tcp_port = Settings.internal.load_balancer_agent.load_port
+load_http_port = Settings.internal.load_balancer_agent.local_port
+
+if !module.parent # Called directly
+	app.listen port, host, (error) ->
+		logger.info "CLSI starting up, listening on #{host}:#{port}"
+
+	loadTcpServer.listen load_tcp_port, host, (error) ->
+		throw error if error?
+		logger.info "Load tcp agent listening on load port #{load_tcp_port}"
+
+	loadHttpServer.listen load_http_port, host, (error) ->
+		throw error if error?
+		logger.info "Load http agent listening on load port #{load_http_port}"
+
+module.exports = app
 
 setInterval () ->
 	ProjectPersistenceManager.clearExpiredProjects()
 , tenMinutes = 10 * 60 * 1000
+

app/coffee/CommandRunner.coffee

@@ -1,12 +1,11 @@
-spawn = require("child_process").spawn
+Settings = require "settings-sharelatex"
 logger = require "logger-sharelatex"
 
-module.exports = CommandRunner =
-	run: (project_id, command, directory, timeout, callback = (error) ->) ->
-		command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
-		logger.log project_id: project_id, command: command, directory: directory, "running command"
-		logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
+if Settings.clsi?.dockerRunner == true
+	commandRunnerPath = "./DockerRunner"
+else 
+	commandRunnerPath = "./LocalCommandRunner"
+logger.info commandRunnerPath:commandRunnerPath, "selecting command runner for clsi"
+CommandRunner = require(commandRunnerPath)
 
-		proc = spawn command[0], command.slice(1), stdio: "inherit", cwd: directory
-		proc.on "close", () ->
-			callback()
+module.exports = CommandRunner

app/coffee/CompileController.coffee

@@ -4,6 +4,7 @@ Settings = require "settings-sharelatex"
 Metrics = require "./Metrics"
 ProjectPersistenceManager = require "./ProjectPersistenceManager"
 logger = require "logger-sharelatex"
+Errors = require "./Errors"
 
 module.exports = CompileController =
 	compile: (req, res, next = (error) ->) ->
@@ -11,35 +12,67 @@ module.exports = CompileController =
 		RequestParser.parse req.body, (error, request) ->
 			return next(error) if error?
 			request.project_id = req.params.project_id
+			request.user_id = req.params.user_id if req.params.user_id?
 			ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) ->
 				return next(error) if error?
-				CompileManager.doCompile request, (error, outputFiles = []) ->
-					if error?
-						logger.error err: error, project_id: request.project_id, "error running compile"
+				CompileManager.doCompileWithLock request, (error, outputFiles = []) ->
+					if error instanceof Errors.AlreadyCompilingError
+						code = 423 # Http 423 Locked
+						status = "compile-in-progress"
+					else if error instanceof Errors.FilesOutOfSyncError
+						code = 409 # Http 409 Conflict
+						status = "retry"
+					else if error?.terminated
+						status = "terminated"
+					else if error?.validate
+						status = "validation-#{error.validate}"
+					else if error?
 						if error.timedout
 							status = "timedout"
+							logger.log err: error, project_id: request.project_id, "timeout running compile"
 						else
 							status = "error"
 							code = 500
+							logger.warn err: error, project_id: request.project_id, "error running compile"
+
 					else
 						status = "failure"
 						for file in outputFiles
 							if file.path?.match(/output\.pdf$/)
 								status = "success"
 								
+						if status == "failure"
+							logger.warn project_id: request.project_id, outputFiles:outputFiles, "project failed to compile successfully, no output.pdf generated"
+
+						# log an error if any core files are found
+						for file in outputFiles
+							if file.path is "core"
+								logger.error project_id:request.project_id, req:req, outputFiles:outputFiles, "core file found in output"
+
 					timer.done()
 					res.status(code or 200).send {
 						compile:
 							status: status
 							error:  error?.message or error
 							outputFiles: outputFiles.map (file) ->
-								url: "#{Settings.apis.clsi.url}/project/#{request.project_id}/output/#{file.path}"
+								url:
+									"#{Settings.apis.clsi.url}/project/#{request.project_id}" +
+									(if request.user_id? then "/user/#{request.user_id}" else "") +
+									(if file.build? then "/build/#{file.build}" else "") +
+									"/output/#{file.path}"
+								path: file.path
 								type: file.type
 								build: file.build
 					}
 
+	stopCompile: (req, res, next) ->
+		{project_id, user_id} = req.params
+		CompileManager.stopCompile project_id, user_id, (error) ->
+			return next(error) if error?
+			res.sendStatus(204)
+
 	clearCache: (req, res, next = (error) ->) ->
-		ProjectPersistenceManager.clearProject req.params.project_id, (error) ->
+		ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) ->
 			return next(error) if error?
 			res.sendStatus(204) # No content
 
@@ -48,10 +81,10 @@ module.exports = CompileController =
 		line   = parseInt(req.query.line, 10)
 		column = parseInt(req.query.column, 10)
 		project_id = req.params.project_id
-
-		CompileManager.syncFromCode project_id, file, line, column, (error, pdfPositions) ->
+		user_id = req.params.user_id
+		CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) ->
 			return next(error) if error?
-			res.send JSON.stringify {
+			res.json {
 				pdf: pdfPositions
 			}
 
@@ -60,19 +93,26 @@ module.exports = CompileController =
 		h      = parseFloat(req.query.h)
 		v      = parseFloat(req.query.v)
 		project_id = req.params.project_id
-
-		CompileManager.syncFromPdf project_id, page, h, v, (error, codePositions) ->
+		user_id = req.params.user_id
+		CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) ->
 			return next(error) if error?
-			res.send JSON.stringify {
+			res.json {
 				code: codePositions
 			}
 
 	wordcount: (req, res, next = (error) ->) ->
 		file   = req.query.file || "main.tex"
 		project_id = req.params.project_id
+		user_id = req.params.user_id
+		image = req.query.image
+		logger.log {image, file, project_id}, "word count request"
 
-		CompileManager.wordcount project_id, file, (error, result) ->
+		CompileManager.wordcount project_id, user_id, file, image, (error, result) ->
 			return next(error) if error?
-			res.send JSON.stringify {
+			res.json {
 				texcount: result
 			}
+
+	status: (req, res, next = (error)-> )->
+		res.send("OK")
+

app/coffee/CompileManager.coffee

@@ -7,82 +7,254 @@ Path = require "path"
 logger = require "logger-sharelatex"
 Metrics = require "./Metrics"
 child_process = require "child_process"
-CommandRunner = require(Settings.clsi?.commandRunner or "./CommandRunner")
+DraftModeManager = require "./DraftModeManager"
+TikzManager = require "./TikzManager"
+LockManager = require "./LockManager"
 fs = require("fs")
+fse = require "fs-extra"
+os = require("os")
+async = require "async"
+Errors = require './Errors'
+CommandRunner = require "./CommandRunner"
+
+getCompileName = (project_id, user_id) ->
+	if user_id? then "#{project_id}-#{user_id}" else project_id
+
+getCompileDir = (project_id, user_id) ->
+	Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
 
 module.exports = CompileManager =
-	doCompile: (request, callback = (error, outputFiles) ->) ->
-		compileDir = Path.join(Settings.path.compilesDir, request.project_id)
 
-		timer = new Metrics.Timer("write-to-disk")
-		logger.log project_id: request.project_id, "starting compile"
-		ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) ->
+	doCompileWithLock: (request, callback = (error, outputFiles) ->) ->
+		compileDir = getCompileDir(request.project_id, request.user_id)
+		lockFile = Path.join(compileDir, ".project-lock")
+		# use a .project-lock file in the compile directory to prevent
+		# simultaneous compiles
+		fse.ensureDir compileDir, (error) ->
 			return callback(error) if error?
-			logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "written files to disk"
+			LockManager.runWithLock lockFile, (releaseLock) ->
+				CompileManager.doCompile(request, releaseLock)
+			, callback
+
+	doCompile: (request, callback = (error, outputFiles) ->) ->
+		compileDir = getCompileDir(request.project_id, request.user_id)
+		timer = new Metrics.Timer("write-to-disk")
+		logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
+		ResourceWriter.syncResourcesToDisk request, compileDir, (error, resourceList) ->
+			# NOTE: resourceList is insecure, it should only be used to exclude files from the output list
+			if error? and error instanceof Errors.FilesOutOfSyncError
+				logger.warn project_id: request.project_id, user_id: request.user_id, "files out of sync, please retry"
+				return callback(error)
+			else if error?
+				logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk"
+				return callback(error)
+			logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk"
 			timer.done()
 
-			timer = new Metrics.Timer("run-compile")
-			Metrics.inc("compiles")
-			LatexRunner.runLatex request.project_id, {
-				directory: compileDir
-				mainFile:  request.rootResourcePath
-				compiler:  request.compiler
-				timeout:   request.timeout
-			}, (error) ->
-				return callback(error) if error?
-				logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "done compile"
-				timer.done()
+			injectDraftModeIfRequired = (callback) ->
+				if request.draft
+					DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback
+				else
+					callback()
 
-				OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) ->
+			createTikzFileIfRequired = (callback) ->
+				TikzManager.checkMainFile compileDir, request.rootResourcePath, resourceList, (error, needsMainFile) ->
 					return callback(error) if error?
-					OutputCacheManager.saveOutputFiles outputFiles, compileDir,  (error, newOutputFiles) ->
-						callback null, newOutputFiles
+					if needsMainFile
+						TikzManager.injectOutputFile compileDir, request.rootResourcePath, callback
+					else
+						callback()
 
-	clearProject: (project_id, _callback = (error) ->) ->
+			# set up environment variables for chktex
+			env = {}
+			# only run chktex on LaTeX files (not knitr .Rtex files or any others)
+			isLaTeXFile = request.rootResourcePath?.match(/\.tex$/i)
+			if request.check? and isLaTeXFile
+				env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16'
+				env['CHKTEX_ULIMIT_OPTIONS'] = '-t 5 -v 64000'
+				if request.check is 'error'
+					env['CHKTEX_EXIT_ON_ERROR'] =  1
+				if request.check is 'validate'
+					env['CHKTEX_VALIDATE'] =  1
+
+			# apply a series of file modifications/creations for draft mode and tikz
+			async.series [injectDraftModeIfRequired, createTikzFileIfRequired], (error) ->
+				return callback(error) if error?
+				timer = new Metrics.Timer("run-compile")
+				# find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
+				tag = request.imageName?.match(/:(.*)/)?[1]?.replace(/\./g,'-') or "default"
+				tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test
+				Metrics.inc("compiles")
+				Metrics.inc("compiles-with-image.#{tag}")
+				compileName = getCompileName(request.project_id, request.user_id)
+				LatexRunner.runLatex compileName, {
+					directory: compileDir
+					mainFile:  request.rootResourcePath
+					compiler:  request.compiler
+					timeout:   request.timeout
+					image:     request.imageName
+					flags:     request.flags
+					environment: env
+				}, (error, output, stats, timings) ->
+					# request was for validation only
+					if request.check is "validate"
+						result = if error?.code then "fail" else "pass"
+						error = new Error("validation")
+						error.validate = result
+					# request was for compile, and failed on validation
+					if request.check is "error" and error?.message is 'exited'
+						error = new Error("compilation")
+						error.validate = "fail"
+					# compile was killed by user, was a validation, or a compile which failed validation
+					if error?.terminated or error?.validate
+						OutputFileFinder.findOutputFiles resourceList, compileDir, (err, outputFiles) ->
+							return callback(err) if err?
+							callback(error, outputFiles) # return output files so user can check logs
+						return
+					# compile completed normally
+					return callback(error) if error?
+					Metrics.inc("compiles-succeeded")
+					for metric_key, metric_value of stats or {}
+						Metrics.count(metric_key, metric_value)
+					for metric_key, metric_value of timings or {}
+						Metrics.timing(metric_key, metric_value)
+					loadavg = os.loadavg?()
+					Metrics.gauge("load-avg", loadavg[0]) if loadavg?
+					ts = timer.done()
+					logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile"
+					if stats?["latex-runs"] > 0
+						Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"])
+					if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0
+						Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"])
+
+					OutputFileFinder.findOutputFiles resourceList, compileDir, (error, outputFiles) ->
+						return callback(error) if error?
+						OutputCacheManager.saveOutputFiles outputFiles, compileDir,  (error, newOutputFiles) ->
+							callback null, newOutputFiles
+
+	stopCompile: (project_id, user_id, callback = (error) ->) ->
+		compileName = getCompileName(project_id, user_id)
+		LatexRunner.killLatex compileName, callback
+
+	clearProject: (project_id, user_id, _callback = (error) ->) ->
 		callback = (error) ->
 			_callback(error)
 			_callback = () ->
 
-		compileDir = Path.join(Settings.path.compilesDir, project_id)
-		proc = child_process.spawn "rm", ["-r", compileDir]
+		compileDir = getCompileDir(project_id, user_id)
 
-		proc.on "error", callback
+		CompileManager._checkDirectory compileDir, (err, exists) ->
+			return callback(err) if err?
+			return callback() if not exists # skip removal if no directory present
 
-		stderr = ""
-		proc.stderr.on "data", (chunk) -> stderr += chunk.toString()
+			proc = child_process.spawn "rm", ["-r", compileDir]
 
-		proc.on "close", (code) ->
-			if code == 0
-				return callback(null)
+			proc.on "error", callback
+
+			stderr = ""
+			proc.stderr.on "data", (chunk) -> stderr += chunk.toString()
+
+			proc.on "close", (code) ->
+				if code == 0
+					return callback(null)
+				else
+					return callback(new Error("rm -r #{compileDir} failed: #{stderr}"))
+
+	_findAllDirs: (callback = (error, allDirs) ->) ->
+		root = Settings.path.compilesDir
+		fs.readdir root, (err, files) ->
+			return callback(err) if err?
+			allDirs = (Path.join(root, file) for file in files)
+			callback(null, allDirs)
+
+	clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) ->
+		now = Date.now()
+		# action for each directory
+		expireIfNeeded = (checkDir, cb) ->
+			fs.stat checkDir, (err, stats) ->
+				return cb() if err?  # ignore errors checking directory
+				age = now - stats.mtime
+				hasExpired = (age > max_cache_age_ms)
+				if hasExpired then fse.remove(checkDir, cb) else cb()
+		# iterate over all project directories
+		CompileManager._findAllDirs (error, allDirs) ->
+			return callback() if error?
+			async.eachSeries allDirs, expireIfNeeded,	callback
+
+	_checkDirectory: (compileDir, callback = (error, exists) ->) ->
+		fs.lstat compileDir, (err, stats) ->
+			if err?.code is 'ENOENT'
+				return callback(null, false) #  directory does not exist
+			else if err?
+				logger.err {dir: compileDir, err:err}, "error on stat of project directory for removal"
+				return callback(err)
+			else if not stats?.isDirectory()
+				logger.err {dir: compileDir, stats:stats}, "bad project directory for removal"
+				return callback new Error("project directory is not directory")
 			else
-				return callback(new Error("rm -r #{compileDir} failed: #{stderr}"))
+				callback(null, true) # directory exists
 
-	syncFromCode: (project_id, file_name, line, column, callback = (error, pdfPositions) ->) ->
+	syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) ->
 		# If LaTeX was run in a virtual environment, the file path that synctex expects
 		# might not match the file path on the host. The .synctex.gz file however, will be accessed
 		# wherever it is on the host.
-		base_dir = Settings.path.synctexBaseDir(project_id)
+		compileName = getCompileName(project_id, user_id)
+		base_dir = Settings.path.synctexBaseDir(compileName)
 		file_path = base_dir + "/" + file_name
-		synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf")
-		CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
-			return callback(error) if error?
-			logger.log project_id: project_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
-			callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
+		compileDir = getCompileDir(project_id, user_id)
+		synctex_path =  "#{base_dir}/output.pdf"
+		command = ["code", synctex_path, file_path, line, column]
+		fse.ensureDir compileDir, (error) ->
+			if error?
+				logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code"
+				return callback(error)
+			CompileManager._runSynctex project_id, user_id, command, (error, stdout) ->
+				return callback(error) if error?
+				logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, command:command, stdout: stdout, "synctex code output"
+				callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
 
-	syncFromPdf: (project_id, page, h, v, callback = (error, filePositions) ->) ->
-		base_dir = Settings.path.synctexBaseDir(project_id)
-		synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf")
-		CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
-			return callback(error) if error?
-			logger.log project_id: project_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
-			callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
+	syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
+		compileName = getCompileName(project_id, user_id)
+		compileDir = getCompileDir(project_id, user_id)
+		base_dir = Settings.path.synctexBaseDir(compileName)
+		synctex_path =  "#{base_dir}/output.pdf"
+		command = ["pdf", synctex_path, page, h, v]
+		fse.ensureDir compileDir, (error) ->
+			if error?
+				logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync to code"
+				return callback(error)
+			CompileManager._runSynctex  project_id, user_id, command, (error, stdout) ->
+				return callback(error) if error?
+				logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
+				callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
 
-	_runSynctex: (args, callback = (error, stdout) ->) ->
-		bin_path = Path.resolve(__dirname + "/../../bin/synctex")
+	_checkFileExists: (path, callback = (error) ->) ->
+		synctexDir = Path.dirname(path)
+		synctexFile = Path.join(synctexDir, "output.synctex.gz")
+		fs.stat synctexDir, (error, stats) ->
+			if error?.code is 'ENOENT'
+				return callback(new Errors.NotFoundError("called synctex with no output directory"))
+			return callback(error) if error?
+			fs.stat synctexFile, (error, stats) ->
+				if error?.code is 'ENOENT'
+					return callback(new Errors.NotFoundError("called synctex with no output file"))
+				return callback(error) if error?
+				return callback(new Error("not a file")) if not stats?.isFile()
+				callback()
+
+	_runSynctex: (project_id, user_id, command, callback = (error, stdout) ->) ->
 		seconds = 1000
-		child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) ->
-			return callback(error) if error?
-			callback(null, stdout)
+
+		command.unshift("/opt/synctex")
+
+		directory = getCompileDir(project_id, user_id)
+		timeout = 60 * 1000 # increased to allow for large projects
+		compileName = getCompileName(project_id, user_id)
+		CommandRunner.run compileName, command, directory, Settings.clsi?.docker.image, timeout, {}, (error, output) ->
+			if error?
+				logger.err err:error, command:command, project_id:project_id, user_id:user_id, "error running synctex"
+				return callback(error)
+			callback(null, output.stdout)
 
 	_parseSynctexFromCodeOutput: (output) ->
 		results = []
@@ -111,17 +283,28 @@ module.exports = CompileManager =
 				}
 		return results
 
-	wordcount: (project_id, file_name, callback = (error, pdfPositions) ->) ->
-		logger.log project_id:project_id, file_name:file_name, "running wordcount"
-		file_path = "$COMPILE_DIR/" + file_name
-		command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"]
-		directory = Path.join(Settings.path.compilesDir, project_id)
-		timeout = 10 * 1000
 
-		CommandRunner.run project_id, command, directory, timeout, (error) ->
-			return callback(error) if error?
-			stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8")
-			callback null, CompileManager._parseWordcountFromOutput(stdout)
+	wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) ->
+		logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount"
+		file_path = "$COMPILE_DIR/" + file_name
+		command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"]
+		compileDir = getCompileDir(project_id, user_id)
+		timeout = 60 * 1000
+		compileName = getCompileName(project_id, user_id)
+		fse.ensureDir compileDir, (error) ->
+			if error?
+				logger.err {error, project_id, user_id, file_name}, "error ensuring dir for sync from code"
+				return callback(error)
+			CommandRunner.run compileName, command, compileDir, image, timeout, {}, (error) ->
+				return callback(error) if error?
+				fs.readFile compileDir + "/" + file_name + ".wc", "utf-8", (err, stdout) ->
+					if err?
+						#call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
+						logger.err node_err:err, command:command, compileDir:compileDir, project_id:project_id, user_id:user_id, "error reading word count output"
+						return callback(err)
+					results = CompileManager._parseWordcountFromOutput(stdout)
+					logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results"
+					callback null, results
 
 	_parseWordcountFromOutput: (output) ->
 		results = {
@@ -133,6 +316,8 @@ module.exports = CompileManager =
 			elements: 0
 			mathInline: 0
 			mathDisplay: 0
+			errors: 0
+			messages: ""
 		}
 		for line in output.split("\n")
 			[data, info] = line.split(":")
@@ -152,4 +337,8 @@ module.exports = CompileManager =
 				results['mathInline'] = parseInt(info, 10)
 			if data.indexOf("Number of math displayed") > -1
 				results['mathDisplay'] = parseInt(info, 10)
+			if data is "(errors"  # errors reported as (errors:123)
+				results['errors'] = parseInt(info, 10)
+			if line.indexOf("!!! ") > -1  # errors logged as !!! message !!!
+				results['messages'] += line + "\n"
 		return results

app/coffee/ContentTypeMapper.coffee

@@ -0,0 +1,24 @@
+Path = require 'path'
+
+# here we coerce html, css and js to text/plain,
+# otherwise choose correct mime type based on file extension,
+# falling back to octet-stream
+module.exports = ContentTypeMapper =
+	map: (path) ->
+		switch Path.extname(path)
+			when '.txt', '.html', '.js', '.css', '.svg'
+				return 'text/plain'
+			when '.csv'
+				return 'text/csv'
+			when '.pdf'
+				return 'application/pdf'
+			when '.png'
+				return 'image/png'
+			when '.jpg', '.jpeg'
+				return 'image/jpeg'
+			when '.tiff'
+				return 'image/tiff'
+			when '.gif'
+				return 'image/gif'
+			else
+				return 'application/octet-stream'

app/coffee/DbQueue.coffee

@@ -0,0 +1,13 @@
+async = require "async"
+Settings = require "settings-sharelatex"
+logger = require("logger-sharelatex")
+queue = async.queue((task, cb)->
+		task(cb)
+	, Settings.parallelSqlQueryLimit)
+
+queue.drain = ()->
+	logger.debug('all items have been processed')
+
+module.exports = 
+	queue: queue
+

app/coffee/DockerLockManager.coffee

@@ -0,0 +1,56 @@
+logger = require "logger-sharelatex"
+
+LockState = {} # locks for docker container operations, by container name
+
+module.exports = LockManager =
+
+	MAX_LOCK_HOLD_TIME: 15000  # how long we can keep a lock
+	MAX_LOCK_WAIT_TIME: 10000  # how long we wait for a lock
+	LOCK_TEST_INTERVAL: 1000   # retry time
+
+	tryLock: (key, callback = (err, gotLock) ->) ->
+		existingLock = LockState[key]
+		if existingLock? # the lock is already taken, check how old it is
+			lockAge = Date.now() - existingLock.created
+			if lockAge < LockManager.MAX_LOCK_HOLD_TIME
+				return callback(null, false) # we didn't get the lock, bail out
+			else
+				logger.error {key: key, lock: existingLock, age:lockAge}, "taking old lock by force"
+		# take the lock
+		LockState[key] = lockValue = {created: Date.now()}
+		callback(null, true, lockValue)
+
+	getLock: (key, callback = (error, lockValue) ->) ->
+		startTime = Date.now()
+		do attempt = () ->
+			LockManager.tryLock key, (error, gotLock, lockValue) ->
+				return callback(error) if error?
+				if gotLock
+					callback(null, lockValue)
+				else if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
+					e = new Error("Lock timeout")
+					e.key = key
+					return callback(e)
+				else
+					setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
+
+	releaseLock: (key, lockValue, callback = (error) ->) ->
+		existingLock = LockState[key]
+		if existingLock is lockValue # lockValue is an object, so we can test by reference
+			delete LockState[key] # our lock, so we can free it
+			callback()
+		else if existingLock? # lock exists but doesn't match ours
+			logger.error {key:key, lock: existingLock}, "tried to release lock taken by force"
+			callback()
+		else
+			logger.error {key:key, lock: existingLock}, "tried to release lock that has gone"
+			callback()
+
+	runWithLock: (key, runner = ( (releaseLock = (error) ->) -> ), callback = ( (error) -> )) ->
+		LockManager.getLock key, (error, lockValue) ->
+			return callback(error) if error?
+			runner (error1, args...) ->
+				LockManager.releaseLock key, lockValue, (error2) ->
+					error = error1 or error2
+					return callback(error) if error?
+					callback(null, args...)

app/coffee/DockerRunner.coffee

@@ -0,0 +1,358 @@
+Settings = require "settings-sharelatex"
+logger = require "logger-sharelatex"
+Docker = require("dockerode")
+dockerode = new Docker()
+crypto = require "crypto"
+async = require "async"
+LockManager = require "./DockerLockManager"
+fs = require "fs"
+Path = require 'path'
+_ = require "underscore"
+
+logger.info "using docker runner"
+
+usingSiblingContainers = () ->
+	Settings?.path?.sandboxedCompilesHostDir?
+
+module.exports = DockerRunner =
+	ERR_NOT_DIRECTORY: new Error("not a directory")
+	ERR_TERMINATED: new Error("terminated")
+	ERR_EXITED: new Error("exited")
+	ERR_TIMED_OUT: new Error("container timed out")
+
+	run: (project_id, command, directory, image, timeout, environment, callback = (error, output) ->) ->
+
+		if usingSiblingContainers()
+			_newPath = Settings.path.sandboxedCompilesHostDir
+			logger.log {path: _newPath}, "altering bind path for sibling containers"
+			# Server Pro, example:
+			#   '/var/lib/sharelatex/data/compiles/<project-id>'
+			#   ... becomes ...
+			#   '/opt/sharelatex_data/data/compiles/<project-id>'
+			directory = Path.join(Settings.path.sandboxedCompilesHostDir, Path.basename(directory))
+
+		volumes = {}
+		volumes[directory] = "/compile"
+
+		command = (arg.toString().replace?('$COMPILE_DIR', "/compile") for arg in command)
+		if !image?
+			image = Settings.clsi.docker.image
+
+		if Settings.texliveImageNameOveride?
+			img = image.split("/")
+			image = "#{Settings.texliveImageNameOveride}/#{img[2]}"
+
+		options      = DockerRunner._getContainerOptions(command, image, volumes, timeout, environment)
+		fingerprint  = DockerRunner._fingerprintContainer(options)
+		options.name = name = "project-#{project_id}-#{fingerprint}"
+
+		# logOptions = _.clone(options)
+		# logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
+		logger.log project_id: project_id, "running docker container"
+		DockerRunner._runAndWaitForContainer options, volumes, timeout, (error, output) ->
+			if error?.message?.match("HTTP code is 500")
+				logger.log err: error, project_id: project_id, "error running container so destroying and retrying"
+				DockerRunner.destroyContainer name, null, true, (error) ->
+					return callback(error) if error?
+					DockerRunner._runAndWaitForContainer options, volumes, timeout, callback
+			else
+				callback(error, output)
+
+		return name # pass back the container name to allow it to be killed
+
+	kill: (container_id, callback = (error) ->) ->
+		logger.log container_id: container_id, "sending kill signal to container"
+		container = dockerode.getContainer(container_id)
+		container.kill (error) ->
+			if error? and error?.message?.match?(/Cannot kill container .* is not running/)
+				logger.warn err: error, container_id: container_id, "container not running, continuing"
+				error = null
+			if error?
+				logger.error err: error, container_id: container_id, "error killing container"
+				return callback(error)
+			else
+				callback()
+
+	_runAndWaitForContainer: (options, volumes, timeout, _callback = (error, output) ->) ->
+		callback = (args...) ->
+			_callback(args...)
+			# Only call the callback once
+			_callback = () ->
+		
+		name = options.name
+
+		streamEnded = false
+		containerReturned = false
+		output = {}
+
+		callbackIfFinished = () ->
+			if streamEnded and containerReturned
+				callback(null, output)
+
+		attachStreamHandler = (error, _output) ->
+			return callback(error) if error?
+			output = _output
+			streamEnded = true
+			callbackIfFinished()
+
+		DockerRunner.startContainer options, volumes, attachStreamHandler, (error, containerId) ->
+			return callback(error) if error?
+			
+			DockerRunner.waitForContainer name, timeout, (error, exitCode) ->
+				return callback(error) if error?
+				if exitCode is 137  # exit status from kill -9
+					err = DockerRunner.ERR_TERMINATED
+					err.terminated = true
+					return callback(err)
+				if exitCode is 1 # exit status from chktex
+					err = DockerRunner.ERR_EXITED
+					err.code = exitCode
+					return callback(err)
+				containerReturned = true
+				options?.HostConfig?.SecurityOpt = null #small log line
+				logger.log err:err, exitCode:exitCode, options:options, "docker container has exited"
+				callbackIfFinished()
+
+	_getContainerOptions: (command, image, volumes, timeout, environment) ->
+		timeoutInSeconds = timeout / 1000
+		
+		dockerVolumes = {}
+		for hostVol, dockerVol of volumes
+			dockerVolumes[dockerVol] = {}
+
+			if volumes[hostVol].slice(-3).indexOf(":r") == -1
+				volumes[hostVol] = "#{dockerVol}:rw"
+
+		# merge settings and environment parameter
+		env = {}
+		for src in [Settings.clsi.docker.env, environment or {}]
+			env[key] = value for key, value of src
+		# set the path based on the image year
+		if m = image.match /:([0-9]+)\.[0-9]+/
+			year = m[1]
+		else
+			year = "2014"
+		env['PATH'] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/#{year}/bin/x86_64-linux/"
+		options =
+			"Cmd"             : command,
+			"Image"           : image
+			"Volumes"         : dockerVolumes
+			"WorkingDir"      : "/compile"
+			"NetworkDisabled" : true
+			"Memory"          : 1024 * 1024 * 1024 * 1024 # 1 Gb
+			"User"            : Settings.clsi.docker.user
+			"Env"             : ("#{key}=#{value}" for key, value of env)	# convert the environment hash to an array
+			"HostConfig"      :
+				"Binds": ("#{hostVol}:#{dockerVol}" for hostVol, dockerVol of volumes)
+				"LogConfig": {"Type": "none", "Config": {}}
+				"Ulimits": [{'Name': 'cpu', 'Soft': timeoutInSeconds+5, 'Hard': timeoutInSeconds+10}]
+				"CapDrop": "ALL"
+				"SecurityOpt": ["no-new-privileges"]
+		
+
+		if Settings.path?.synctexBinHostPath?
+			options["HostConfig"]["Binds"].push("#{Settings.path.synctexBinHostPath}:/opt/synctex:ro")
+
+		if Settings.clsi.docker.seccomp_profile?
+			options.HostConfig.SecurityOpt.push "seccomp=#{Settings.clsi.docker.seccomp_profile}"
+
+		return options
+
+	_fingerprintContainer: (containerOptions) ->
+		# Yay, Hashing!
+		json = JSON.stringify(containerOptions)
+		return crypto.createHash("md5").update(json).digest("hex")
+
+	startContainer: (options, volumes, attachStreamHandler, callback) ->
+		LockManager.runWithLock options.name, (releaseLock) ->
+			# Check that volumes exist before starting the container.
+			# When a container is started with volume pointing to a
+			# non-existent directory then docker creates the directory but
+			# with root ownership.
+			DockerRunner._checkVolumes options, volumes, (err) ->
+				return releaseLock(err) if err?
+				DockerRunner._startContainer options, volumes, attachStreamHandler, releaseLock
+		, callback
+
+	# Check that volumes exist and are directories
+	_checkVolumes: (options, volumes, callback = (error, containerName) ->) ->
+		if usingSiblingContainers()
+			# Server Pro, with sibling-containers active, skip checks
+			return callback(null)
+
+		checkVolume = (path, cb) ->
+			fs.stat path, (err, stats) ->
+				return cb(err) if err?
+				return cb(DockerRunner.ERR_NOT_DIRECTORY) if not stats?.isDirectory()
+				cb()
+		jobs = []
+		for vol of volumes
+			do (vol) ->
+				jobs.push (cb) -> checkVolume(vol, cb)
+		async.series jobs, callback
+
+	_startContainer: (options, volumes, attachStreamHandler, callback = ((error, output) ->)) ->
+		callback = _.once(callback)
+		name = options.name
+
+		logger.log {container_name: name}, "starting container"
+		container = dockerode.getContainer(name)
+
+		createAndStartContainer = ->
+			dockerode.createContainer options, (error, container) ->
+				return callback(error) if error?
+				startExistingContainer()
+
+		startExistingContainer = ->
+			DockerRunner.attachToContainer options.name, attachStreamHandler, (error)->
+				return callback(error) if error?
+				container.start (error) ->
+					if error? and error?.statusCode != 304 #already running
+						return callback(error)
+					else
+						callback()
+
+		container.inspect (error, stats)->
+			if error?.statusCode == 404
+				createAndStartContainer()
+			else if error?
+				logger.err {container_name: name, error:error}, "unable to inspect container to start"
+				return callback(error)
+			else
+				startExistingContainer()
+
+
+	attachToContainer: (containerId, attachStreamHandler, attachStartCallback) ->
+		container = dockerode.getContainer(containerId)
+		container.attach {stdout: 1, stderr: 1, stream: 1}, (error, stream) ->
+			if error?
+				logger.error err: error, container_id: containerId, "error attaching to container"
+				return attachStartCallback(error)
+			else
+				attachStartCallback()
+
+
+			logger.log container_id: containerId, "attached to container"
+
+			MAX_OUTPUT = 1024 * 1024 # limit output to 1MB
+			createStringOutputStream = (name) ->
+				return {
+					data: ""
+					overflowed: false
+					write: (data) ->
+						return if @overflowed
+						if @data.length < MAX_OUTPUT
+							@data += data
+						else
+							logger.error container_id: containerId, length: @data.length, maxLen: MAX_OUTPUT, "#{name} exceeds max size"
+							@data += "(...truncated at #{MAX_OUTPUT} chars...)"
+							@overflowed = true
+							# kill container if too much output
+							# docker.containers.kill(containerId, () ->)
+				}
+
+			stdout = createStringOutputStream "stdout"
+			stderr = createStringOutputStream "stderr"
+
+			container.modem.demuxStream(stream, stdout, stderr)
+
+			stream.on "error", (err) ->
+				logger.error err: err, container_id: containerId, "error reading from container stream"
+
+			stream.on "end", () ->
+				attachStreamHandler null, {stdout: stdout.data, stderr: stderr.data}
+
+	waitForContainer: (containerId, timeout, _callback = (error, exitCode) ->) ->
+		callback = (args...) ->
+			_callback(args...)
+			# Only call the callback once
+			_callback = () ->
+
+		container = dockerode.getContainer(containerId)
+
+		timedOut = false
+		timeoutId = setTimeout () ->
+			timedOut = true
+			logger.log container_id: containerId, "timeout reached, killing container"
+			container.kill(() ->)
+		, timeout
+			
+		logger.log container_id: containerId, "waiting for docker container"
+		container.wait (error, res) ->
+			if error?
+				clearTimeout timeoutId
+				logger.error err: error, container_id: containerId, "error waiting for container"
+				return callback(error)
+			if timedOut
+				logger.log containerId: containerId, "docker container timed out"
+				error = DockerRunner.ERR_TIMED_OUT
+				error.timedout = true
+				callback error
+			else
+				clearTimeout timeoutId
+				logger.log container_id: containerId, exitCode: res.StatusCode, "docker container returned"
+				callback null, res.StatusCode
+
+	destroyContainer: (containerName, containerId, shouldForce, callback = (error) ->) ->
+		# We want the containerName for the lock and, ideally, the
+		# containerId to delete.  There is a bug in the docker.io module
+		# where if you delete by name and there is an error, it throws an
+		# async exception, but if you delete by id it just does a normal
+		# error callback. We fall back to deleting by name if no id is
+		# supplied.
+		LockManager.runWithLock containerName, (releaseLock) ->
+			DockerRunner._destroyContainer containerId or containerName, shouldForce, releaseLock
+		, callback
+
+	_destroyContainer: (containerId, shouldForce, callback = (error) ->) ->
+		logger.log container_id: containerId, "destroying docker container"
+		container = dockerode.getContainer(containerId)
+		container.remove {force: shouldForce == true}, (error) ->
+			if error? and error?.statusCode == 404
+				logger.warn err: error, container_id: containerId, "container not found, continuing"
+				error = null
+			if error?
+				logger.error err: error, container_id: containerId, "error destroying container"
+			else
+				logger.log container_id: containerId, "destroyed container"
+			callback(error)
+
+	# handle expiry of docker containers
+
+	MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge or oneHour = 60 * 60 * 1000
+
+	examineOldContainer: (container, callback = (error, name, id, ttl)->) ->
+		name = container.Name or container.Names?[0]
+		created = container.Created * 1000 # creation time is returned in seconds
+		now = Date.now()
+		age = now - created
+		maxAge = DockerRunner.MAX_CONTAINER_AGE
+		ttl = maxAge - age
+		logger.log {containerName: name, created: created, now: now, age: age, maxAge: maxAge, ttl: ttl}, "checking whether to destroy container"
+		callback(null, name, container.Id, ttl)
+
+	destroyOldContainers: (callback = (error) ->) ->
+		dockerode.listContainers all: true, (error, containers) ->
+			return callback(error) if error?
+			jobs = []
+			for container in containers or []
+				do (container) ->
+					DockerRunner.examineOldContainer container, (err, name, id, ttl) ->
+						if name.slice(0, 9) == '/project-' && ttl <= 0
+							jobs.push	(cb) ->
+								DockerRunner.destroyContainer name, id, false, () -> cb()
+								# Ignore errors because some containers get stuck but
+								# will be destroyed next time
+			async.series jobs, callback
+
+	startContainerMonitor: () ->
+		logger.log {maxAge: DockerRunner.MAX_CONTAINER_AGE}, "starting container expiry"
+		# randomise the start time
+		randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
+		setTimeout () ->
+			setInterval () ->
+				DockerRunner.destroyOldContainers()
+			, oneHour = 60 * 60 * 1000
+		, randomDelay
+
+DockerRunner.startContainerMonitor()

app/coffee/DraftModeManager.coffee

@@ -0,0 +1,24 @@
+fs = require "fs"
+logger = require "logger-sharelatex"
+
+module.exports = DraftModeManager =
+	injectDraftMode: (filename, callback = (error) ->) ->
+		fs.readFile filename, "utf8", (error, content) ->
+			return callback(error) if error?
+			# avoid adding draft mode more than once
+			if content?.indexOf("\\documentclass\[draft") >= 0
+				return callback()
+			modified_content = DraftModeManager._injectDraftOption content
+			logger.log {
+				content: content.slice(0,1024), # \documentclass is normally v near the top
+				modified_content: modified_content.slice(0,1024),
+				filename
+			}, "injected draft class"
+			fs.writeFile filename, modified_content, callback
+	
+	_injectDraftOption: (content) ->
+		content
+			# With existing options (must be first, otherwise both are applied)
+			.replace(/\\documentclass\[/g, "\\documentclass[draft,")
+			# Without existing options
+			.replace(/\\documentclass\{/g, "\\documentclass[draft]{")

app/coffee/Errors.coffee

@@ -0,0 +1,25 @@
+NotFoundError = (message) ->
+	error = new Error(message)
+	error.name = "NotFoundError"
+	error.__proto__ = NotFoundError.prototype
+	return error
+NotFoundError.prototype.__proto__ = Error.prototype
+
+FilesOutOfSyncError = (message) ->
+	error = new Error(message)
+	error.name = "FilesOutOfSyncError"
+	error.__proto__ = FilesOutOfSyncError.prototype
+	return error
+FilesOutOfSyncError.prototype.__proto__ = Error.prototype
+
+AlreadyCompilingError = (message) ->
+	error = new Error(message)
+	error.name = "AlreadyCompilingError"
+	error.__proto__ = AlreadyCompilingError.prototype
+	return error
+AlreadyCompilingError.prototype.__proto__ = Error.prototype
+
+module.exports = Errors =
+	NotFoundError: NotFoundError
+	FilesOutOfSyncError: FilesOutOfSyncError
+	AlreadyCompilingError: AlreadyCompilingError

app/coffee/LatexRunner.coffee

@@ -2,56 +2,94 @@ Path = require "path"
 Settings = require "settings-sharelatex"
 logger = require "logger-sharelatex"
 Metrics = require "./Metrics"
-CommandRunner = require(Settings.clsi?.commandRunner or "./CommandRunner")
+CommandRunner = require "./CommandRunner"
+
+ProcessTable = {}  # table of currently running jobs (pids or docker container names)
 
 module.exports = LatexRunner =
 	runLatex: (project_id, options, callback = (error) ->) ->
-		{directory, mainFile, compiler, timeout} = options
+		{directory, mainFile, compiler, timeout, image, environment, flags} = options
 		compiler ||= "pdflatex"
 		timeout  ||= 60000 # milliseconds
 
-		logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, "starting compile"
+		logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, environment: environment, flags:flags, "starting compile"
 
 		# We want to run latexmk on the tex file which we will automatically
 		# generate from the Rtex/Rmd/md file.
 		mainFile = mainFile.replace(/\.(Rtex|md|Rmd)$/, ".tex")
 
 		if compiler == "pdflatex"
-			command = LatexRunner._pdflatexCommand mainFile
+			command = LatexRunner._pdflatexCommand mainFile, flags
 		else if compiler == "latex"
-			command = LatexRunner._latexCommand mainFile
+			command = LatexRunner._latexCommand mainFile, flags
 		else if compiler == "xelatex"
-			command = LatexRunner._xelatexCommand mainFile
+			command = LatexRunner._xelatexCommand mainFile, flags
 		else if compiler == "lualatex"
-			command = LatexRunner._lualatexCommand mainFile
+			command = LatexRunner._lualatexCommand mainFile, flags
 		else
 			return callback new Error("unknown compiler: #{compiler}")
 
-		CommandRunner.run project_id, command, directory, timeout, callback
+		if Settings.clsi?.strace
+			command = ["strace", "-o", "strace", "-ff"].concat(command)
 
-	_latexmkBaseCommand: [ "latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR"]
+		id = "#{project_id}" # record running project under this id
 
-	_pdflatexCommand: (mainFile) ->
-		LatexRunner._latexmkBaseCommand.concat [
-			"-pdf", "-e", "$pdflatex='pdflatex -synctex=1 -interaction=batchmode %O %S'",
+		ProcessTable[id] = CommandRunner.run project_id, command, directory, image, timeout, environment, (error, output) ->
+			delete ProcessTable[id]
+			return callback(error) if error?
+			runs = output?.stderr?.match(/^Run number \d+ of .*latex/mg)?.length or 0
+			failed = if output?.stdout?.match(/^Latexmk: Errors/m)? then 1 else 0
+			# counters from latexmk output
+			stats = {}
+			stats["latexmk-errors"] = failed
+			stats["latex-runs"] = runs
+			stats["latex-runs-with-errors"] = if failed then runs else 0
+			stats["latex-runs-#{runs}"] = 1
+			stats["latex-runs-with-errors-#{runs}"] = if failed then 1 else 0
+			# timing information from /usr/bin/time
+			timings = {}
+			stderr = output?.stderr
+			timings["cpu-percent"] = stderr?.match(/Percent of CPU this job got: (\d+)/m)?[1] or 0
+			timings["cpu-time"] = stderr?.match(/User time.*: (\d+.\d+)/m)?[1] or 0
+			timings["sys-time"] = stderr?.match(/System time.*: (\d+.\d+)/m)?[1] or 0
+			callback error, output, stats, timings
+
+	killLatex: (project_id, callback = (error) ->) ->
+		id = "#{project_id}"
+		logger.log {id:id}, "killing running compile"
+		if not ProcessTable[id]?
+			logger.warn {id}, "no such project to kill"
+			return callback(null)
+		else
+			CommandRunner.kill ProcessTable[id], callback
+
+	_latexmkBaseCommand: (flags) ->
+		args = ["latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR", "-synctex=1","-interaction=batchmode"]
+		if flags
+			args = args.concat(flags)
+		(Settings?.clsi?.latexmkCommandPrefix || []).concat(args)
+
+	_pdflatexCommand: (mainFile, flags) ->
+		LatexRunner._latexmkBaseCommand(flags).concat [
+			"-pdf",
 			Path.join("$COMPILE_DIR", mainFile)
 		]
 
-	_latexCommand: (mainFile) ->
-		LatexRunner._latexmkBaseCommand.concat [
-			"-pdfdvi", "-e", "$latex='latex -synctex=1 -interaction=batchmode %O %S'",
+	_latexCommand: (mainFile, flags) ->
+		LatexRunner._latexmkBaseCommand(flags).concat [
+			"-pdfdvi",
 			Path.join("$COMPILE_DIR", mainFile)
 		]
 
-	_xelatexCommand: (mainFile) ->
-		LatexRunner._latexmkBaseCommand.concat [
-			"-xelatex", "-e", "$pdflatex='xelatex -synctex=1 -interaction=batchmode %O %S'",
+	_xelatexCommand: (mainFile, flags) ->
+		LatexRunner._latexmkBaseCommand(flags).concat [
+			"-xelatex",
 			Path.join("$COMPILE_DIR", mainFile)
 		]
 
-	_lualatexCommand: (mainFile) ->
-		LatexRunner._latexmkBaseCommand.concat [
-			"-pdf", "-e", "$pdflatex='lualatex -synctex=1 -interaction=batchmode %O %S'",
+	_lualatexCommand: (mainFile, flags) ->
+		LatexRunner._latexmkBaseCommand(flags).concat [
+			"-lualatex",
 			Path.join("$COMPILE_DIR", mainFile)
 		]
 

app/coffee/LocalCommandRunner.coffee

@@ -0,0 +1,48 @@
+spawn = require("child_process").spawn
+logger = require "logger-sharelatex"
+
+logger.info "using standard command runner"
+
+module.exports = CommandRunner =
+	run: (project_id, command, directory, image, timeout, environment, callback = (error) ->) ->
+		command = (arg.toString().replace('$COMPILE_DIR', directory) for arg in command)
+		logger.log project_id: project_id, command: command, directory: directory, "running command"
+		logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
+
+		# merge environment settings
+		env = {}
+		env[key] = value for key, value of process.env
+		env[key] = value for key, value of environment
+
+		# run command as detached process so it has its own process group (which can be killed if needed)
+		proc = spawn command[0], command.slice(1), cwd: directory, env: env
+
+		stdout = ""
+		proc.stdout.on "data", (data)->
+			stdout += data
+
+		proc.on "error", (err)->
+			logger.err err:err, project_id:project_id, command: command, directory: directory, "error running command"
+			callback(err)
+
+		proc.on "close", (code, signal) ->
+			logger.info code:code, signal:signal, project_id:project_id, "command exited"
+			if signal is 'SIGTERM' # signal from kill method below
+				err = new Error("terminated")
+				err.terminated = true
+				return callback(err)
+			else if code is 1 # exit status from chktex
+				err = new Error("exited")
+				err.code = code
+				return callback(err)
+			else
+				callback(null, {"stdout": stdout})
+
+		return proc.pid # return process id to allow job to be killed if necessary
+
+	kill: (pid, callback = (error) ->) ->
+		try
+			process.kill -pid # kill all processes in group
+		catch err
+			return callback(err)
+		callback()

app/coffee/LockManager.coffee

@@ -0,0 +1,31 @@
+Settings = require('settings-sharelatex')
+logger = require "logger-sharelatex"
+Lockfile = require('lockfile') # from https://github.com/npm/lockfile
+Errors = require "./Errors"
+fs = require("fs")
+Path = require("path")
+module.exports = LockManager =
+	LOCK_TEST_INTERVAL: 1000 # 50ms between each test of the lock
+	MAX_LOCK_WAIT_TIME: 15000 # 10s maximum time to spend trying to get the lock
+	LOCK_STALE: 5*60*1000 # 5 mins time until lock auto expires
+
+	runWithLock: (path, runner = ((releaseLock = (error) ->) ->), callback = ((error) ->)) ->
+		lockOpts =
+			wait: @MAX_LOCK_WAIT_TIME
+			pollPeriod: @LOCK_TEST_INTERVAL
+			stale: @LOCK_STALE
+		Lockfile.lock path, lockOpts, (error) ->
+			if error?.code is 'EEXIST'
+				return callback new Errors.AlreadyCompilingError("compile in progress")
+			else if error?
+				fs.lstat path, (statLockErr, statLock)->
+					fs.lstat Path.dirname(path), (statDirErr, statDir)->
+						fs.readdir Path.dirname(path), (readdirErr, readdirDir)->
+							logger.err error:error, path:path, statLock:statLock, statLockErr:statLockErr, statDir:statDir, statDirErr: statDirErr, readdirErr:readdirErr, readdirDir:readdirDir, "unable to get lock"
+							return callback(error)
+			else
+				runner (error1, args...) ->
+					Lockfile.unlock path, (error2) ->
+						error = error1 or error2
+						return callback(error) if error?
+						callback(null, args...)

app/coffee/OutputCacheManager.coffee

@@ -4,12 +4,17 @@ fse = require "fs-extra"
 Path = require "path"
 logger = require "logger-sharelatex"
 _ = require "underscore"
+Settings = require "settings-sharelatex"
+crypto = require "crypto"
 
 OutputFileOptimiser = require "./OutputFileOptimiser"
 
 module.exports = OutputCacheManager =
 	CACHE_SUBDIR: '.cache/clsi'
-	BUILD_REGEX: /^[0-9a-f]+$/  # build id is Date.now() converted to hex
+	ARCHIVE_SUBDIR: '.archive/clsi'
+	# build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
+	# for backwards compatibility, make the randombytes part optional
+	BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/
 	CACHE_LIMIT: 2  # maximum number of cache directories
 	CACHE_AGE: 60*60*1000 # up to one hour old
 
@@ -21,40 +26,33 @@ module.exports = OutputCacheManager =
 			# for invalid build id, return top level
 			return file
 
+	generateBuildId: (callback = (error, buildId) ->) ->
+		# generate a secure build id from Date.now() and 8 random bytes in hex
+		crypto.randomBytes 8, (err, buf) ->
+			return callback(err) if err?
+			random = buf.toString('hex')
+			date = Date.now().toString(16)
+			callback err, "#{date}-#{random}"
+
 	saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) ->
+		OutputCacheManager.generateBuildId (err, buildId) ->
+			return callback(err) if err?
+			OutputCacheManager.saveOutputFilesInBuildDir outputFiles, compileDir, buildId, callback
+
+	saveOutputFilesInBuildDir: (outputFiles, compileDir, buildId, callback = (error) ->) ->
 		# make a compileDir/CACHE_SUBDIR/build_id directory and
 		# copy all the output files into it
 		cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
 		# Put the files into a new cache subdirectory
-		buildId = Date.now().toString(16)
 		cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
-		# let file expiry run in the background
-		OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId}
+		# Is it a per-user compile? check if compile directory is PROJECTID-USERID
+		perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/)
 
-		checkFile = (src, callback) ->
-			# check if we have a valid file to copy into the cache
-			fs.stat src, (err, stats) ->
+		# Archive logs in background
+		if Settings.clsi?.archive_logs or Settings.clsi?.strace
+			OutputCacheManager.archiveLogs outputFiles, compileDir, buildId, (err) ->
 				if err?
-					# some problem reading the file
-					logger.error err: err, file: src, "stat error for file in cache"
-					callback(err)
-				else if not stats.isFile()
-					# other filetype - reject it
-					logger.error err: err, src: src, dst: dst, stat: stats, "nonfile output - refusing to copy to cache"
-					callback(new Error("output file is not a file"), file)
-				else
-					# it's a plain file, ok to copy
-					callback(null)
-
-		copyFile = (src, dst, callback) ->
-			# copy output file into the cache
-			fse.copy src, dst, (err) ->
-				if err?
-					logger.error err: err, src: src, dst: dst, "copy error for file in cache"
-					callback(err)
-				else
-					# call the optimiser for the file too
-					OutputFileOptimiser.optimiseFile src, dst, callback
+					logger.warn err:err, "erroring archiving log files"
 
 		# make the new cache directory
 		fse.ensureDir cacheDir, (err) ->
@@ -63,21 +61,57 @@ module.exports = OutputCacheManager =
 				callback(err, outputFiles)
 			else
 				# copy all the output files into the new cache directory
+				results = []
 				async.mapSeries outputFiles, (file, cb) ->
+					# don't send dot files as output, express doesn't serve them
+					if OutputCacheManager._fileIsHidden(file.path)
+						logger.warn compileDir: compileDir, path: file.path, "ignoring dotfile in output"
+						return cb()
+					# copy other files into cache directory if valid
 					newFile = _.clone(file)
 					[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
-					checkFile src, (err) ->
-						copyFile src, dst, (err) ->
-							if not err?
+					OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
+						return cb(err) if err?
+						if !isSafe
+							return cb()
+						OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) ->
+							return cb(err) if err?
+							if !shouldCopy
+								return cb()
+							OutputCacheManager._copyFile src, dst, (err) ->
+								return cb(err) if err?
 								newFile.build = buildId  # attach a build id if we cached the file
-							cb(err, newFile)
-				, (err, results) ->
+								results.push newFile
+								cb()
+				, (err) ->
 					if err?
 						# pass back the original files if we encountered *any* error
 						callback(err, outputFiles)
+						# clean up the directory we just created
+						fse.remove cacheDir, (err) ->
+							if err?
+								logger.error err: err, dir: cacheDir, "error removing cache dir after failure"
 					else
 						# pass back the list of new files in the cache
 						callback(err, results)
+						# let file expiry run in the background, expire all previous files if per-user
+						OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 1 else null}
+
+	archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) ->
+		archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId)
+		logger.log {dir: archiveDir}, "archiving log files for project"
+		fse.ensureDir archiveDir, (err) ->
+			return callback(err) if err?
+			async.mapSeries outputFiles, (file, cb) ->
+				[src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]
+				OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
+					return cb(err) if err?
+					return cb() if !isSafe
+					OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) ->
+						return cb(err) if err?
+						return cb() if !shouldArchive
+						OutputCacheManager._copyFile src, dst, cb
+			, callback
 
 	expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
 		# look in compileDir for build dirs and delete if > N or age of mod time > T
@@ -92,10 +126,13 @@ module.exports = OutputCacheManager =
 
 			isExpired = (dir, index) ->
 				return false if options?.keep == dir
+				# remove any directories over the requested (non-null) limit
+				return true if options?.limit? and index > options.limit
 				# remove any directories over the hard limit
 				return true if index > OutputCacheManager.CACHE_LIMIT
-				# we can get the build time from the directory name
-				dirTime = parseInt(dir, 16)
+				# we can get the build time from the first part of the directory name DDDD-RRRR
+				# DDDD is date and RRRR is random bytes
+				dirTime = parseInt(dir.split('-')?[0], 16)
 				age = currentTime - dirTime
 				return age > OutputCacheManager.CACHE_AGE
 
@@ -111,3 +148,52 @@ module.exports = OutputCacheManager =
 			async.eachSeries toRemove, (dir, cb) ->
 				removeDir dir, cb
 			, callback
+
+	_fileIsHidden: (path) ->
+		return path?.match(/^\.|\/./)?
+
+	_checkFileIsSafe: (src, callback = (error, isSafe) ->) ->
+		# check if we have a valid file to copy into the cache
+		fs.stat src, (err, stats) ->
+			if err?.code is 'ENOENT'
+				logger.warn err: err, file: src, "file has disappeared before copying to build cache"
+				callback(err, false)
+			else if err?
+				# some other problem reading the file
+				logger.error err: err, file: src, "stat error for file in cache"
+				callback(err, false)
+			else if not stats.isFile()
+				# other filetype - reject it
+				logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache"
+				callback(null, false)
+			else
+				# it's a plain file, ok to copy
+				callback(null, true)
+
+	_copyFile: (src, dst, callback) ->
+		# copy output file into the cache
+		fse.copy src, dst, (err) ->
+			if err?.code is 'ENOENT'
+				logger.warn err: err, file: src, "file has disappeared when copying to build cache"
+				callback(err, false)
+			else if err?
+				logger.error err: err, src: src, dst: dst, "copy error for file in cache"
+				callback(err)
+			else
+				if Settings.clsi?.optimiseInDocker
+					# don't run any optimisations on the pdf when they are done
+					# in the docker container
+					callback()
+				else
+					# call the optimiser for the file too
+					OutputFileOptimiser.optimiseFile src, dst, callback
+
+	_checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) ->
+		return callback(null, !Path.basename(src).match(/^strace/))
+
+	_checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) ->
+		if Path.basename(src).match(/^strace/)
+			return callback(null, true)
+		if Settings.clsi?.archive_logs and Path.basename(src) in ["output.log", "output.blg"]
+			return callback(null, true)
+		return callback(null, false)

app/coffee/OutputFileFinder.coffee

@@ -5,16 +5,15 @@ spawn = require("child_process").spawn
 logger = require "logger-sharelatex"
 
 module.exports = OutputFileFinder =
-	findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
+	findOutputFiles: (resources, directory, callback = (error, outputFiles, allFiles) ->) ->
 		incomingResources = {}
 		for resource in resources
 			incomingResources[resource.path] = true
 			
-		logger.log directory: directory, "getting output files"
-
 		OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
-			return callback(error) if error?
-			jobs = []
+			if error?
+				logger.err err:error, "error finding all output files"
+				return callback(error)
 			outputFiles = []
 			for file in allFiles
 				if !incomingResources[file]
@@ -22,14 +21,16 @@ module.exports = OutputFileFinder =
 						path: file
 						type: file.match(/\.([^\.]+)$/)?[1]
 					}
-			callback null, outputFiles
+			callback null, outputFiles, allFiles
 
 	_getAllFiles: (directory, _callback = (error, fileList) ->) ->
 		callback = (error, fileList) ->
 			_callback(error, fileList)
 			_callback = () ->
 
-		args = [directory, "-name", ".cache", "-prune", "-o", "-type", "f", "-print"]
+		# don't include clsi-specific files/directories in the output list
+		EXCLUDE_DIRS = ["-name", ".cache", "-o", "-name", ".archive","-o", "-name", ".project-*"]
+		args = [directory, "(", EXCLUDE_DIRS..., ")", "-prune", "-o", "-type", "f", "-print"]
 		logger.log args: args, "running find command"
 
 		proc = spawn("find", args)

app/coffee/OutputFileOptimiser.coffee

@@ -2,6 +2,7 @@ fs = require "fs"
 Path = require "path"
 spawn = require("child_process").spawn
 logger = require "logger-sharelatex"
+Metrics = require "./Metrics"
 _ = require "underscore"
 
 module.exports = OutputFileOptimiser =
@@ -10,15 +11,31 @@ module.exports = OutputFileOptimiser =
 		# check output file (src) and see if we can optimise it, storing
 		# the result in the build directory (dst)
 		if src.match(/\/output\.pdf$/)
-			OutputFileOptimiser.optimisePDF src, dst, callback
+			OutputFileOptimiser.checkIfPDFIsOptimised src, (err, isOptimised) ->
+				return callback(null) if err? or isOptimised
+				OutputFileOptimiser.optimisePDF src, dst, callback
 		else
 			callback (null)
 
+	checkIfPDFIsOptimised: (file, callback) ->
+		SIZE = 16*1024 # check the header of the pdf
+		result = new Buffer(SIZE)
+		result.fill(0) # prevent leakage of uninitialised buffer
+		fs.open file, "r", (err, fd) ->
+			return callback(err) if err?
+			fs.read fd, result, 0, SIZE, 0, (errRead, bytesRead, buffer) ->
+				fs.close fd, (errClose) ->
+					return callback(errRead) if errRead?
+					return callback(errClose) if errReadClose?
+					isOptimised = buffer.toString('ascii').indexOf("/Linearized 1") >= 0
+					callback(null, isOptimised)
+
 	optimisePDF: (src, dst, callback = (error) ->) ->
 		tmpOutput = dst + '.opt'
 		args = ["--linearize", src, tmpOutput]
 		logger.log args: args, "running qpdf command"
 
+		timer = new Metrics.Timer("qpdf")
 		proc = spawn("qpdf", args)
 		stdout = ""
 		proc.stdout.on "data", (chunk) ->
@@ -28,6 +45,7 @@ module.exports = OutputFileOptimiser =
 			logger.warn {err, args}, "qpdf failed"
 			callback(null) # ignore the error
 		proc.on "close", (code) ->
+			timer.done()
 			if code != 0
 				logger.warn {code, args}, "qpdf returned error"
 				return callback(null) # ignore the error

app/coffee/ProjectPersistenceManager.coffee

@@ -1,21 +1,28 @@
 UrlCache = require "./UrlCache"
 CompileManager = require "./CompileManager"
 db = require "./db"
+dbQueue = require "./DbQueue"
 async = require "async"
 logger = require "logger-sharelatex"
+oneDay = 24 * 60 * 60 * 1000
+Settings = require "settings-sharelatex"
 
 module.exports = ProjectPersistenceManager =
-	EXPIRY_TIMEOUT: oneDay = 24 * 60 * 60 * 1000 #ms
+
+	EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5
 
 	markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
-		db.Project.findOrCreate(where: {project_id: project_id})
-			.spread(
-				(project, created) ->
-					project.updateAttributes(lastAccessed: new Date())
-						.then(() -> callback())
-						.error callback
-			)
-			.error callback
+		job = (cb)->
+			db.Project.findOrCreate(where: {project_id: project_id})
+				.spread(
+					(project, created) ->
+						project.updateAttributes(lastAccessed: new Date())
+							.then(() -> cb())
+							.error cb
+				)
+				.error cb
+		dbQueue.queue.push(job, callback)
+
 
 	clearExpiredProjects: (callback = (error) ->) ->
 		ProjectPersistenceManager._findExpiredProjectIds (error, project_ids) ->
@@ -24,29 +31,54 @@ module.exports = ProjectPersistenceManager =
 			jobs = for project_id in (project_ids or [])
 				do (project_id) ->
 					(callback) ->
-						ProjectPersistenceManager.clearProject project_id, (err) ->
+						ProjectPersistenceManager.clearProjectFromCache project_id, (err) ->
 							if err?
 								logger.error err: err, project_id: project_id, "error clearing project"
 							callback()
-			async.series jobs, callback
-
-	clearProject: (project_id, callback = (error) ->) ->
-		logger.log project_id: project_id, "clearing project"
-		CompileManager.clearProject project_id, (error) ->
-			return callback(error) if error?
-			UrlCache.clearProject project_id, (error) ->
+			async.series jobs, (error) ->
 				return callback(error) if error?
-				ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
-					return callback(error) if error?
-					callback()
+				CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) ->
+					callback() # ignore any errors from deleting directories
+
+	clearProject: (project_id, user_id, callback = (error) ->) ->
+		logger.log project_id: project_id, user_id:user_id, "clearing project for user"
+		CompileManager.clearProject project_id, user_id, (error) ->
+			return callback(error) if error?
+			ProjectPersistenceManager.clearProjectFromCache project_id, (error) ->
+				return callback(error) if error?
+				callback()
+
+	clearProjectFromCache: (project_id, callback = (error) ->) ->
+		logger.log project_id: project_id, "clearing project from cache"
+		UrlCache.clearProject project_id, (error) ->
+			if error?
+				logger.err error:error, project_id: project_id, "error clearing project from cache"
+				return callback(error) 
+			ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
+				if error?
+					logger.err error:error, project_id:project_id, "error clearing project from database"
+				callback(error)
 
 	_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
-		db.Project.destroy(where: {project_id: project_id})
-			.then(() -> callback())
-			.error callback
+		logger.log project_id:project_id, "clearing project from database"
+		job = (cb)->
+			db.Project.destroy(where: {project_id: project_id})
+				.then(() -> cb())
+				.error cb
+		dbQueue.queue.push(job, callback)
+
 
 	_findExpiredProjectIds: (callback = (error, project_ids) ->) ->
-		db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)])
-			.then((projects) ->
-				callback null, projects.map((project) -> project.project_id)
-			).error callback
+		job = (cb)->
+			keepProjectsFrom = new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)
+			q = {}
+			q[db.op.lt] = keepProjectsFrom
+			db.Project.findAll(where:{lastAccessed:q})
+				.then((projects) ->
+					cb null, projects.map((project) -> project.project_id)
+				).error cb
+
+		dbQueue.queue.push(job, callback)
+
+
+logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout"

app/coffee/RequestParser.coffee

@@ -1,6 +1,8 @@
+settings = require("settings-sharelatex")
+
 module.exports = RequestParser =
 	VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
-	MAX_TIMEOUT: 300
+	MAX_TIMEOUT: 600
 
 	parse: (body, callback = (error, data) ->) ->
 		response = {}
@@ -21,6 +23,41 @@ module.exports = RequestParser =
 				compile.options.timeout
 				default: RequestParser.MAX_TIMEOUT
 				type: "number"
+			response.imageName = @_parseAttribute "imageName",
+				compile.options.imageName,
+				type: "string"
+			response.draft = @_parseAttribute "draft",
+				compile.options.draft,
+				default: false,
+				type: "boolean"
+			response.check = @_parseAttribute "check",
+				compile.options.check,
+				type: "string"
+			response.flags = @_parseAttribute "flags",
+				compile.options.flags,
+				default: [],
+				type: "object"
+
+			# The syncType specifies whether the request contains all
+			# resources (full) or only those resources to be updated
+			# in-place (incremental).
+			response.syncType = @_parseAttribute "syncType",
+				compile.options.syncType,
+				validValues: ["full", "incremental"]
+				type: "string"
+
+			# The syncState is an identifier passed in with the request
+			# which has the property that it changes when any resource is
+			# added, deleted, moved or renamed.
+			#
+			# on syncType full the syncState identifier is passed in and
+			# stored
+			#
+			# on syncType incremental the syncState identifier must match
+			# the stored value
+			response.syncState = @_parseAttribute "syncState",
+				compile.options.syncState,
+				type: "string"
 
 			if response.timeout > RequestParser.MAX_TIMEOUT
 				response.timeout = RequestParser.MAX_TIMEOUT
@@ -32,7 +69,13 @@ module.exports = RequestParser =
 				compile.rootResourcePath
 				default: "main.tex"
 				type: "string"
-			response.rootResourcePath = RequestParser._sanitizePath(rootResourcePath)
+			originalRootResourcePath = rootResourcePath
+			sanitizedRootResourcePath = RequestParser._sanitizePath(rootResourcePath)
+			response.rootResourcePath = RequestParser._checkPath(sanitizedRootResourcePath)
+
+			for resource in response.resources
+				if resource.path == originalRootResourcePath
+					resource.path = sanitizedRootResourcePath
 		catch error
 			return callback error
 
@@ -48,7 +91,7 @@ module.exports = RequestParser =
 				throw "resource modified date could not be understood: #{resource.modified}"
 
 		if !resource.url? and !resource.content?
-		   	throw "all resources should have either a url or content attribute"
+			throw "all resources should have either a url or content attribute"
 		if resource.content? and typeof resource.content != "string"
 			throw "content attribute should be a string"
 		if resource.url? and typeof resource.url != "string"
@@ -71,9 +114,15 @@ module.exports = RequestParser =
 					throw "#{name} attribute should be a #{options.type}"
 		else
 			return options.default if options.default?
-			throw "Default not implemented"
 		return attribute
 
 	_sanitizePath: (path) ->
 		# See http://php.net/manual/en/function.escapeshellcmd.php
 		path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "")
+
+	_checkPath: (path) ->
+		# check that the request does not use a relative path
+		for dir in path.split('/')
+			if dir == '..'
+				throw "relative path in root resource"
+		return path

app/coffee/ResourceStateManager.coffee

@@ -0,0 +1,72 @@
+Path = require "path"
+fs = require "fs"
+logger = require "logger-sharelatex"
+settings = require("settings-sharelatex")
+Errors = require "./Errors"
+SafeReader = require "./SafeReader"
+
+module.exports = ResourceStateManager =
+
+	# The sync state is an identifier which must match for an
+	# incremental update to be allowed.
+	#
+	# The initial value is passed in and stored on a full
+	# compile, along with the list of resources..
+	#
+	# Subsequent incremental compiles must come with the same value - if
+	# not they will be rejected with a 409 Conflict response. The
+	# previous list of resources is returned.
+	#
+	# An incremental compile can only update existing files with new
+	# content.  The sync state identifier must change if any docs or
+	# files are moved, added, deleted or renamed.
+
+	SYNC_STATE_FILE: ".project-sync-state"
+	SYNC_STATE_MAX_SIZE: 128*1024
+
+	saveProjectState: (state, resources, basePath, callback = (error) ->) ->
+		stateFile = Path.join(basePath, @SYNC_STATE_FILE)
+		if not state? # remove the file if no state passed in
+			logger.log state:state, basePath:basePath, "clearing sync state"
+			fs.unlink stateFile, (err) ->
+				if err? and err.code isnt 'ENOENT'
+					return callback(err)
+				else
+					return callback()
+		else
+			logger.log state:state, basePath:basePath, "writing sync state"
+			resourceList = (resource.path for resource in resources)
+			fs.writeFile stateFile, [resourceList..., "stateHash:#{state}"].join("\n"), callback
+
+	checkProjectStateMatches: (state, basePath, callback = (error, resources) ->) ->
+		stateFile = Path.join(basePath, @SYNC_STATE_FILE)
+		size = @SYNC_STATE_MAX_SIZE
+		SafeReader.readFile stateFile, size, 'utf8', (err, result, bytesRead) ->
+			return callback(err) if err?
+			if bytesRead is size
+				logger.error file:stateFile, size:size, bytesRead:bytesRead, "project state file truncated"
+			[resourceList..., oldState] = result?.toString()?.split("\n") or []
+			newState = "stateHash:#{state}"
+			logger.log state:state, oldState: oldState, basePath:basePath, stateMatches: (newState is oldState), "checking sync state"
+			if newState isnt oldState
+				return callback new Errors.FilesOutOfSyncError("invalid state for incremental update")
+			else
+				resources = ({path: path} for path in resourceList)
+				callback(null, resources)
+
+	checkResourceFiles: (resources, allFiles, basePath, callback = (error) ->) ->
+		# check the paths are all relative to current directory
+		for file in resources or []
+			for dir in file?.path?.split('/')
+				if dir == '..'
+					return callback new Error("relative path in resource file list")
+		# check if any of the input files are not present in list of files
+		seenFile = {}
+		for file in allFiles
+			seenFile[file] = true
+		missingFiles = (resource.path for resource in resources when not seenFile[resource.path])
+		if missingFiles?.length > 0
+			logger.err missingFiles:missingFiles, basePath:basePath, allFiles:allFiles, resources:resources, "missing input files for project"
+			return callback new Errors.FilesOutOfSyncError("resource files missing in incremental update")
+		else
+			callback()

app/coffee/ResourceWriter.coffee

@@ -4,25 +4,71 @@ fs = require "fs"
 async = require "async"
 mkdirp = require "mkdirp"
 OutputFileFinder = require "./OutputFileFinder"
+ResourceStateManager = require "./ResourceStateManager"
 Metrics = require "./Metrics"
 logger = require "logger-sharelatex"
+settings = require("settings-sharelatex")
+
+parallelFileDownloads = settings.parallelFileDownloads or 1
 
 module.exports = ResourceWriter =
-	syncResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
-		@_removeExtraneousFiles resources, basePath, (error) =>
+
+	syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) ->
+		if request.syncType is "incremental"
+			logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
+			ResourceStateManager.checkProjectStateMatches request.syncState, basePath, (error, resourceList) ->
+				return callback(error) if error?
+				ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) ->
+					return callback(error) if error?
+					ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) ->
+						return callback(error) if error?
+						ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
+							return callback(error) if error?
+							callback(null, resourceList)
+		else
+			logger.log project_id: request.project_id, user_id: request.user_id, "full sync"
+			@saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) ->
+				return callback(error) if error?
+				ResourceStateManager.saveProjectState request.syncState, request.resources, basePath, (error) ->
+					return callback(error) if error?
+					callback(null, request.resources)
+
+	saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
+		@_createDirectory basePath, (error) =>
 			return callback(error) if error?
 			jobs = for resource in resources
 				do (resource) =>
 					(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
-			async.series jobs, callback
+			async.parallelLimit jobs, parallelFileDownloads, callback
 
-	_removeExtraneousFiles: (resources, basePath, _callback = (error) ->) ->
+	saveAllResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
+		@_createDirectory basePath, (error) =>
+			return callback(error) if error?
+			@_removeExtraneousFiles resources, basePath, (error) =>
+				return callback(error) if error?
+				jobs = for resource in resources
+					do (resource) =>
+						(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
+				async.parallelLimit jobs, parallelFileDownloads, callback
+
+	_createDirectory: (basePath, callback = (error) ->) ->
+		fs.mkdir basePath, (err) ->
+			if err?
+				if err.code is 'EEXIST'
+					return callback()
+				else
+					logger.log {err: err, dir:basePath}, "error creating directory"
+					return callback(err)
+			else
+				return callback()
+
+	_removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) ->
 		timer = new Metrics.Timer("unlink-output-files")
-		callback = (error) ->
+		callback = (error, result...) ->
 			timer.done()
-			_callback(error)
+			_callback(error, result...)
 
-		OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles) ->
+		OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) ->
 			return callback(error) if error?
 
 			jobs = []
@@ -30,37 +76,67 @@ module.exports = ResourceWriter =
 				do (file) ->
 					path = file.path
 					should_delete = true
-					if path.match(/^output\./) or path.match(/\.aux$/)
+					if path.match(/^output\./) or path.match(/\.aux$/) or path.match(/^cache\//) # knitr cache
 						should_delete = false
-					if path == "output.pdf" or path == "output.dvi" or path == "output.log"
+					if path.match(/^output-.*/) # Tikz cached figures (default case)
+						should_delete = false
+					if path.match(/\.(pdf|dpth|md5)$/) # Tikz cached figures (by extension)
+						should_delete = false
+					if path.match(/\.(pygtex|pygstyle)$/) or path.match(/(^|\/)_minted-[^\/]+\//) # minted files/directory
+						should_delete = false
+					if path.match(/\.md\.tex$/) or path.match(/(^|\/)_markdown_[^\/]+\//) # markdown files/directory
+						should_delete = false
+					if path.match(/-eps-converted-to\.pdf$/) # Epstopdf generated files
+						should_delete = false 
+					if path == "output.pdf" or path == "output.dvi" or path == "output.log" or path == "output.xdv"
+						should_delete = true
+					if path == "output.tex" # created by TikzManager if present in output files
 						should_delete = true
 					if should_delete
 						jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
 
-			async.series jobs, callback
+			async.series jobs, (error) ->
+				return callback(error) if error?
+				callback(null, outputFiles, allFiles)
 
 	_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
 		fs.stat path, (error, stat) ->
-			return callback(error) if error?
-			if stat.isFile()
-				fs.unlink path, callback
+			if error? and error.code is 'ENOENT'
+				return callback()
+			else if error?
+				logger.err {err: error, path: path}, "error stating file in deleteFileIfNotDirectory"
+				return callback(error)
+			else if stat.isFile()
+				fs.unlink path, (error) ->
+					if error?
+						logger.err {err: error, path: path}, "error removing file in deleteFileIfNotDirectory"
+						callback(error)
+					else
+						callback()
 			else
 				callback()
 
 	_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
-		path = Path.normalize(Path.join(basePath, resource.path))
-		if (path.slice(0, basePath.length) != basePath)
-			return callback new Error("resource path is outside root directory")
-
-		mkdirp Path.dirname(path), (error) ->
+		ResourceWriter.checkPath basePath, resource.path, (error, path) ->
 			return callback(error) if error?
-			# TODO: Don't overwrite file if it hasn't been modified
-			if resource.url?
-				UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)->
-					if err?
-						logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources"
-					callback() #try and continue compiling even if http resource can not be downloaded at this time
-			else
-				fs.writeFile path, resource.content, callback
-
+			mkdirp Path.dirname(path), (error) ->
+				return callback(error) if error?
+				# TODO: Don't overwrite file if it hasn't been modified
+				if resource.url?
+					UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)->
+						if err?
+							logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources"
+						callback() #try and continue compiling even if http resource can not be downloaded at this time
+				else
+					process = require("process")
+					fs.writeFile path, resource.content, callback
+					try 
+						result = fs.lstatSync(path)
+					catch e
 
+	checkPath: (basePath, resourcePath, callback) ->
+		path = Path.normalize(Path.join(basePath, resourcePath))
+		if (path.slice(0, basePath.length + 1) != basePath + "/")
+			return callback new Error("resource path is outside root directory")
+		else
+			return callback(null, path)

app/coffee/SafeReader.coffee

@@ -0,0 +1,25 @@
+fs = require "fs"
+logger = require "logger-sharelatex"
+
+module.exports = SafeReader =
+
+	# safely read up to size bytes from a file and return result as a
+	# string
+
+	readFile: (file, size, encoding, callback = (error, result) ->) ->
+		fs.open file, 'r', (err, fd) ->
+			return callback() if err? and err.code is 'ENOENT'
+			return callback(err) if err?
+
+			# safely return always closing the file
+			callbackWithClose = (err, result...) ->
+				fs.close fd, (err1) ->
+					return callback(err) if err?
+					return callback(err1) if err1?
+					callback(null, result...)
+
+			buff = new Buffer(size, 0) # fill with zeros
+			fs.read fd, buff, 0, buff.length, 0, (err, bytesRead, buffer) ->
+				return callbackWithClose(err) if err?
+				result = buffer.toString(encoding, 0, bytesRead)
+				callbackWithClose(null, result, bytesRead)

app/coffee/StaticServerForbidSymlinks.coffee

@@ -29,10 +29,10 @@ module.exports = ForbidSymlinks = (staticFn, root, options) ->
 		# check that the requested path is not a symlink
 		fs.realpath requestedFsPath, (err, realFsPath)->
 			if err?
-				logger.warn err:err, requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "error checking file access"
 				if err.code == 'ENOENT'
 					return res.sendStatus(404)
 				else
+					logger.error err:err, requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "error checking file access"
 					return res.sendStatus(500)
 			else if requestedFsPath != realFsPath
 				logger.warn requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "trying to access a different file (symlink), aborting"

app/coffee/TikzManager.coffee

@@ -0,0 +1,37 @@
+fs = require "fs"
+Path = require "path"
+ResourceWriter = require "./ResourceWriter"
+SafeReader = require "./SafeReader"
+logger = require "logger-sharelatex"
+
+# for \tikzexternalize or pstool to work the main file needs to match the
+# jobname.  Since we set the -jobname to output, we have to create a
+# copy of the main file as 'output.tex'.
+
+module.exports = TikzManager =
+
+	checkMainFile: (compileDir, mainFile, resources, callback = (error, needsMainFile) ->) ->
+		# if there's already an output.tex file, we don't want to touch it
+		for resource in resources
+			if resource.path is "output.tex"
+				logger.log compileDir: compileDir, mainFile: mainFile, "output.tex already in resources"
+				return callback(null, false)
+		# if there's no output.tex, see if we are using tikz/pgf or pstool in the main file
+		ResourceWriter.checkPath compileDir, mainFile, (error, path) ->
+			return callback(error) if error?
+			SafeReader.readFile path, 65536, "utf8", (error, content) ->
+				return callback(error) if error?
+				usesTikzExternalize = content?.indexOf("\\tikzexternalize") >= 0
+				usesPsTool = content?.indexOf("{pstool}") >= 0
+				logger.log compileDir: compileDir, mainFile: mainFile, usesTikzExternalize:usesTikzExternalize, usesPsTool: usesPsTool, "checked for packages needing main file as output.tex"
+				needsMainFile = (usesTikzExternalize || usesPsTool)
+				callback null, needsMainFile
+
+	injectOutputFile: (compileDir, mainFile, callback = (error) ->) ->
+		ResourceWriter.checkPath compileDir, mainFile, (error, path) ->
+			return callback(error) if error?
+			fs.readFile path, "utf8", (error, content) ->
+				return callback(error) if error?
+				logger.log compileDir: compileDir, mainFile: mainFile, "copied file to output.tex as project uses packages which require it"
+				# use wx flag to ensure that output file does not already exist
+				fs.writeFile Path.join(compileDir, "output.tex"), content, {flag:'wx'}, callback

app/coffee/UrlCache.coffee

@@ -1,4 +1,5 @@
 db = require("./db")
+dbQueue = require "./DbQueue"
 UrlFetcher = require("./UrlFetcher")
 Settings = require("settings-sharelatex")
 crypto = require("crypto")
@@ -51,7 +52,6 @@ module.exports = UrlCache =
 	_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
 		if !lastModified?
 			return callback null, true
-
 		UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
 			return callback(error) if error?
 			if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
@@ -87,35 +87,48 @@ module.exports = UrlCache =
 				callback null
 
 	_deleteUrlCacheFromDisk: (project_id, url, callback = (error) ->) ->
-		fs.unlink UrlCache._cacheFilePathForUrl(project_id, url), callback
+		fs.unlink UrlCache._cacheFilePathForUrl(project_id, url), (error) ->
+			if error? and error.code != 'ENOENT' # no error if the file isn't present
+				return callback(error)
+			else
+				return callback()
 
 	_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
-		db.UrlCache.find(where: { url: url, project_id: project_id })
-			.then((urlDetails) -> callback null, urlDetails)
-			.error callback
+		job = (cb)->
+			db.UrlCache.find(where: { url: url, project_id: project_id })
+				.then((urlDetails) -> cb null, urlDetails)
+				.error cb
+		dbQueue.queue.push job, callback
 
 	_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
-		db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
-			.spread(
-				(urlDetails, created) ->
-					urlDetails.updateAttributes(lastModified: lastModified)
-						.then(() -> callback())
-						.error(callback)
-			)
-			.error callback
+		job = (cb)->
+			db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
+				.spread(
+					(urlDetails, created) ->
+						urlDetails.updateAttributes(lastModified: lastModified)
+							.then(() -> cb())
+							.error(cb)
+				)
+				.error cb
+		dbQueue.queue.push(job, callback)
 
 	_clearUrlDetails: (project_id, url, callback = (error) ->) ->
-		db.UrlCache.destroy(where: {url: url, project_id: project_id})
-			.then(() -> callback null)
-			.error callback
+		job = (cb)->
+			db.UrlCache.destroy(where: {url: url, project_id: project_id})
+				.then(() -> cb null)
+				.error cb
+		dbQueue.queue.push(job, callback)
+
 
 	_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
-		db.UrlCache.findAll(where: { project_id: project_id })
-			.then(
-				(urlEntries) ->
-					callback null, urlEntries.map((entry) -> entry.url)
-			)
-			.error callback
+		job = (cb)->
+			db.UrlCache.findAll(where: { project_id: project_id })
+				.then(
+					(urlEntries) ->
+						cb null, urlEntries.map((entry) -> entry.url)
+				)
+				.error cb
+		dbQueue.queue.push(job, callback)
 
 		
 		

app/coffee/UrlFetcher.coffee

@@ -1,6 +1,8 @@
 request = require("request").defaults(jar: false)
 fs = require("fs")
 logger = require "logger-sharelatex"
+settings = require("settings-sharelatex")
+URL = require('url');
 
 oneMinute = 60 * 1000
 
@@ -11,6 +13,9 @@ module.exports = UrlFetcher =
 			_callback(error)
 			_callback = () ->
 
+		if settings.filestoreDomainOveride?
+			p = URL.parse(url).path
+			url = "#{settings.filestoreDomainOveride}#{p}"
 		timeoutHandler = setTimeout () ->
 			timeoutHandler = null
 			logger.error url:url, filePath: filePath, "Timed out downloading file to cache"

app/coffee/db.coffee

@@ -1,9 +1,12 @@
 Sequelize = require("sequelize")
 Settings = require("settings-sharelatex")
 _ = require("underscore")
+logger = require "logger-sharelatex"
 
 options = _.extend {logging:false}, Settings.mysql.clsi
 
+logger.log dbPath:Settings.mysql.clsi.storage, "connecting to db"
+
 sequelize = new Sequelize(
 	Settings.mysql.clsi.database,
 	Settings.mysql.clsi.username,
@@ -11,6 +14,12 @@ sequelize = new Sequelize(
 	options
 )
 
+if Settings.mysql.clsi.dialect == "sqlite"
+	logger.log "running PRAGMA journal_mode=WAL;"
+	sequelize.query("PRAGMA journal_mode=WAL;")
+	sequelize.query("PRAGMA synchronous=OFF;")
+	sequelize.query("PRAGMA read_uncommitted = true;")
+
 module.exports =
 	UrlCache: sequelize.define("UrlCache", {
 		url: Sequelize.STRING
@@ -32,5 +41,15 @@ module.exports =
 		]
 	})
 
-	sync: () -> sequelize.sync()
+	op: Sequelize.Op
+	
+	sync: () -> 
+		logger.log dbPath:Settings.mysql.clsi.storage, "syncing db schema"
+		sequelize.sync()
+			.then(-> 
+				logger.log "db sync complete"
+			).catch((err)->
+				console.log err, "error syncing"
+			)
+
 	

bin/acceptance_test

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e;
+MOCHA="node_modules/.bin/mocha --recursive --reporter spec --timeout 15000"
+$MOCHA "$@"

buildscript.txt

@@ -0,0 +1,9 @@
+clsi
+--language=coffeescript
+--node-version=10.15.0
+--acceptance-creds=None
+--dependencies=mongo,redis
+--docker-repos=gcr.io/overleaf-ops
+--env-pass-through=TEXLIVE_IMAGE
+--build-target=docker
+--script-version=1.1.22

cloudbuild.yaml

@@ -0,0 +1,39 @@
+steps:
+- id: texlive
+  name: 'gcr.io/overleaf-ops/texlive-full:2017.1'
+- id: build
+  name: 'gcr.io/overleaf-ops/cloud-builder'
+  args: 
+  - 'build'
+  env:
+  - 'BUILD_NUMBER=$SHORT_SHA'
+  - 'BRANCH_NAME=$BRANCH_NAME'
+  waitFor: ['-']
+- id: test_unit
+  name: 'gcr.io/overleaf-ops/cloud-builder'
+  args: 
+  - 'test_unit'
+  env:
+  - 'DOCKER_COMPOSE_FLAGS=-f docker-compose.ci.yml'
+  - 'BUILD_NUMBER=$SHORT_SHA'
+  - 'BRANCH_NAME=$BRANCH_NAME'
+  waitFor:
+  - build
+- id: test_acceptance
+  name: 'gcr.io/overleaf-ops/cloud-builder'
+  args: 
+  - 'test_acceptance'
+  env:
+  - 'DOCKER_COMPOSE_FLAGS=-f docker-compose.ci.yml'
+  - 'BUILD_NUMBER=$SHORT_SHA'
+  - 'BRANCH_NAME=$BRANCH_NAME'
+  - 'TEXLIVE_IMAGE=gcr.io/overleaf-ops/texlive-full:2017.1'
+  waitFor:
+  - build
+  - texlive
+images:
+  - 'gcr.io/$PROJECT_ID/clsi:${BRANCH_NAME}-${SHORT_SHA}'
+timeout: 1800s
+options:
+  diskSizeGb: 200
+  machineType: 'N1_HIGHCPU_8'

config/settings.defaults.coffee

@@ -7,33 +7,65 @@ module.exports =
 		clsi:
 			database: "clsi"
 			username: "clsi"
-			password: null
 			dialect: "sqlite"
-			storage: Path.resolve(__dirname + "/../db.sqlite")
+			storage: process.env["SQLITE_PATH"] or Path.resolve(__dirname + "/../db.sqlite")
+			pool:
+				max: 1
+				min: 1
+			retry:
+				max: 10
+
+	compileSizeLimit:  process.env["COMPILE_SIZE_LIMIT"] or "7mb"
 	
 	path:
 		compilesDir:  Path.resolve(__dirname + "/../compiles")
 		clsiCacheDir: Path.resolve(__dirname + "/../cache")
 		synctexBaseDir: (project_id) -> Path.join(@compilesDir, project_id)
 
-	# clsi:
-	# 	commandRunner: "docker-runner-sharelatex"
-	# 	docker:
-	# 		image: "quay.io/sharelatex/texlive-full"
-	# 		env:
-	# 			PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/2013/bin/x86_64-linux/"
-	# 			HOME: "/tmp"
-	# 		modem:
-	# 			socketPath: false
-	# 		user: "tex"
-
 	internal:
 		clsi:
 			port: 3013
-			host: "localhost"
+			host: process.env["LISTEN_ADDRESS"] or "localhost"
 			
+		load_balancer_agent:
+			report_load:true
+			load_port: 3048
+			local_port: 3049
 	apis:
 		clsi:
-			url: "http://localhost:3013"
+			url: "http://#{process.env['CLSI_HOST'] or 'localhost'}:3013"
 
-	smokeTest: false
+			
+	smokeTest: process.env["SMOKE_TEST"] or false
+	project_cache_length_ms: 1000 * 60 * 60 * 24
+	parallelFileDownloads: process.env["FILESTORE_PARALLEL_FILE_DOWNLOADS"] or 1
+	parallelSqlQueryLimit: process.env["FILESTORE_PARALLEL_SQL_QUERY_LIMIT"] or 1
+	filestoreDomainOveride: process.env["FILESTORE_DOMAIN_OVERRIDE"]
+	texliveImageNameOveride: process.env["TEX_LIVE_IMAGE_NAME_OVERRIDE"]
+	sentry:
+		dsn: process.env['SENTRY_DSN']
+
+
+if process.env["DOCKER_RUNNER"]
+	module.exports.clsi =
+		dockerRunner: process.env["DOCKER_RUNNER"] == "true"
+		docker:
+			image: process.env["TEXLIVE_IMAGE"] or "quay.io/sharelatex/texlive-full:2017.1"
+			env:
+				HOME: "/tmp"
+			socketPath: "/var/run/docker.sock"
+			user: process.env["TEXLIVE_IMAGE_USER"] or "tex"
+		expireProjectAfterIdleMs: 24 * 60 * 60 * 1000
+		checkProjectsIntervalMs: 10 * 60 * 1000
+
+	try
+		seccomp_profile_path = Path.resolve(__dirname + "/../seccomp/clsi-profile.json")
+		module.exports.clsi.docker.seccomp_profile = JSON.stringify(JSON.parse(require("fs").readFileSync(seccomp_profile_path)))
+	catch error
+		console.log error, "could not load seccom profile from #{seccomp_profile_path}"
+
+	module.exports.path.synctexBaseDir = -> "/compile"	
+	
+	module.exports.path.sandboxedCompilesHostDir = process.env["COMPILES_HOST_DIR"]
+
+	module.exports.path.synctexBinHostPath = process.env["SYNCTEX_BIN_HOST_PATH"]

debug

@@ -0,0 +1,5 @@
+#!/bin/bash
+echo "hello world"
+sleep 3
+echo "awake"
+/opt/synctex pdf /compile/output.pdf 1 100 200

docker-compose-config.yml

@@ -0,0 +1,32 @@
+version: "2"
+
+services:
+  dev:
+    environment:
+      TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
+      TEXLIVE_IMAGE_USER: "tex"
+      SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
+      DOCKER_RUNNER: "true"
+      COMPILES_HOST_DIR: $PWD/compiles
+      SYNCTEX_BIN_HOST_PATH: $PWD/bin/synctex
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./compiles:/app/compiles
+      - ./cache:/app/cache
+      - ./bin/synctex:/app/bin/synctex
+
+      
+  ci:
+    environment:
+      TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
+      TEXLIVE_IMAGE_USER: "tex"
+      SHARELATEX_CONFIG: /app/config/settings.defaults.coffee
+      DOCKER_RUNNER: "true"
+      COMPILES_HOST_DIR: $PWD/compiles
+      SYNCTEX_BIN_HOST_PATH: $PWD/bin/synctex
+      SQLITE_PATH: /app/compiles/db.sqlite
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:rw
+      - ./compiles:/app/compiles
+      - ./cache:/app/cache
+      - ./bin/synctex:/app/bin/synctex

docker-compose.ci.yml

@@ -0,0 +1,49 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.22
+
+version: "2"
+
+services:
+  test_unit:
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    command: npm run test:unit:_run
+    environment:
+      NODE_ENV: test
+
+
+  test_acceptance:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    extends:
+      file: docker-compose-config.yml
+      service: ci
+    environment:
+      ELASTIC_SEARCH_DSN: es:9200
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      MOCHA_GREP: ${MOCHA_GREP}
+      NODE_ENV: test
+      TEXLIVE_IMAGE:
+    depends_on:
+      - mongo
+      - redis
+    command: npm run test:acceptance:_run
+
+
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4

docker-compose.yml

@@ -0,0 +1,56 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.22
+
+version: "2"
+
+services:
+  test_unit:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      MOCHA_GREP: ${MOCHA_GREP}
+      NODE_ENV: test
+    command: npm run test:unit
+
+  test_acceptance:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    extends:
+      file: docker-compose-config.yml
+      service: dev
+    environment:
+      ELASTIC_SEARCH_DSN: es:9200
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      MOCHA_GREP: ${MOCHA_GREP}
+      LOG_LEVEL: ERROR
+      NODE_ENV: test
+    depends_on:
+      - mongo
+      - redis
+    command: npm run test:acceptance
+
+
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4
+
+

entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -o pipefail
+
+/app/inner-entrypoint.sh "$@" 2>&1 | ts
+

inner-entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -x
+
+date
+echo "Changing permissions of /var/run/docker.sock for sibling containers"
+ls -al /var/run/docker.sock
+docker --version
+cat /etc/passwd
+
+DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock)
+groupadd --non-unique --gid ${DOCKER_GROUP} dockeronhost
+usermod -aG dockeronhost node
+
+mkdir -p /app/cache
+chown -R node:node /app/cache
+
+mkdir -p /app/compiles
+chown -R node:node /app/compiles
+
+chown -R node:node /app/bin/synctex
+mkdir -p /app/test/acceptance/fixtures/tmp/
+chown -R node:node /app
+
+chown -R node:node /app/bin
+
+exec runuser -u node -- "$@"

install_deps.sh

@@ -0,0 +1,4 @@
+/bin/sh
+wget -qO- https://get.docker.com/ | sh
+apt-get install poppler-utils vim ghostscript --yes
+npm rebuild

kube.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: clsi
+  namespace: default
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: clsi
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: clsi
+  namespace: default
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        run: clsi
+    spec:
+      containers:
+      - name: clsi
+        image: gcr.io/henry-terraform-admin/clsi
+        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: status
+            port: 80
+          periodSeconds: 5
+          initialDelaySeconds: 0
+          failureThreshold: 3
+          successThreshold: 1
+
+
+

nodemon.json

@@ -0,0 +1,19 @@
+{
+  "ignore": [
+    ".git",
+    "node_modules/"
+  ],
+  "verbose": true,
+  "legacyWatch": true,
+  "execMap": {
+    "js": "npm run start"
+  },
+
+  "watch": [
+    "app/coffee/",
+    "app.coffee",
+    "config/"
+  ],
+  "ext": "coffee"
+
+}

package.json

@@ -6,41 +6,49 @@
     "type": "git",
     "url": "https://github.com/sharelatex/clsi-sharelatex.git"
   },
+  "scripts": {
+    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
+    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
+    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
+    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
+    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
+    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
+    "nodemon": "nodemon --config nodemon.json",
+    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
+  },
   "author": "James Allen <james@sharelatex.com>",
   "dependencies": {
     "async": "0.2.9",
+    "body-parser": "^1.2.0",
+    "dockerode": "^2.5.3",
+    "express": "^4.2.0",
+    "fs-extra": "^0.16.3",
+    "heapdump": "^0.3.5",
+    "lockfile": "^1.0.3",
+    "logger-sharelatex": "^1.7.0",
     "lynx": "0.0.11",
+    "metrics-sharelatex": "^2.2.0",
     "mkdirp": "0.3.5",
     "mysql": "2.6.2",
-    "request": "~2.21.0",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0",
-    "sequelize": "^2.1.3",
-    "wrench": "~1.5.4",
+    "request": "^2.21.0",
+    "sequelize": "^4.38.0",
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.1.0",
     "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v0.2.0",
-    "sqlite3": "~2.2.0",
-    "express": "^4.2.0",
-    "body-parser": "^1.2.0",
-    "fs-extra": "^0.16.3",
+    "sqlite3": "^4.0.6",
     "underscore": "^1.8.2",
-    "v8-profiler": "^5.2.4",
-    "heapdump": "^0.3.5"
+    "v8-profiler-node8": "^6.0.1",
+    "wrench": "~1.5.4"
   },
   "devDependencies": {
-    "mocha": "1.10.0",
-    "coffee-script": "1.6.0",
-    "chai": "~1.8.1",
-    "sinon": "~1.7.3",
-    "grunt": "~0.4.2",
-    "grunt-contrib-coffee": "~0.7.0",
-    "grunt-contrib-clean": "~0.5.0",
-    "grunt-shell": "~0.6.1",
-    "grunt-mocha-test": "~0.8.1",
-    "sandboxed-module": "~0.3.0",
-    "timekeeper": "0.0.4",
-    "grunt-execute": "^0.1.5",
     "bunyan": "^0.22.1",
-    "grunt-bunyan": "^0.5.0"
+    "chai": "~1.8.1",
+    "coffeescript": "1.6.0",
+    "mocha": "^4.0.1",
+    "sandboxed-module": "~0.3.0",
+    "sinon": "~1.7.3",
+    "timekeeper": "0.0.4"
   }
 }

patch-texlive-dockerfile

@@ -0,0 +1,3 @@
+FROM quay.io/sharelatex/texlive-full:2017.1
+
+# RUN usermod -u 1001 tex

seccomp/clsi-profile.json

@@ -0,0 +1,836 @@
+{
+    "defaultAction": "SCMP_ACT_ERRNO",
+    "architectures": [
+        "SCMP_ARCH_X86_64",
+        "SCMP_ARCH_X86",
+        "SCMP_ARCH_X32"
+    ],
+    "syscalls": [
+        {
+            "name": "access",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "arch_prctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "brk",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "chdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "chmod",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_getres",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_gettime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_nanosleep",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clone",
+            "action": "SCMP_ACT_ALLOW",
+            "args": [
+                {
+                    "index": 0,
+                    "value": 2080505856,
+                    "valueTwo": 0,
+                    "op": "SCMP_CMP_MASKED_EQ"
+                }
+            ]
+        },
+        {
+            "name": "close",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "copy_file_range",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "creat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup3",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "execve",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "execveat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "exit",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "exit_group",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "faccessat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fadvise64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fadvise64_64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fallocate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchmod",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchmodat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fcntl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fcntl64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fdatasync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fork",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatfs64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fsync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ftruncate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ftruncate64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "futex",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "futimesat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getcpu",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getcwd",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getdents",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getdents64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getegid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getegid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "geteuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "geteuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgroups",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgroups32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpgrp",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getppid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpriority",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresgid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getrlimit",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "get_robust_list",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getrusage",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getsid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "gettid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ioctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "kill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "_llseek",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lseek",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lstat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lstat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "madvise",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mkdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mkdirat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mmap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mmap2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mprotect",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mremap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "munmap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "newfstatat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "open",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "openat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pause",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pipe",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pipe2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "prctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pread64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "preadv",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "prlimit64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pwrite64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pwritev",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "read",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readlink",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readlinkat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readv",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rename",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "renameat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "renameat2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "restart_syscall",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rmdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigaction",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigpending",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigprocmask",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigqueueinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigreturn",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigsuspend",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigtimedwait",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_tgsigqueueinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getaffinity",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getparam",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_get_priority_max",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_get_priority_min",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getscheduler",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_rr_get_interval",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_yield",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sendfile",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sendfile64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgroups",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgroups32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "set_robust_list",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "set_tid_address",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sigaltstack",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "stat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "stat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statfs64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sync_file_range",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "syncfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sysinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "tgkill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_create",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_delete",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_getoverrun",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_gettime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_settime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "times",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "tkill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "truncate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "truncate64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "umask",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "uname",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "unlink",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "unlinkat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utimensat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utimes",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "vfork",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "vhangup",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "wait4",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "waitid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "write",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "writev",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pread",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "capget",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "capset",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchown",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+          "name": "gettimeofday",
+          "action": "SCMP_ACT_ALLOW",
+          "args": []
+        }, {
+          "name": "epoll_pwait",
+          "action": "SCMP_ACT_ALLOW",
+          "args": []
+        }
+    ]
+}

synctex.profile

@@ -0,0 +1,34 @@
+include /etc/firejail/disable-common.inc
+include /etc/firejail/disable-devel.inc
+# include /etc/firejail/disable-mgmt.inc  ## removed in 0.9.40
+# include /etc/firejail/disable-secret.inc ## removed in 0.9.40
+
+read-only /bin
+blacklist /boot
+blacklist /dev
+read-only /etc
+blacklist /home # blacklisted for synctex
+read-only /lib
+read-only /lib64
+blacklist /media
+blacklist /mnt
+blacklist /opt
+blacklist /root
+read-only /run
+blacklist /sbin
+blacklist /selinux
+blacklist /src
+blacklist /sys
+read-only /usr
+
+caps.drop all
+noroot
+nogroups
+net none
+private-tmp
+private-dev
+shell none
+seccomp
+nonewprivs
+
+

test/acceptance/coffee/BrokenLatexFileTests.coffee

@@ -1,9 +1,10 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Broken LaTeX file", ->
-	before ->
+	before (done)->
 		@broken_request =
 			resources: [
 				path: "main.tex"
@@ -24,6 +25,7 @@ describe "Broken LaTeX file", ->
 					\\end{document}
 				'''
 			]
+		ClsiApp.ensureRunning done
 		
 	describe "on first run", ->
 		before (done) ->

test/acceptance/coffee/DeleteOldFilesTest.coffee

@@ -1,9 +1,10 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Deleting Old Files", ->
-	before ->
+	before (done)->
 		@request =
 			resources: [
 				path: "main.tex"
@@ -14,6 +15,7 @@ describe "Deleting Old Files", ->
 					\\end{document}
 				'''
 			]
+		ClsiApp.ensureRunning done
 
 	describe "on first run", ->
 		before (done) ->

test/acceptance/coffee/ExampleDocumentTests.coffee

@@ -3,27 +3,56 @@ request = require "request"
 require("chai").should()
 fs = require "fs"
 ChildProcess = require "child_process"
-
-fixturePath = (path) -> __dirname + "/../fixtures/" + path
-
+ClsiApp = require "./helpers/ClsiApp"
+logger = require("logger-sharelatex")
+Path = require("path")
+fixturePath = (path) -> Path.normalize(__dirname + "/../fixtures/" + path)
+process = require "process"
+console.log process.pid, process.ppid, process.getuid(),process.getgroups(), "PID"
 try
+	console.log "creating tmp directory", fixturePath("tmp")
 	fs.mkdirSync(fixturePath("tmp"))
-catch e
+catch err
+	console.log err, fixturePath("tmp"), "unable to create fixture tmp path"
+
+MOCHA_LATEX_TIMEOUT = 60 * 1000
 
 convertToPng = (pdfPath, pngPath, callback = (error) ->) ->
-	convert = ChildProcess.exec "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
+	command = "convert #{fixturePath(pdfPath)} #{fixturePath(pngPath)}"
+	console.log "COMMAND"
+	console.log command
+	convert = ChildProcess.exec command
+	stdout = ""
+	convert.stdout.on "data", (chunk) -> console.log "STDOUT", chunk.toString()
+	convert.stderr.on "data", (chunk) -> console.log "STDERR", chunk.toString()
 	convert.on "exit", () ->
 		callback()
 
 compare = (originalPath, generatedPath, callback = (error, same) ->) ->
-	proc = ChildProcess.exec "compare -metric mae #{fixturePath(originalPath)} #{fixturePath(generatedPath)} #{fixturePath("tmp/diff.png")}"
+	diff_file = "#{fixturePath(generatedPath)}-diff.png"
+	proc = ChildProcess.exec "compare -metric mae #{fixturePath(originalPath)} #{fixturePath(generatedPath)} #{diff_file}"
 	stderr = ""
 	proc.stderr.on "data", (chunk) -> stderr += chunk
 	proc.on "exit", () ->
 		if stderr.trim() == "0 (0)"
+			# remove output diff if test matches expected image
+			fs.unlink diff_file, (err) ->
+				if err
+					throw err
+			callback null, true
+		else
+			console.log "compare result", stderr
+			callback null, false
+
+checkPdfInfo = (pdfPath, callback = (error, output) ->) ->
+	proc = ChildProcess.exec "pdfinfo #{fixturePath(pdfPath)}"
+	stdout = ""
+	proc.stdout.on "data", (chunk) -> stdout += chunk
+	proc.stderr.on "data", (chunk) -> console.log "STDERR", chunk.toString()
+	proc.on "exit", () ->
+		if stdout.match(/Optimized:\s+yes/)
 			callback null, true
 		else
-			console.log stderr
 			callback null, false
 
 compareMultiplePages = (project_id, callback = (error) ->) ->
@@ -39,44 +68,61 @@ compareMultiplePages = (project_id, callback = (error) ->) ->
 					compareNext page_no + 1, callback
 	compareNext 0, callback
 
+comparePdf = (project_id, example_dir, callback = (error) ->) ->
+	console.log "CONVERT"
+	console.log "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png"
+	convertToPng "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png", (error) =>
+		throw error if error?
+		convertToPng "examples/#{example_dir}/output.pdf", "tmp/#{project_id}-source.png", (error) =>
+			throw error if error?
+			fs.stat fixturePath("tmp/#{project_id}-source-0.png"), (error, stat) =>
+				if error?
+					compare  "tmp/#{project_id}-source.png", "tmp/#{project_id}-generated.png", (error, same) =>
+						throw error if error?
+						same.should.equal true
+						callback()
+				else
+					compareMultiplePages project_id, (error) ->
+						throw error if error?
+						callback()
+
 downloadAndComparePdf = (project_id, example_dir, url, callback = (error) ->) ->
 	writeStream = fs.createWriteStream(fixturePath("tmp/#{project_id}.pdf"))
 	request.get(url).pipe(writeStream)
+	console.log("writing file out", fixturePath("tmp/#{project_id}.pdf"))
 	writeStream.on "close", () =>
-		convertToPng "tmp/#{project_id}.pdf", "tmp/#{project_id}-generated.png", (error) =>
+		checkPdfInfo "tmp/#{project_id}.pdf", (error, optimised) =>
 			throw error if error?
-			convertToPng "examples/#{example_dir}/output.pdf", "tmp/#{project_id}-source.png", (error) =>
-				throw error if error?
-				fs.stat fixturePath("tmp/#{project_id}-source-0.png"), (error, stat) =>
-					if error?
-						compare  "tmp/#{project_id}-source.png", "tmp/#{project_id}-generated.png", (error, same) =>
-							throw error if error?
-							same.should.equal true
-							callback()
-					else
-						compareMultiplePages project_id, (error) ->
-							throw error if error?
-							callback()	
+			optimised.should.equal true
+			comparePdf project_id, example_dir, callback
 
 Client.runServer(4242, fixturePath("examples"))
 
 describe "Example Documents", ->
 	before (done) ->
-		ChildProcess.exec("rm test/acceptance/fixtures/tmp/*").on "exit", () -> done()
+		ChildProcess.exec("rm test/acceptance/fixtures/tmp/*").on "exit", () -> 
+			ClsiApp.ensureRunning done
+
 
 	for example_dir in fs.readdirSync fixturePath("examples")
 		do (example_dir) ->
 			describe example_dir, ->
 				before ->
-					@project_id = Client.randomId()
+					@project_id = Client.randomId() + "_" + example_dir
 
 				it "should generate the correct pdf", (done) ->
+					this.timeout(MOCHA_LATEX_TIMEOUT)
 					Client.compileDirectory @project_id, fixturePath("examples"), example_dir, 4242, (error, res, body) =>
+						if error || body?.compile?.status is "failure"
+							console.log "DEBUG: error", error, "body", JSON.stringify(body)
 						pdf = Client.getOutputFile body, "pdf"
 						downloadAndComparePdf(@project_id, example_dir, pdf.url, done)
 
 				it "should generate the correct pdf on the second run as well", (done) ->
+					this.timeout(MOCHA_LATEX_TIMEOUT)
 					Client.compileDirectory @project_id, fixturePath("examples"), example_dir, 4242, (error, res, body) =>
+						if error || body?.compile?.status is "failure"
+							console.log "DEBUG: error", error, "body", JSON.stringify(body)
 						pdf = Client.getOutputFile body, "pdf"
 						downloadAndComparePdf(@project_id, example_dir, pdf.url, done)
 

test/acceptance/coffee/SimpleLatexFileTests.coffee

@@ -1,6 +1,7 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Simple LaTeX file", ->
 	before (done) ->
@@ -15,7 +16,8 @@ describe "Simple LaTeX file", ->
 					\\end{document}
 				'''
 			]
-		Client.compile @project_id, @request, (@error, @res, @body) => done()
+		ClsiApp.ensureRunning =>
+			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	it "should return the PDF", ->
 		pdf = Client.getOutputFile(@body, "pdf")

test/acceptance/coffee/SynctexTests.coffee

@@ -2,21 +2,25 @@ Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
 expect = require("chai").expect
+ClsiApp = require "./helpers/ClsiApp"
+crypto = require("crypto")
 
 describe "Syncing", ->
 	before (done) ->
-		@request =
-			resources: [
-				path: "main.tex"
-				content: '''
+		content = '''
 					\\documentclass{article}
 					\\begin{document}
 					Hello world
 					\\end{document}
 				'''
+		@request =
+			resources: [
+				path: "main.tex"
+				content: content
 			]
 		@project_id = Client.randomId()
-		Client.compile @project_id, @request, (@error, @res, @body) => done()
+		ClsiApp.ensureRunning =>
+			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	describe "from code to pdf", ->
 		it "should return the correct location", (done) ->
@@ -29,7 +33,7 @@ describe "Syncing", ->
 
 	describe "from pdf to code", ->
 		it "should return the correct location", (done) ->
-			Client.syncFromPdf @project_id, 1, 100, 200, (error, codePositions) ->
+			Client.syncFromPdf @project_id, 1, 100, 200, (error, codePositions) =>
 				throw error if error?
 				expect(codePositions).to.deep.equal(
 					code: [ { file: 'main.tex', line: 3, column: -1 } ]

test/acceptance/coffee/TimeoutTests.coffee

@@ -1,23 +1,27 @@
 Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
+ClsiApp = require "./helpers/ClsiApp"
+
 
 describe "Timed out compile", ->
 	before (done) ->
 		@request =
 			options:
-				timeout: 0.01 #seconds 
+				timeout: 10 #seconds
 			resources: [
 				path: "main.tex"
 				content: '''
 					\\documentclass{article}
 					\\begin{document}
-					Hello world
+					\\def\\x{Hello!\\par\\x}
+					\\x					
 					\\end{document}
 				'''
 			]
 		@project_id = Client.randomId()
-		Client.compile @project_id, @request, (@error, @res, @body) => done()
+		ClsiApp.ensureRunning =>
+			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	it "should return a timeout error", ->
 		@body.compile.error.should.equal "container timed out"

test/acceptance/coffee/UrlCachingTests.coffee

@@ -2,6 +2,7 @@ Client = require "./helpers/Client"
 request = require "request"
 require("chai").should()
 sinon = require "sinon"
+ClsiApp = require "./helpers/ClsiApp"
 
 host = "localhost"
 
@@ -46,7 +47,8 @@ describe "Url Caching", ->
 				}]
 
 			sinon.spy Server, "getFile"
-			Client.compile @project_id, @request, (@error, @res, @body) => done()
+			ClsiApp.ensureRunning =>
+				Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 		afterEach ->
 			Server.getFile.restore()

test/acceptance/coffee/WordcountTests.coffee

@@ -4,6 +4,7 @@ require("chai").should()
 expect = require("chai").expect
 path = require("path")
 fs = require("fs")
+ClsiApp = require "./helpers/ClsiApp"
 
 describe "Syncing", ->
 	before (done) ->
@@ -13,7 +14,8 @@ describe "Syncing", ->
 				content: fs.readFileSync(path.join(__dirname,"../fixtures/naugty_strings.txt"),"utf-8")
 			]
 		@project_id = Client.randomId()
-		Client.compile @project_id, @request, (@error, @res, @body) => done()
+		ClsiApp.ensureRunning =>
+			Client.compile @project_id, @request, (@error, @res, @body) => done()
 
 	describe "wordcount file", ->
 		it "should return wordcount info", (done) ->
@@ -29,6 +31,8 @@ describe "Syncing", ->
 						elements: 0
 						mathInline: 6
 						mathDisplay: 0
+						errors: 0
+						messages: ""
 					}
 				)
 				done()

test/acceptance/coffee/helpers/Client.coffee

@@ -30,7 +30,11 @@ module.exports = Client =
 		express = require("express")
 		app = express()
 		app.use express.static(directory)
-		app.listen(port, host)
+		console.log("starting test server on", port, host)
+		app.listen(port, host).on "error", (error) ->
+			console.error "error starting server:", error.message
+			process.exit(1)
+
 
 	syncFromCode: (project_id, file, line, column, callback = (error, pdfPositions) ->) ->
 		request.get {

test/acceptance/coffee/helpers/ClsiApp.coffee

@@ -0,0 +1,24 @@
+app = require('../../../../app')
+require("logger-sharelatex").logger.level("info")
+logger = require("logger-sharelatex")
+Settings = require("settings-sharelatex")
+
+module.exports =
+	running: false
+	initing: false
+	callbacks: []
+	ensureRunning: (callback = (error) ->) ->
+		if @running
+			return callback()
+		else if @initing
+			@callbacks.push callback
+		else
+			@initing = true
+			@callbacks.push callback
+			app.listen Settings.internal?.clsi?.port, "localhost", (error) => 
+				throw error if error?
+				@running = true
+				logger.log("clsi running in dev mode")
+
+				for callback in @callbacks
+					callback()

test/acceptance/fixtures/examples/fontawesome/main.tex

@@ -0,0 +1,12 @@
+\documentclass{article}
+\usepackage{fontawesome}
+
+\begin{document}
+Cloud \faCloud
+
+Cog \faCog
+
+Database \faDatabase
+
+Leaf \faLeaf
+\end{document}

test/acceptance/fixtures/examples/fontawesome_xelatex/main.tex

@@ -0,0 +1,16 @@
+\documentclass{article}
+\usepackage{fontspec}
+\defaultfontfeatures{Extension = .otf} % this is needed because
+                                       % fontawesome package loads by
+                                       % font name only
+\usepackage{fontawesome}
+
+\begin{document}
+Cloud \faCloud
+
+Cog \faCog
+
+Database \faDatabase
+
+Leaf \faLeaf
+\end{document}

test/acceptance/fixtures/examples/fontawesome_xelatex/options.json

@@ -0,0 +1,3 @@
+{
+	"compiler": "xelatex"
+}

test/acceptance/fixtures/examples/hebrew/main.tex

@@ -0,0 +1,14 @@
+\documentclass{article}
+\usepackage[utf8x]{inputenc}
+\usepackage[hebrew,english]{babel}
+
+\begin{document}
+\selectlanguage{hebrew}
+
+ כדי לכתוב משהו באנגלית חייבים להשתמש במקרו הבא וכאן
+
+ ממשיכים לכתוב בעברית. טקסט נוסחאות תמיד יהיה בכיוון שמאל-לימין
+
+\selectlanguage{english}
+This is a test.
+\end{document}

test/acceptance/fixtures/examples/knitr_utf8/main.Rtex

@@ -0,0 +1,35 @@
+\documentclass{article}
+\usepackage[utf8]{inputenc}
+\usepackage[spanish]{babel}
+ 
+\begin{document}
+ 
+\tableofcontents
+ 
+\vspace{2cm} %Add a 2cm space
+ 
+\begin{abstract}
+Este es un breve resumen del contenido del 
+documento escrito en español.
+\end{abstract}
+ 
+\section{Sección Introductoria}
+Esta es la primera sección, podemos agregar 
+algunos elementos adicionales y todo será 
+escrito correctamente. Más aún, si una palabra 
+es demaciado larga y tiene que ser truncada, 
+babel tratará de truncarla correctamente 
+dependiendo del idioma.
+ 
+\section{Sección con teoremas}
+Esta sección es para ver que pasa con los comandos 
+que definen texto
+ 
+%% chunk options: cache this chunk
+%% begin.rcode my-cache, cache=TRUE
+% set.seed(123)
+% x = runif(10)
+% sd(x)  # standard deviation
+%% end.rcode
+ 
+\end{document}

test/acceptance/fixtures/examples/latex_compiler/main.tex

@@ -1,4 +1,4 @@
-\documentclass{article}
+\documentclass[a4paper]{article}
 
 \usepackage{graphicx}
 

test/acceptance/fixtures/examples/tikz_feynman/main.tex

@@ -0,0 +1,66 @@
+\RequirePackage{luatex85}
+\documentclass[tikz]{standalone}
+
+\usepackage[compat=1.1.0]{tikz-feynman}
+
+\begin{document}
+\feynmandiagram [horizontal=a to b] {
+  i1 -- [fermion] a -- [fermion] i2,
+  a -- [photon] b,
+  f1 -- [fermion] b -- [fermion] f2,
+};
+
+\feynmandiagram [horizontal=a to b] {
+  i1 [particle=\(e^{-}\)] -- [fermion] a -- [fermion] i2 [particle=\(e^{+}\)],
+  a -- [photon, edge label=\(\gamma\), momentum'=\(k\)] b,
+  f1 [particle=\(\mu^{+}\)] -- [fermion] b -- [fermion] f2 [particle=\(\mu^{-}\)],
+};
+
+\feynmandiagram [large, vertical=e to f] {
+  a -- [fermion] b -- [photon, momentum=\(k\)] c -- [fermion] d,
+  b -- [fermion, momentum'=\(p_{1}\)] e -- [fermion, momentum'=\(p_{2}\)] c,
+  e -- [gluon] f,
+  h -- [fermion] f -- [fermion] i,
+};
+
+\begin{tikzpicture}
+  \begin{feynman}
+    \vertex (a1) {\(\overline b\)};
+    \vertex[right=1cm of a1] (a2);
+    \vertex[right=1cm of a2] (a3);
+    \vertex[right=1cm of a3] (a4) {\(b\)};
+    \vertex[right=1cm of a4] (a5);
+    \vertex[right=2cm of a5] (a6) {\(u\)};
+
+    \vertex[below=2em of a1] (b1) {\(d\)};
+    \vertex[right=1cm of b1] (b2);
+    \vertex[right=1cm of b2] (b3);
+    \vertex[right=1cm of b3] (b4) {\(\overline d\)};
+    \vertex[below=2em of a6] (b5) {\(\overline d\)};
+
+    \vertex[above=of a6] (c1) {\(\overline u\)};
+    \vertex[above=2em of c1] (c3) {\(d\)};
+    \vertex at ($(c1)!0.5!(c3) - (1cm, 0)$) (c2);
+
+    \diagram* {
+      {[edges=fermion]
+        (b1) -- (b2) -- (a2) -- (a1),
+        (b5) -- (b4) -- (b3) -- (a3) -- (a4) -- (a5) -- (a6),
+      },
+      (a2) -- [boson, edge label=\(W\)] (a3),
+      (b2) -- [boson, edge label'=\(W\)] (b3),
+
+      (c1) -- [fermion, out=180, in=-45] (c2) -- [fermion, out=45, in=180] (c3),
+      (a5) -- [boson, bend left, edge label=\(W^{-}\)] (c2),
+    };
+
+    \draw [decoration={brace}, decorate] (b1.south west) -- (a1.north west)
+    node [pos=0.5, left] {\(B^{0}\)};
+    \draw [decoration={brace}, decorate] (c3.north east) -- (c1.south east)
+    node [pos=0.5, right] {\(\pi^{-}\)};
+    \draw [decoration={brace}, decorate] (a6.north east) -- (b5.south east)
+    node [pos=0.5, right] {\(\pi^{+}\)};
+  \end{feynman}
+\end{tikzpicture}
+
+\end{document}

test/acceptance/fixtures/examples/tikz_feynman/options.json

@@ -0,0 +1,3 @@
+{
+	"compiler": "lualatex"
+}

test/acceptance/scripts/full-test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -x
+
+export SHARELATEX_CONFIG=`pwd`/test/acceptance/scripts/settings.test.coffee
+
+echo ">> Starting server..."
+
+grunt --no-color >server.log 2>&1 &
+
+echo ">> Server started"
+
+sleep 5
+
+echo ">> Running acceptance tests..."
+grunt --no-color mochaTest:acceptance
+_test_exit_code=$?
+
+echo ">> Killing server"
+
+kill %1
+
+echo ">> Done"
+
+exit $_test_exit_code

test/acceptance/scripts/settings.test.coffee

@@ -0,0 +1,47 @@
+Path = require "path"
+
+module.exports =
+	# Options are passed to Sequelize.
+	# See http://sequelizejs.com/documentation#usage-options for details
+	mysql:
+		clsi:
+			database: "clsi"
+			username: "clsi"
+			password: null
+			dialect: "sqlite"
+			storage: Path.resolve("db.sqlite")
+
+	path:
+		compilesDir:  Path.resolve(__dirname + "/../../../compiles")
+		clsiCacheDir: Path.resolve(__dirname + "/../../../cache")
+		#synctexBaseDir: (project_id) -> Path.join(@compilesDir, project_id)
+		synctexBaseDir: () -> "/compile"
+		sandboxedCompilesHostDir: process.env['SANDBOXED_COMPILES_HOST_DIR']
+
+	clsi:
+		#strace: true
+		#archive_logs: true
+		commandRunner: "docker-runner-sharelatex"
+		latexmkCommandPrefix: ["/usr/bin/time", "-v"]         # on Linux
+		docker:
+			image: process.env.TEXLIVE_IMAGE || "texlive-full:2017.1-opt"
+			env:
+				PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/2017/bin/x86_64-linux/"
+				HOME: "/tmp"
+			modem:
+				socketPath: false
+			user: process.env.SIBLING_CONTAINER_USER ||"111"
+
+	internal:
+		clsi:
+			port: 3013
+			load_port: 3044
+			host: "localhost"
+
+	apis:
+		clsi:
+			url: "http://localhost:3013"
+
+	smokeTest: false
+	project_cache_length_ms: 1000 * 60 * 60 * 24
+	parallelFileDownloads:1

test/smoke/coffee/SmokeTests.coffee

@@ -6,19 +6,48 @@ Settings = require "settings-sharelatex"
 
 buildUrl = (path) -> "http://#{Settings.internal.clsi.host}:#{Settings.internal.clsi.port}/#{path}"
 
+url = buildUrl("project/smoketest-#{process.pid}/compile")
+
 describe "Running a compile", ->
 	before (done) ->
 		request.post {
-			url: buildUrl("project/smoketest/compile")
+			url: url
 			json:
 				compile:
 					resources: [
 						path: "main.tex"
 						content: """
-							\\documentclass{article}
-							\\begin{document}
-							Hello world
-							\\end{document}
+% Membrane-like surface
+% Author: Yotam Avital
+\\documentclass{article}
+\\usepackage{tikz}
+\\usetikzlibrary{calc,fadings,decorations.pathreplacing}
+\\begin{document}
+\\begin{tikzpicture}
+  \\def\\nuPi{3.1459265}
+  \\foreach \\i in {5,4,...,2}{% This one doesn't matter
+    \\foreach \\j in {3,2,...,0}{% This will crate a membrane
+                               % with the front lipids visible
+      % top layer
+      \\pgfmathsetmacro{\\dx}{rand*0.1}% A random variance in the x coordinate
+      \\pgfmathsetmacro{\\dy}{rand*0.1}% A random variance in the y coordinate,
+                                     % gives a hight fill to the lipid
+      \\pgfmathsetmacro{\\rot}{rand*0.1}% A random variance in the
+                                      % molecule orientation      
+      \\shade[ball color=red] ({\\i+\\dx+\\rot},{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)}) circle(0.45);
+      \\shade[ball color=gray] (\\i+\\dx,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-0.9}) circle(0.45);
+      \\shade[ball color=gray] (\\i+\\dx-\\rot,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-1.8}) circle(0.45);
+      % bottom layer
+      \\pgfmathsetmacro{\\dx}{rand*0.1}
+      \\pgfmathsetmacro{\\dy}{rand*0.1}
+      \\pgfmathsetmacro{\\rot}{rand*0.1}
+      \\shade[ball color=gray] (\\i+\\dx+\\rot,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-2.8}) circle(0.45);
+      \\shade[ball color=gray] (\\i+\\dx,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-3.7}) circle(0.45);
+      \\shade[ball color=red] (\\i+\\dx-\\rot,{0.5*\\j+\\dy+0.4*sin(\\i*\\nuPi*10)-4.6}) circle(0.45);
+    }
+  }
+\\end{tikzpicture}
+\\end{document}
 						"""
 					]
 		}, (@error, @response, @body) =>

test/unit/coffee/CompileControllerTests.coffee

@@ -14,7 +14,7 @@ describe "CompileController", ->
 					clsi:
 						url: "http://clsi.example.com"
 			"./ProjectPersistenceManager": @ProjectPersistenceManager = {}
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), err:sinon.stub(), warn: sinon.stub()}
 		@Settings.externalUrl = "http://www.example.com"
 		@req = {}
 		@res = {}
@@ -49,7 +49,7 @@ describe "CompileController", ->
 
 		describe "successfully", ->
 			beforeEach ->
-				@CompileManager.doCompile = sinon.stub().callsArgWith(1, null, @output_files)
+				@CompileManager.doCompileWithLock = sinon.stub().callsArgWith(1, null, @output_files)
 				@CompileController.compile @req, @res
 
 			it "should parse the request", ->
@@ -58,7 +58,7 @@ describe "CompileController", ->
 					.should.equal true
 
 			it "should run the compile for the specified project", ->
-				@CompileManager.doCompile
+				@CompileManager.doCompileWithLock
 					.calledWith(@request_with_project_id)
 					.should.equal true
 
@@ -75,7 +75,8 @@ describe "CompileController", ->
 							status: "success"
 							error: null
 							outputFiles: @output_files.map (file) =>
-								url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/output/#{file.path}"
+								url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/build/#{file.build}/output/#{file.path}"
+								path: file.path
 								type: file.type
 								build: file.build
 					)
@@ -83,7 +84,7 @@ describe "CompileController", ->
 			
 		describe "with an error", ->
 			beforeEach ->
-				@CompileManager.doCompile = sinon.stub().callsArgWith(1, new Error(@message = "error message"), null)
+				@CompileManager.doCompileWithLock = sinon.stub().callsArgWith(1, new Error(@message = "error message"), null)
 				@CompileController.compile @req, @res
 		
 			it "should return the JSON response with the error", ->
@@ -101,7 +102,7 @@ describe "CompileController", ->
 			beforeEach ->
 				@error = new Error(@message = "container timed out")
 				@error.timedout = true
-				@CompileManager.doCompile = sinon.stub().callsArgWith(1, @error, null)
+				@CompileManager.doCompileWithLock = sinon.stub().callsArgWith(1, @error, null)
 				@CompileController.compile @req, @res
 		
 			it "should return the JSON response with the timeout status", ->
@@ -117,7 +118,7 @@ describe "CompileController", ->
 
 		describe "when the request returns no output files", ->
 			beforeEach ->
-				@CompileManager.doCompile = sinon.stub().callsArgWith(1, null, [])
+				@CompileManager.doCompileWithLock = sinon.stub().callsArgWith(1, null, [])
 				@CompileController.compile @req, @res
 		
 			it "should return the JSON response with the failure status", ->
@@ -143,19 +144,19 @@ describe "CompileController", ->
 				file: @file
 				line: @line.toString()
 				column: @column.toString()
-			@res.send = sinon.stub()
+			@res.json = sinon.stub()
 
-			@CompileManager.syncFromCode = sinon.stub().callsArgWith(4, null, @pdfPositions = ["mock-positions"])
+			@CompileManager.syncFromCode = sinon.stub().callsArgWith(5, null, @pdfPositions = ["mock-positions"])
 			@CompileController.syncFromCode @req, @res, @next
 
 		it "should find the corresponding location in the PDF", ->
 			@CompileManager.syncFromCode
-				.calledWith(@project_id, @file, @line, @column)
+				.calledWith(@project_id, undefined, @file, @line, @column)
 				.should.equal true
 
 		it "should return the positions", ->
-			@res.send
-				.calledWith(JSON.stringify
+			@res.json
+				.calledWith(
 					pdf: @pdfPositions
 				)
 				.should.equal true
@@ -172,19 +173,19 @@ describe "CompileController", ->
 				page: @page.toString()
 				h: @h.toString()
 				v: @v.toString()
-			@res.send = sinon.stub()
+			@res.json = sinon.stub()
 
-			@CompileManager.syncFromPdf = sinon.stub().callsArgWith(4, null, @codePositions = ["mock-positions"])
+			@CompileManager.syncFromPdf = sinon.stub().callsArgWith(5, null, @codePositions = ["mock-positions"])
 			@CompileController.syncFromPdf @req, @res, @next
 
 		it "should find the corresponding location in the code", ->
 			@CompileManager.syncFromPdf
-				.calledWith(@project_id, @page, @h, @v)
+				.calledWith(@project_id, undefined, @page, @h, @v)
 				.should.equal true
 
 		it "should return the positions", ->
-			@res.send
-				.calledWith(JSON.stringify
+			@res.json
+				.calledWith(
 					code: @codePositions
 				)
 				.should.equal true
@@ -197,19 +198,20 @@ describe "CompileController", ->
 				project_id: @project_id
 			@req.query =
 				file: @file
-			@res.send = sinon.stub()
+				image: @image = "example.com/image"
+			@res.json = sinon.stub()
 
-			@CompileManager.wordcount = sinon.stub().callsArgWith(2, null, @texcount = ["mock-texcount"])
+			@CompileManager.wordcount = sinon.stub().callsArgWith(4, null, @texcount = ["mock-texcount"])
 			@CompileController.wordcount @req, @res, @next
 
 		it "should return the word count of a file", ->
 			@CompileManager.wordcount
-				.calledWith(@project_id, @file)
+				.calledWith(@project_id, undefined, @file, @image)
 				.should.equal true
 
 		it "should return the texcount info", ->
-			@res.send
-				.calledWith(JSON.stringify
+			@res.json
+				.calledWith(
 					texcount: @texcount
 				)
 				.should.equal true

test/unit/coffee/CompileManagerTests.coffee

@@ -13,12 +13,74 @@ describe "CompileManager", ->
 			"./ResourceWriter": @ResourceWriter = {}
 			"./OutputFileFinder": @OutputFileFinder = {}
 			"./OutputCacheManager": @OutputCacheManager = {}
-			"settings-sharelatex": @Settings = { path: compilesDir: "/compiles/dir" }
-			"logger-sharelatex": @logger = { log: sinon.stub() }
+			"settings-sharelatex": @Settings =
+				path:
+					compilesDir: "/compiles/dir"
+				synctexBaseDir: -> "/compile"
+				clsi:
+					docker:
+						image: "SOMEIMAGE"
+
+			"logger-sharelatex": @logger = { log: sinon.stub() , info:->}
 			"child_process": @child_process = {}
 			"./CommandRunner": @CommandRunner = {}
+			"./DraftModeManager": @DraftModeManager = {}
+			"./TikzManager": @TikzManager = {}
+			"./LockManager": @LockManager = {}
 			"fs": @fs = {}
+			"fs-extra": @fse = { ensureDir: sinon.stub().callsArg(1) }
 		@callback = sinon.stub()
+		@project_id = "project-id-123"
+		@user_id = "1234"
+	describe "doCompileWithLock", ->
+		beforeEach ->
+			@request =
+				resources: @resources = "mock-resources"
+				project_id: @project_id
+				user_id: @user_id
+			@output_files = ["foo", "bar"]
+			@Settings.compileDir = "compiles"
+			@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
+			@CompileManager.doCompile = sinon.stub().callsArgWith(1, null, @output_files)
+			@LockManager.runWithLock = (lockFile, runner, callback) ->
+				runner (err, result...) ->
+					callback(err, result...)
+
+		describe "when the project is not locked", ->
+			beforeEach ->
+				@CompileManager.doCompileWithLock @request, @callback
+
+			it "should ensure that the compile directory exists", ->
+				@fse.ensureDir.calledWith(@compileDir)
+				.should.equal true
+
+			it "should call doCompile with the request", ->
+				@CompileManager.doCompile
+				.calledWith(@request)
+				.should.equal true
+
+			it "should call the callback with the output files", ->
+				@callback.calledWithExactly(null, @output_files)
+				.should.equal true
+
+		describe "when the project is locked", ->
+			beforeEach ->
+				@error = new Error("locked")
+				@LockManager.runWithLock = (lockFile, runner, callback) =>
+					callback(@error)
+				@CompileManager.doCompileWithLock @request, @callback
+
+			it "should ensure that the compile directory exists", ->
+				@fse.ensureDir.calledWith(@compileDir)
+				.should.equal true
+
+			it "should not call doCompile with the request", ->
+				@CompileManager.doCompile
+				.called.should.equal false
+
+			it "should call the callback with the error", ->
+				@callback.calledWithExactly(@error)
+				.should.equal true
 
 	describe "doCompile", ->
 		beforeEach ->
@@ -41,54 +103,117 @@ describe "CompileManager", ->
 			@request =
 				resources: @resources = "mock-resources"
 				rootResourcePath: @rootResourcePath = "main.tex"
-				project_id: @project_id = "project-id-123"
+				project_id: @project_id
+				user_id: @user_id
 				compiler: @compiler = "pdflatex"
 				timeout: @timeout = 42000
+				imageName: @image = "example.com/image"
+				flags: @flags = ["-file-line-error"]
+			@env = {}
 			@Settings.compileDir = "compiles"
-			@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}"
-			@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3)
+			@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
+			@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArgWith(2, null, @resources)
 			@LatexRunner.runLatex = sinon.stub().callsArg(2)
 			@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
 			@OutputCacheManager.saveOutputFiles = sinon.stub().callsArgWith(2, null, @build_files)
-			@CompileManager.doCompile @request, @callback
+			@DraftModeManager.injectDraftMode = sinon.stub().callsArg(1)
+			@TikzManager.checkMainFile = sinon.stub().callsArg(3, false)
 
-		it "should write the resources to disk", ->
-			@ResourceWriter.syncResourcesToDisk
-				.calledWith(@project_id, @resources, @compileDir)
-				.should.equal true
+		describe "normally", ->
+			beforeEach ->
+				@CompileManager.doCompile @request, @callback
 
-		it "should run LaTeX", ->
-			@LatexRunner.runLatex
-				.calledWith(@project_id, {
-					directory: @compileDir
-					mainFile:  @rootResourcePath
-					compiler:  @compiler
-					timeout:   @timeout
-				})
-				.should.equal true
+			it "should write the resources to disk", ->
+				@ResourceWriter.syncResourcesToDisk
+					.calledWith(@request, @compileDir)
+					.should.equal true
 
-		it "should find the output files", ->
-			@OutputFileFinder.findOutputFiles
-				.calledWith(@resources, @compileDir)
-				.should.equal true
+			it "should run LaTeX", ->
+				@LatexRunner.runLatex
+					.calledWith("#{@project_id}-#{@user_id}", {
+						directory: @compileDir
+						mainFile:  @rootResourcePath
+						compiler:  @compiler
+						timeout:   @timeout
+						image:     @image
+						flags:     @flags
+						environment: @env
+					})
+					.should.equal true
 
-		it "should return the output files", ->
-			@callback.calledWith(null, @build_files).should.equal true
+			it "should find the output files", ->
+				@OutputFileFinder.findOutputFiles
+					.calledWith(@resources, @compileDir)
+					.should.equal true
+
+			it "should return the output files", ->
+				@callback.calledWith(null, @build_files).should.equal true
+
+			it "should not inject draft mode by default", ->
+				@DraftModeManager.injectDraftMode.called.should.equal false
+
+		describe "with draft mode", ->
+			beforeEach ->
+				@request.draft = true
+				@CompileManager.doCompile @request, @callback
+
+			it "should inject the draft mode header", ->
+				@DraftModeManager.injectDraftMode
+					.calledWith(@compileDir + "/" + @rootResourcePath)
+					.should.equal true
+
+		describe "with a check option", ->
+			beforeEach ->
+				@request.check = "error"
+				@CompileManager.doCompile @request, @callback
+
+			it "should run chktex", ->
+				@LatexRunner.runLatex
+					.calledWith("#{@project_id}-#{@user_id}", {
+						directory: @compileDir
+						mainFile:  @rootResourcePath
+						compiler:  @compiler
+						timeout:   @timeout
+						image:     @image
+						flags:     @flags
+						environment: {'CHKTEX_OPTIONS': '-nall -e9 -e10 -w15 -w16', 'CHKTEX_EXIT_ON_ERROR':1, 'CHKTEX_ULIMIT_OPTIONS': '-t 5 -v 64000'}
+					})
+					.should.equal true
+
+		describe "with a knitr file and check options", ->
+			beforeEach ->
+				@request.rootResourcePath = "main.Rtex"
+				@request.check = "error"
+				@CompileManager.doCompile @request, @callback
+
+			it "should not run chktex", ->
+				@LatexRunner.runLatex
+					.calledWith("#{@project_id}-#{@user_id}", {
+						directory: @compileDir
+						mainFile:  "main.Rtex"
+						compiler:  @compiler
+						timeout:   @timeout
+						image:     @image
+						flags:     @flags
+						environment: @env
+					})
+					.should.equal true
 
 	describe "clearProject", ->
 		describe "succesfully", ->
 			beforeEach ->
 				@Settings.compileDir = "compiles"
+				@fs.lstat = sinon.stub().callsArgWith(1, null,{isDirectory: ()->true})
 				@proc = new EventEmitter()
 				@proc.stdout = new EventEmitter()
 				@proc.stderr = new EventEmitter()
 				@child_process.spawn = sinon.stub().returns(@proc)
-				@CompileManager.clearProject @project_id, @callback
+				@CompileManager.clearProject @project_id, @user_id, @callback
 				@proc.emit "close", 0
 
 			it "should remove the project directory", ->
 				@child_process.spawn
-					.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"])
+					.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"])
 					.should.equal true
 
 			it "should call the callback", ->
@@ -97,17 +222,18 @@ describe "CompileManager", ->
 		describe "with a non-success status code", ->
 			beforeEach ->
 				@Settings.compileDir = "compiles"
+				@fs.lstat = sinon.stub().callsArgWith(1, null,{isDirectory: ()->true})
 				@proc = new EventEmitter()
 				@proc.stdout = new EventEmitter()
 				@proc.stderr = new EventEmitter()
 				@child_process.spawn = sinon.stub().returns(@proc)
-				@CompileManager.clearProject @project_id, @callback
+				@CompileManager.clearProject @project_id, @user_id, @callback
 				@proc.stderr.emit "data", @error = "oops"
 				@proc.emit "close", 1
 
 			it "should remove the project directory", ->
 				@child_process.spawn
-					.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"])
+					.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"])
 					.should.equal true
 
 			it "should call the callback with an error from the stderr", ->
@@ -115,7 +241,7 @@ describe "CompileManager", ->
 					.calledWith(new Error())
 					.should.equal true
 
-				@callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id} failed: #{@error}"
+				@callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id}-#{@user_id} failed: #{@error}"
 
 	describe "syncing", ->
 		beforeEach ->
@@ -128,20 +254,28 @@ describe "CompileManager", ->
 			@column = 3
 			@file_name = "main.tex"
 			@child_process.execFile = sinon.stub()
-			@Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}"
+			@Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
 
 		describe "syncFromCode", ->
 			beforeEach ->
-				@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n", "")
-				@CompileManager.syncFromCode @project_id, @file_name, @line, @column, @callback
+				@fs.stat = sinon.stub().callsArgWith(1, null,{isFile: ()->true})
+				@stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n"
+				@CommandRunner.run = sinon.stub().callsArgWith(6, null, {stdout:@stdout})
+				@CompileManager.syncFromCode @project_id, @user_id, @file_name, @line, @column, @callback
 
 			it "should execute the synctex binary", ->
 				bin_path = Path.resolve(__dirname + "/../../../bin/synctex")
-				synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf"
-				file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}"
-				@child_process.execFile
-					.calledWith(bin_path, ["code", synctex_path, file_path, @line, @column], timeout: 10000)
-					.should.equal true
+				synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf"
+				file_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}"
+				@CommandRunner.run
+					.calledWith(
+						"#{@project_id}-#{@user_id}",
+						['/opt/synctex', 'code', synctex_path, file_path, @line, @column],
+						"#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}",
+						@Settings.clsi.docker.image,
+						60000,
+						{}
+						).should.equal true
 
 			it "should call the callback with the parsed output", ->
 				@callback
@@ -156,15 +290,22 @@ describe "CompileManager", ->
 
 		describe "syncFromPdf", ->
 			beforeEach ->
-				@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}\t#{@line}\t#{@column}\n", "")
-				@CompileManager.syncFromPdf @project_id, @page, @h, @v, @callback
+				@fs.stat = sinon.stub().callsArgWith(1, null,{isFile: ()->true})
+				@stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n"
+				@CommandRunner.run = sinon.stub().callsArgWith(6, null, {stdout:@stdout})
+				@CompileManager.syncFromPdf @project_id, @user_id, @page, @h, @v, @callback
 
 			it "should execute the synctex binary", ->
 				bin_path = Path.resolve(__dirname + "/../../../bin/synctex")
-				synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf"
-				@child_process.execFile
-					.calledWith(bin_path, ["pdf", synctex_path, @page, @h, @v], timeout: 10000)
-					.should.equal true
+				synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf"
+				@CommandRunner.run
+					.calledWith(
+						"#{@project_id}-#{@user_id}",
+						['/opt/synctex', "pdf", synctex_path, @page, @h, @v],
+						"#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}",
+						@Settings.clsi.docker.image,
+						60000,
+						{}).should.equal true
 
 			it "should call the callback with the parsed output", ->
 				@callback
@@ -177,24 +318,25 @@ describe "CompileManager", ->
 
 	describe "wordcount", ->
 		beforeEach ->
-			@CommandRunner.run = sinon.stub().callsArg(4)
-			@fs.readFileSync = sinon.stub().returns @stdout = "Encoding: ascii\nWords in text: 2"
+			@CommandRunner.run = sinon.stub().callsArg(6)
+			@fs.readFile = sinon.stub().callsArgWith(2, null, @stdout = "Encoding: ascii\nWords in text: 2")
 			@callback  = sinon.stub()
 
-			@project_id = "project-id-123"
-			@timeout = 10 * 1000
+			@project_id
+			@timeout = 60 * 1000
 			@file_name = "main.tex"
 			@Settings.path.compilesDir = "/local/compile/directory"
+			@image = "example.com/image"
 
-			@CompileManager.wordcount @project_id, @file_name, @callback
+			@CompileManager.wordcount @project_id, @user_id, @file_name, @image, @callback
 
 		it "should run the texcount command", ->
-			@directory = "#{@Settings.path.compilesDir}/#{@project_id}"
+			@directory = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
 			@file_path = "$COMPILE_DIR/#{@file_name}"
-			@command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"]
+			@command =[ "texcount", "-nocol", "-inc", @file_path, "-out=" + @file_path + ".wc"]
 
 			@CommandRunner.run
-				.calledWith(@project_id, @command, @directory, @timeout)
+				.calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout, {})
 				.should.equal true
 
 		it "should call the callback with the parsed output", ->
@@ -208,5 +350,7 @@ describe "CompileManager", ->
 					elements: 0
 					mathInline: 0
 					mathDisplay: 0
+					errors: 0
+					messages: ""
 				})
 				.should.equal true

test/unit/coffee/ContentTypeMapperTests.coffee

@@ -0,0 +1,55 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/ContentTypeMapper'
+
+describe 'ContentTypeMapper', ->
+
+	beforeEach ->
+		@ContentTypeMapper = SandboxedModule.require modulePath
+
+	describe 'map', ->
+
+		it 'should map .txt to text/plain', ->
+			content_type = @ContentTypeMapper.map('example.txt')
+			content_type.should.equal 'text/plain'
+
+		it 'should map .csv to text/csv', ->
+			content_type = @ContentTypeMapper.map('example.csv')
+			content_type.should.equal 'text/csv'
+
+		it 'should map .pdf to application/pdf', ->
+			content_type = @ContentTypeMapper.map('example.pdf')
+			content_type.should.equal 'application/pdf'
+
+		it 'should fall back to octet-stream', ->
+			content_type = @ContentTypeMapper.map('example.unknown')
+			content_type.should.equal 'application/octet-stream'
+
+		describe 'coercing web files to plain text', ->
+
+			it 'should map .js to plain text', ->
+				content_type = @ContentTypeMapper.map('example.js')
+				content_type.should.equal 'text/plain'
+
+			it 'should map .html to plain text', ->
+				content_type = @ContentTypeMapper.map('example.html')
+				content_type.should.equal 'text/plain'
+
+			it 'should map .css to plain text', ->
+				content_type = @ContentTypeMapper.map('example.css')
+				content_type.should.equal 'text/plain'
+
+		describe 'image files', ->
+
+			it 'should map .png to image/png', ->
+				content_type = @ContentTypeMapper.map('example.png')
+				content_type.should.equal 'image/png'
+
+			it 'should map .jpeg to image/jpeg', ->
+				content_type = @ContentTypeMapper.map('example.jpeg')
+				content_type.should.equal 'image/jpeg'
+				
+			it 'should map .svg to text/plain to protect against XSS (SVG can execute JS)', ->
+				content_type = @ContentTypeMapper.map('example.svg')
+				content_type.should.equal 'text/plain'

test/unit/coffee/DockerLockManagerTests.coffee

@@ -0,0 +1,145 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+require "coffee-script"
+modulePath = require('path').join __dirname, '../../../app/coffee/DockerLockManager'
+
+describe "LockManager", ->
+	beforeEach ->
+		@LockManager = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @Settings =
+				clsi: docker: {}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+
+	describe "runWithLock", ->
+		describe "with a single lock", ->
+			beforeEach (done) ->
+				@callback  = sinon.stub()
+				@LockManager.runWithLock "lock-one", (releaseLock) ->
+					setTimeout () ->
+						releaseLock(null, "hello", "world")
+					, 100
+				, (err, args...) =>
+					@callback(err,args...)
+					done()
+
+			it "should call the callback", ->
+				@callback.calledWith(null,"hello","world").should.equal true
+
+		describe "with two locks", ->
+			beforeEach (done) ->
+				@callback1 = sinon.stub()
+				@callback2 = sinon.stub()
+				@LockManager.runWithLock "lock-one", (releaseLock) ->
+					setTimeout () ->
+						releaseLock(null, "hello", "world","one")
+					, 100
+				, (err, args...) =>
+					@callback1(err,args...)
+				@LockManager.runWithLock "lock-two", (releaseLock) ->
+					setTimeout () ->
+						releaseLock(null, "hello", "world","two")
+					, 200
+				, (err, args...) =>
+					@callback2(err,args...)
+					done()
+
+			it "should call the first callback", ->
+				@callback1.calledWith(null,"hello","world","one").should.equal true
+
+			it "should call the second callback", ->
+				@callback2.calledWith(null,"hello","world","two").should.equal true
+
+		describe "with lock contention", ->
+			describe "where the first lock is released quickly", ->
+				beforeEach (done) ->
+					@LockManager.MAX_LOCK_WAIT_TIME = 1000
+					@LockManager.LOCK_TEST_INTERVAL = 100
+					@callback1 = sinon.stub()
+					@callback2 = sinon.stub()
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","one")
+						, 100
+					, (err, args...) =>
+						@callback1(err,args...)
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","two")
+						, 200
+					, (err, args...) =>
+						@callback2(err,args...)
+						done()
+
+				it "should call the first callback", ->
+					@callback1.calledWith(null,"hello","world","one").should.equal true
+
+				it "should call the second callback", ->
+					@callback2.calledWith(null,"hello","world","two").should.equal true
+
+			describe "where the first lock is held longer than the waiting time", ->
+				beforeEach (done) ->
+					@LockManager.MAX_LOCK_HOLD_TIME = 10000
+					@LockManager.MAX_LOCK_WAIT_TIME = 1000
+					@LockManager.LOCK_TEST_INTERVAL = 100
+					@callback1 = sinon.stub()
+					@callback2 = sinon.stub()
+					doneOne = doneTwo = false
+					finish = (key) ->
+						doneOne = true if key is 1
+						doneTwo = true if key is 2
+						done() if doneOne and doneTwo
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","one")
+						, 1100
+					, (err, args...) =>
+						@callback1(err,args...)
+						finish(1)
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","two")
+						, 100
+					, (err, args...) =>
+						@callback2(err,args...)
+						finish(2)
+
+				it "should call the first callback", ->
+					@callback1.calledWith(null,"hello","world","one").should.equal true
+
+				it "should call the second callback with an error", ->
+					error = sinon.match.instanceOf Error
+					@callback2.calledWith(error).should.equal true
+
+			describe "where the first lock is held longer than the max holding time", ->
+				beforeEach (done) ->
+					@LockManager.MAX_LOCK_HOLD_TIME = 1000
+					@LockManager.MAX_LOCK_WAIT_TIME = 2000
+					@LockManager.LOCK_TEST_INTERVAL = 100
+					@callback1 = sinon.stub()
+					@callback2 = sinon.stub()
+					doneOne = doneTwo = false
+					finish = (key) ->
+						doneOne = true if key is 1
+						doneTwo = true if key is 2
+						done() if doneOne and doneTwo
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","one")
+						, 1500
+					, (err, args...) =>
+						@callback1(err,args...)
+						finish(1)
+					@LockManager.runWithLock "lock", (releaseLock) ->
+						setTimeout () ->
+							releaseLock(null, "hello", "world","two")
+						, 100
+					, (err, args...) =>
+						@callback2(err,args...)
+						finish(2)
+
+				it "should call the first callback", ->
+					@callback1.calledWith(null,"hello","world","one").should.equal true
+
+				it "should call the second callback", ->
+					@callback2.calledWith(null,"hello","world","two").should.equal true

test/unit/coffee/DockerRunnerTests.coffee

@@ -0,0 +1,509 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+expect = require('chai').expect
+require "coffee-script"
+modulePath = require('path').join __dirname, '../../../app/coffee/DockerRunner'
+Path = require "path"
+
+describe "DockerRunner", ->
+	beforeEach ->
+		@container = container = {}
+		@DockerRunner = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @Settings =
+				clsi: docker: {}
+				path: {}
+			"logger-sharelatex": @logger = {
+				log: sinon.stub(),
+				error: sinon.stub(),
+				info: sinon.stub(),
+				warn: sinon.stub()
+			}
+			"dockerode": class Docker
+				getContainer: sinon.stub().returns(container)
+				createContainer: sinon.stub().yields(null, container)
+				listContainers: sinon.stub()
+			"fs": @fs = { stat: sinon.stub().yields(null,{isDirectory:()->true}) }
+			"./Metrics":
+				Timer: class Timer
+					done: () ->
+			"./LockManager":
+				runWithLock: (key, runner, callback) -> runner(callback)
+		@Docker = Docker
+		@getContainer = Docker::getContainer
+		@createContainer = Docker::createContainer
+		@listContainers = Docker::listContainers
+
+		@directory = "/local/compile/directory"
+		@mainFile  = "main-file.tex"
+		@compiler  = "pdflatex"
+		@image     = "example.com/sharelatex/image:2016.2"
+		@env       = {}
+		@callback  = sinon.stub()
+		@project_id = "project-id-123"
+		@volumes =
+			"/local/compile/directory": "/compile"
+		@Settings.clsi.docker.image = @defaultImage = "default-image"
+		@Settings.clsi.docker.env   = PATH: "mock-path"
+
+	describe "run", ->
+		beforeEach (done)->
+			@DockerRunner._getContainerOptions = sinon.stub().returns(@options = {mockoptions: "foo"})
+			@DockerRunner._fingerprintContainer = sinon.stub().returns(@fingerprint = "fingerprint")
+
+			@name = "project-#{@project_id}-#{@fingerprint}"
+
+			@command = ["mock", "command", "--outdir=$COMPILE_DIR"]
+			@command_with_dir = ["mock", "command", "--outdir=/compile"]
+			@timeout = 42000
+			done()
+
+		describe "successfully", ->
+			beforeEach (done)->
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, (err, output)=>
+					@callback(err, output)
+					done()
+
+			it "should generate the options for the container", ->
+				@DockerRunner._getContainerOptions
+					.calledWith(@command_with_dir, @image, @volumes, @timeout)
+					.should.equal true
+
+			it "should generate the fingerprint from the returned options", ->
+				@DockerRunner._fingerprintContainer
+					.calledWith(@options)
+					.should.equal true
+
+			it "should do the run", ->
+				@DockerRunner._runAndWaitForContainer	
+					.calledWith(@options, @volumes, @timeout)		
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.calledWith(null, @output).should.equal true
+
+		describe 'when path.sandboxedCompilesHostDir is set', ->
+
+			beforeEach ->
+				@Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles'
+				@directory = '/var/lib/sharelatex/data/compiles/xyz'
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, @callback
+
+			it 'should re-write the bind directory', ->
+				volumes = @DockerRunner._runAndWaitForContainer.lastCall.args[1]
+				expect(volumes).to.deep.equal {
+					'/some/host/dir/compiles/xyz': '/compile'
+				}
+
+			it "should call the callback", ->
+				@callback.calledWith(null, @output).should.equal true
+
+		describe "when the run throws an error", ->
+			beforeEach ->
+				firstTime = true
+				@output = "mock-output"
+				@DockerRunner._runAndWaitForContainer = (options, volumes, timeout, callback = (error, output)->) =>
+					if firstTime
+						firstTime = false
+						callback new Error("HTTP code is 500 which indicates error: server error")
+					else
+						callback(null, @output)
+				sinon.spy @DockerRunner, "_runAndWaitForContainer"
+				@DockerRunner.destroyContainer = sinon.stub().callsArg(3)
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, @callback
+
+			it "should do the run twice", ->
+				@DockerRunner._runAndWaitForContainer	
+					.calledTwice.should.equal true
+
+			it "should destroy the container in between", ->
+				@DockerRunner.destroyContainer
+					.calledWith(@name, null)
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.calledWith(null, @output).should.equal true
+		
+		describe "with no image", ->
+			beforeEach ->
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, null, @timeout, @env, @callback
+
+			it "should use the default image", ->
+				@DockerRunner._getContainerOptions
+					.calledWith(@command_with_dir, @defaultImage, @volumes, @timeout)
+					.should.equal true
+		
+		describe "with image override", ->
+			beforeEach ->
+				@Settings.texliveImageNameOveride = "overrideimage.com/something"
+				@DockerRunner._runAndWaitForContainer = sinon.stub().callsArgWith(3, null, @output = "mock-output")
+				@DockerRunner.run @project_id, @command, @directory, @image, @timeout, @env, @callback
+
+			it "should use the override and keep the tag", ->
+				image = @DockerRunner._getContainerOptions.args[0][1]
+				image.should.equal "overrideimage.com/something/image:2016.2"
+
+	describe "_runAndWaitForContainer", ->
+		beforeEach ->
+			@options = {mockoptions: "foo", name: @name = "mock-name"}
+			@DockerRunner.startContainer = (options, volumes, attachStreamHandler, callback) =>
+				attachStreamHandler(null, @output = "mock-output")
+				callback(null, @containerId = "container-id")
+			sinon.spy @DockerRunner, "startContainer"
+			@DockerRunner.waitForContainer = sinon.stub().callsArgWith(2, null, @exitCode = 42)
+			@DockerRunner._runAndWaitForContainer @options, @volumes, @timeout, @callback
+
+		it "should create/start the container", ->
+			@DockerRunner.startContainer
+				.calledWith(@options, @volumes)
+				.should.equal true
+
+		it "should wait for the container to finish", ->
+			@DockerRunner.waitForContainer
+				.calledWith(@name, @timeout)
+				.should.equal true
+
+		it "should call the callback with the output", ->
+			@callback.calledWith(null, @output).should.equal true
+
+	describe "startContainer", ->
+		beforeEach ->
+			@attachStreamHandler = sinon.stub()
+			@attachStreamHandler.cock = true
+			@options = {mockoptions: "foo", name: "mock-name"}
+			@container.inspect = sinon.stub().callsArgWith(0)
+			@DockerRunner.attachToContainer = (containerId, attachStreamHandler, cb)=> 
+				attachStreamHandler()
+				cb()
+			sinon.spy @DockerRunner, "attachToContainer"
+
+
+
+		describe "when the container exists", ->
+			beforeEach ->
+				@container.inspect = sinon.stub().callsArgWith(0)
+				@container.start = sinon.stub().yields()
+
+				@DockerRunner.startContainer @options, @volumes, @callback, ->
+
+			it "should start the container with the given name", ->
+				@getContainer
+					.calledWith(@options.name)
+					.should.equal true
+				@container.start
+					.called
+					.should.equal true
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+			
+			it "should attach to the container", ->
+				@DockerRunner.attachToContainer.called.should.equal true
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+	
+			it "should attach before the container starts", ->
+				sinon.assert.callOrder(@DockerRunner.attachToContainer, @container.start)
+						
+		describe "when the container does not exist", ->
+			beforeEach ()->
+				exists = false
+				@container.start = sinon.stub().yields()
+				@container.inspect = sinon.stub().callsArgWith(0, {statusCode:404})
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should create the container", ->
+				@createContainer
+					.calledWith(@options)
+					.should.equal true
+
+			it "should call the callback and stream handler", ->
+				@attachStreamHandler.called.should.equal true
+				@callback.called.should.equal true
+			
+			it "should attach to the container", ->
+				@DockerRunner.attachToContainer.called.should.equal true
+
+			it "should attach before the container starts", ->
+				sinon.assert.callOrder(@DockerRunner.attachToContainer, @container.start)
+
+
+		describe "when the container is already running", ->
+			beforeEach ->
+				error = new Error("HTTP code is 304 which indicates error: server error - start: Cannot start container #{@name}: The container MOCKID is already running.")
+				error.statusCode = 304
+				@container.start = sinon.stub().yields(error)
+				@container.inspect = sinon.stub().callsArgWith(0)
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback  and stream handler without an error", ->
+				@attachStreamHandler.called.should.equal true
+				@callback.called.should.equal true
+
+		describe "when a volume does not exist", ->
+			beforeEach ()->
+				@fs.stat = sinon.stub().yields(new Error("no such path"))
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error()).should.equal true
+
+		describe "when a volume exists but is not a directory", ->
+			beforeEach ->
+				@fs.stat = sinon.stub().yields(null, {isDirectory: () -> return false})
+				@DockerRunner.startContainer @options, @volumes, @attachStreamHandler, @callback
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error()).should.equal true
+
+		describe "when a volume does not exist, but sibling-containers are used", ->
+			beforeEach ->
+				@fs.stat = sinon.stub().yields(new Error("no such path"))
+				@Settings.path.sandboxedCompilesHostDir = '/some/path'
+				@container.start = sinon.stub().yields()
+				@DockerRunner.startContainer @options, @volumes, @callback
+
+			afterEach ->
+				delete @Settings.path.sandboxedCompilesHostDir
+
+			it "should start the container with the given name", ->
+				@getContainer
+					.calledWith(@options.name)
+					.should.equal true
+				@container.start
+					.called
+					.should.equal true
+
+			it "should not try to create the container", ->
+				@createContainer.called.should.equal false
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+				@callback.calledWith(new Error()).should.equal false
+
+		describe "when the container tries to be created, but already has been (race condition)", ->
+
+	describe "waitForContainer", ->
+		beforeEach ->
+			@containerId = "container-id"
+			@timeout = 5000
+			@container.wait = sinon.stub().yields(null, StatusCode: @statusCode = 42)
+			@container.kill = sinon.stub().yields()
+			
+		describe "when the container returns in time", ->
+			beforeEach ->
+				@DockerRunner.waitForContainer @containerId, @timeout, @callback
+
+			it "should wait for the container", ->
+				@getContainer
+					.calledWith(@containerId)
+					.should.equal true
+				@container.wait
+					.called
+					.should.equal true
+
+			it "should call the callback with the exit", ->
+				@callback
+					.calledWith(null, @statusCode)
+					.should.equal true
+
+		describe "when the container does not return before the timeout", ->
+			beforeEach (done) ->
+				@container.wait = (callback = (error, exitCode) ->) ->
+					setTimeout () ->
+						callback(null, StatusCode: 42)
+					, 100
+				@timeout = 5
+				@DockerRunner.waitForContainer @containerId, @timeout, (args...) =>
+					@callback(args...)
+					done()
+
+			it "should call kill on the container", ->
+				@getContainer
+					.calledWith(@containerId)
+					.should.equal true
+				@container.kill
+					.called
+					.should.equal true
+
+			it "should call the callback with an error", ->
+				error = new Error("container timed out")
+				error.timedout = true
+				@callback
+					.calledWith(error)
+					.should.equal true
+
+	describe "destroyOldContainers", ->
+		beforeEach (done) ->
+			oneHourInSeconds = 60 * 60
+			oneHourInMilliseconds = oneHourInSeconds * 1000
+			nowInSeconds = Date.now()/1000
+			@containers = [{
+				Name: "/project-old-container-name"
+				Id: "old-container-id"
+				Created: nowInSeconds - oneHourInSeconds - 100
+			}, {
+				Name: "/project-new-container-name"
+				Id: "new-container-id"
+				Created: nowInSeconds - oneHourInSeconds + 100
+			}, {
+				Name: "/totally-not-a-project-container"
+				Id: "some-random-id"
+				Created: nowInSeconds - (2 * oneHourInSeconds )
+			}]
+			@DockerRunner.MAX_CONTAINER_AGE = oneHourInMilliseconds
+			@listContainers.callsArgWith(1, null, @containers)
+			@DockerRunner.destroyContainer = sinon.stub().callsArg(3)
+			@DockerRunner.destroyOldContainers (error) =>
+				@callback(error)
+				done()
+
+		it "should list all containers", ->
+			@listContainers
+				.calledWith(all: true)
+				.should.equal true
+
+		it "should destroy old containers", ->
+			@DockerRunner.destroyContainer
+				.callCount
+				.should.equal 1
+			@DockerRunner.destroyContainer
+				.calledWith("/project-old-container-name", "old-container-id")
+				.should.equal true
+
+		it "should not destroy new containers", ->
+			@DockerRunner.destroyContainer
+				.calledWith("/project-new-container-name", "new-container-id")
+				.should.equal false
+
+		it "should not destroy non-project containers", ->
+			@DockerRunner.destroyContainer
+				.calledWith("/totally-not-a-project-container", "some-random-id")
+				.should.equal false
+
+		it "should callback the callback", ->
+			@callback.called.should.equal true
+
+
+	describe '_destroyContainer', ->
+		beforeEach ->
+			@containerId = 'some_id'
+			@fakeContainer =
+				remove: sinon.stub().callsArgWith(1, null)
+			@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+		it 'should get the container', (done) ->
+			@DockerRunner._destroyContainer @containerId, false, (err) =>
+				@Docker::getContainer.callCount.should.equal 1
+				@Docker::getContainer.calledWith(@containerId).should.equal true
+				done()
+
+		it 'should try to force-destroy the container when shouldForce=true', (done) ->
+			@DockerRunner._destroyContainer @containerId, true, (err) =>
+				@fakeContainer.remove.callCount.should.equal 1
+				@fakeContainer.remove.calledWith({force: true}).should.equal true
+				done()
+
+		it 'should not try to force-destroy the container when shouldForce=false', (done) ->
+			@DockerRunner._destroyContainer @containerId, false, (err) =>
+				@fakeContainer.remove.callCount.should.equal 1
+				@fakeContainer.remove.calledWith({force: false}).should.equal true
+				done()
+
+		it 'should not produce an error', (done) ->
+			@DockerRunner._destroyContainer @containerId, false, (err) =>
+				expect(err).to.equal null
+				done()
+
+		describe 'when the container is already gone', ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 404
+				@fakeContainer =
+					remove: sinon.stub().callsArgWith(1, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should not produce an error', (done) ->
+				@DockerRunner._destroyContainer @containerId, false, (err) =>
+					expect(err).to.equal null
+					done()
+
+		describe 'when container.destroy produces an error', (done) ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 500
+				@fakeContainer =
+					remove: sinon.stub().callsArgWith(1, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should produce an error', (done) ->
+				@DockerRunner._destroyContainer @containerId, false, (err) =>
+					expect(err).to.not.equal null
+					expect(err).to.equal @fakeError
+					done()
+
+
+	describe 'kill', ->
+		beforeEach ->
+			@containerId = 'some_id'
+			@fakeContainer =
+				kill: sinon.stub().callsArgWith(0, null)
+			@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+		it 'should get the container', (done) ->
+			@DockerRunner.kill @containerId, (err) =>
+				@Docker::getContainer.callCount.should.equal 1
+				@Docker::getContainer.calledWith(@containerId).should.equal true
+				done()
+
+		it 'should try to force-destroy the container', (done) ->
+			@DockerRunner.kill @containerId, (err) =>
+				@fakeContainer.kill.callCount.should.equal 1
+				done()
+
+		it 'should not produce an error', (done) ->
+			@DockerRunner.kill @containerId, (err) =>
+				expect(err).to.equal undefined
+				done()
+
+		describe 'when the container is not actually running', ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 500
+				@fakeError.message = 'Cannot kill container <whatever> is not running'
+				@fakeContainer =
+					kill: sinon.stub().callsArgWith(0, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should not produce an error', (done) ->
+				@DockerRunner.kill @containerId, (err) =>
+					expect(err).to.equal undefined
+					done()
+
+		describe 'when container.kill produces a legitimate error', (done) ->
+			beforeEach ->
+				@fakeError = new Error('woops')
+				@fakeError.statusCode = 500
+				@fakeError.message = 'Totally legitimate reason to throw an error'
+				@fakeContainer =
+					kill: sinon.stub().callsArgWith(0, @fakeError)
+				@Docker::getContainer = sinon.stub().returns(@fakeContainer)
+
+			it 'should produce an error', (done) ->
+				@DockerRunner.kill @containerId, (err) =>
+					expect(err).to.not.equal undefined
+					expect(err).to.equal @fakeError
+					done()

test/unit/coffee/DraftModeManagerTests.coffee

@@ -0,0 +1,61 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/DraftModeManager'
+
+describe 'DraftModeManager', ->
+	beforeEach ->
+		@DraftModeManager = SandboxedModule.require modulePath, requires:
+			"fs": @fs = {}
+			"logger-sharelatex": @logger = {log: () ->}
+		
+	describe "_injectDraftOption", ->
+		it "should add draft option into documentclass with existing options", ->
+			@DraftModeManager
+				._injectDraftOption('''
+					\\documentclass[a4paper,foo=bar]{article}
+				''')
+				.should.equal('''
+					\\documentclass[draft,a4paper,foo=bar]{article}
+				''')
+
+		it "should add draft option into documentclass with no options", ->
+			@DraftModeManager
+				._injectDraftOption('''
+					\\documentclass{article}
+				''')
+				.should.equal('''
+					\\documentclass[draft]{article}
+				''')
+	
+	describe "injectDraftMode", ->
+		beforeEach ->
+			@filename = "/mock/filename.tex"
+			@callback = sinon.stub()
+			content = '''
+				\\documentclass{article}
+				\\begin{document}
+				Hello world
+				\\end{document}
+			'''
+			@fs.readFile = sinon.stub().callsArgWith(2, null, content)
+			@fs.writeFile = sinon.stub().callsArg(2)
+			@DraftModeManager.injectDraftMode @filename, @callback
+	
+		it "should read the file", ->
+			@fs.readFile
+				.calledWith(@filename, "utf8")
+				.should.equal true
+		
+		it "should write the modified file", ->
+			@fs.writeFile
+				.calledWith(@filename, """
+					\\documentclass[draft]{article}
+					\\begin{document}
+					Hello world
+					\\end{document}
+				""")
+				.should.equal true
+		
+		it "should call the callback", ->
+			@callback.called.should.equal true

test/unit/coffee/LatexRunnerTests.coffee

@@ -19,12 +19,14 @@ describe "LatexRunner", ->
 		@directory = "/local/compile/directory"
 		@mainFile  = "main-file.tex"
 		@compiler  = "pdflatex"
+		@image     = "example.com/image"
 		@callback  = sinon.stub()
 		@project_id = "project-id-123"
+		@env       = {'foo': '123'}
 
 	describe "runLatex", ->
 		beforeEach ->
-			@CommandRunner.run = sinon.stub().callsArg(4)
+			@CommandRunner.run = sinon.stub().callsArg(6)
 
 		describe "normally", ->
 			beforeEach ->
@@ -33,11 +35,13 @@ describe "LatexRunner", ->
 					mainFile:  @mainFile
 					compiler:  @compiler
 					timeout:   @timeout = 42000
+					image:     @image
+					environment: @env
 					@callback
 
 			it "should run the latex command", ->
 				@CommandRunner.run
-					.calledWith(@project_id, sinon.match.any, @directory, @timeout)
+					.calledWith(@project_id, sinon.match.any, @directory, @image, @timeout, @env)
 					.should.equal true
 
 		describe "with an .Rtex main file", ->
@@ -46,6 +50,7 @@ describe "LatexRunner", ->
 					directory: @directory
 					mainFile:  "main-file.Rtex"
 					compiler:  @compiler
+					image:     @image
 					timeout:   @timeout = 42000
 					@callback
 
@@ -54,3 +59,21 @@ describe "LatexRunner", ->
 				mainFile = command.slice(-1)[0]
 				mainFile.should.equal "$COMPILE_DIR/main-file.tex"
 
+		describe "with a flags option", ->
+			beforeEach ->
+				@LatexRunner.runLatex @project_id,
+					directory: @directory
+					mainFile:  @mainFile
+					compiler:  @compiler
+					image:     @image
+					timeout:   @timeout = 42000
+					flags:     ["-file-line-error", "-halt-on-error"]
+					@callback
+
+			it "should include the flags in the command", ->
+				command = @CommandRunner.run.args[0][1]
+				flags = command.filter (arg) ->
+					(arg == "-file-line-error") || (arg == "-halt-on-error")
+				flags.length.should.equal 2
+				flags[0].should.equal "-file-line-error"
+				flags[1].should.equal "-halt-on-error"

test/unit/coffee/LockManagerTests.coffee

@@ -0,0 +1,57 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/LockManager'
+Path = require "path"
+Errors = require "../../../app/js/Errors"
+
+describe "DockerLockManager", ->
+	beforeEach ->
+		@LockManager = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": {}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), err:-> }
+			"fs":
+				lstat:sinon.stub().callsArgWith(1)
+				readdir: sinon.stub().callsArgWith(1)
+			"lockfile": @Lockfile = {}
+		@lockFile = "/local/compile/directory/.project-lock"
+
+	describe "runWithLock", ->
+		beforeEach ->
+			@runner = sinon.stub().callsArgWith(0, null, "foo", "bar")
+			@callback = sinon.stub()
+
+		describe "normally", ->
+			beforeEach ->
+				@Lockfile.lock = sinon.stub().callsArgWith(2, null)
+				@Lockfile.unlock = sinon.stub().callsArgWith(1, null)
+				@LockManager.runWithLock @lockFile, @runner, @callback
+
+			it "should run the compile", ->
+				@runner
+					.calledWith()
+					.should.equal true
+
+			it "should call the callback with the response from the compile", ->
+				@callback
+					.calledWithExactly(null, "foo", "bar")
+					.should.equal true
+
+		describe "when the project is locked", ->
+			beforeEach ->
+				@error = new Error()
+				@error.code =  "EEXIST"
+				@Lockfile.lock = sinon.stub().callsArgWith(2,@error)
+				@Lockfile.unlock = sinon.stub().callsArgWith(1, null)
+				@LockManager.runWithLock @lockFile, @runner, @callback
+
+			it "should not run the compile", ->
+				@runner
+					.called
+					.should.equal false
+
+			it "should return an error", ->
+				error = new Errors.AlreadyCompilingError()
+				@callback
+					.calledWithExactly(error)
+					.should.equal true

test/unit/coffee/OutputFileOptimiserTests.coffee

@@ -0,0 +1,103 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/OutputFileOptimiser'
+path = require "path"
+expect = require("chai").expect
+EventEmitter = require("events").EventEmitter
+
+describe "OutputFileOptimiser", ->
+	beforeEach ->
+		@OutputFileOptimiser = SandboxedModule.require modulePath, requires:
+			"fs": @fs = {}
+			"path": @Path = {}
+			"child_process": spawn: @spawn = sinon.stub()
+			"logger-sharelatex": { log: sinon.stub(), warn: sinon.stub() }
+			"./Metrics" : {}
+		@directory = "/test/dir"
+		@callback = sinon.stub()
+
+	describe "optimiseFile", ->
+		beforeEach ->
+			@src = "./output.pdf"
+			@dst = "./output.pdf"
+
+		describe "when the file is not a pdf file", ->
+			beforeEach (done)->
+				@src = "./output.log"
+				@OutputFileOptimiser.checkIfPDFIsOptimised = sinon.stub().callsArgWith(1, null, false)
+				@OutputFileOptimiser.optimisePDF = sinon.stub().callsArgWith(2, null)
+				@OutputFileOptimiser.optimiseFile @src, @dst, done
+
+			it "should not check if the file is optimised", ->
+				@OutputFileOptimiser.checkIfPDFIsOptimised.calledWith(@src).should.equal false
+
+			it "should not optimise the file", ->
+				@OutputFileOptimiser.optimisePDF.calledWith(@src, @dst).should.equal false
+
+		describe "when the pdf file is not optimised", ->
+			beforeEach (done) ->
+				@OutputFileOptimiser.checkIfPDFIsOptimised = sinon.stub().callsArgWith(1, null, false)
+				@OutputFileOptimiser.optimisePDF = sinon.stub().callsArgWith(2, null)
+				@OutputFileOptimiser.optimiseFile @src, @dst, done
+
+			it "should check if the pdf is optimised", ->
+				@OutputFileOptimiser.checkIfPDFIsOptimised.calledWith(@src).should.equal true
+
+			it "should optimise the pdf", ->
+				@OutputFileOptimiser.optimisePDF.calledWith(@src, @dst).should.equal true
+
+		describe "when the pdf file is optimised", ->
+			beforeEach (done) ->
+				@OutputFileOptimiser.checkIfPDFIsOptimised = sinon.stub().callsArgWith(1, null, true)
+				@OutputFileOptimiser.optimisePDF = sinon.stub().callsArgWith(2, null)
+				@OutputFileOptimiser.optimiseFile @src, @dst, done
+
+			it "should check if the pdf is optimised", ->
+				@OutputFileOptimiser.checkIfPDFIsOptimised.calledWith(@src).should.equal true
+
+			it "should not optimise the pdf", ->
+				@OutputFileOptimiser.optimisePDF.calledWith(@src, @dst).should.equal false
+
+	describe "checkIfPDFISOptimised", ->
+		beforeEach () ->
+			@callback = sinon.stub()
+			@fd = 1234
+			@fs.open = sinon.stub().yields(null, @fd)
+			@fs.read = sinon.stub().withArgs(@fd).yields(null, 100, new Buffer("hello /Linearized 1"))
+			@fs.close = sinon.stub().withArgs(@fd).yields(null)
+			@OutputFileOptimiser.checkIfPDFIsOptimised @src, @callback
+
+		describe "for a linearised file", ->
+			beforeEach () ->
+				@fs.read = sinon.stub().withArgs(@fd).yields(null, 100, new Buffer("hello /Linearized 1"))
+				@OutputFileOptimiser.checkIfPDFIsOptimised @src, @callback
+
+			it "should open the file", ->
+				@fs.open.calledWith(@src, "r").should.equal true
+
+			it "should read the header", ->
+				@fs.read.calledWith(@fd).should.equal true
+
+			it "should close the file", ->
+				@fs.close.calledWith(@fd).should.equal true
+
+			it "should call the callback with a true result", ->
+				@callback.calledWith(null, true).should.equal true
+
+		describe "for an unlinearised file", ->
+			beforeEach () ->
+				@fs.read = sinon.stub().withArgs(@fd).yields(null, 100, new Buffer("hello not linearized 1"))
+				@OutputFileOptimiser.checkIfPDFIsOptimised @src, @callback
+
+			it "should open the file", ->
+				@fs.open.calledWith(@src, "r").should.equal true
+
+			it "should read the header", ->
+				@fs.read.calledWith(@fd).should.equal true
+
+			it "should close the file", ->
+				@fs.close.calledWith(@fd).should.equal true
+
+			it "should call the callback with a false result", ->
+				@callback.calledWith(null, false).should.equal true

test/unit/coffee/ProjectPersistenceManagerTests.coffee

@@ -13,6 +13,7 @@ describe "ProjectPersistenceManager", ->
 			"./db": @db = {}
 		@callback = sinon.stub()
 		@project_id = "project-id-123"
+		@user_id = "1234"
 
 	describe "clearExpiredProjects", ->
 		beforeEach ->
@@ -21,12 +22,13 @@ describe "ProjectPersistenceManager", ->
 				"project-id-2"
 			]
 			@ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids)
-			@ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1)
+			@ProjectPersistenceManager.clearProjectFromCache = sinon.stub().callsArg(1)
+			@CompileManager.clearExpiredProjects = sinon.stub().callsArg(1)
 			@ProjectPersistenceManager.clearExpiredProjects @callback
 
 		it "should clear each expired project", ->
 			for project_id in @project_ids
-				@ProjectPersistenceManager.clearProject
+				@ProjectPersistenceManager.clearProjectFromCache
 					.calledWith(project_id)
 					.should.equal true
 
@@ -37,8 +39,8 @@ describe "ProjectPersistenceManager", ->
 		beforeEach ->
 			@ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1)
 			@UrlCache.clearProject = sinon.stub().callsArg(1)
-			@CompileManager.clearProject = sinon.stub().callsArg(1)
-			@ProjectPersistenceManager.clearProject @project_id, @callback
+			@CompileManager.clearProject = sinon.stub().callsArg(2)
+			@ProjectPersistenceManager.clearProject @project_id, @user_id, @callback
 
 		it "should clear the project from the database", ->
 			@ProjectPersistenceManager._clearProjectFromDatabase
@@ -52,7 +54,7 @@ describe "ProjectPersistenceManager", ->
 
 		it "should clear the project compile folder", ->
 			@CompileManager.clearProject
-				.calledWith(@project_id)
+				.calledWith(@project_id, @user_id)
 				.should.equal true
 
 		it "should call the callback", ->

test/unit/coffee/RequestParserTests.coffee

@@ -1,6 +1,7 @@
 SandboxedModule = require('sandboxed-module')
 sinon = require('sinon')
 require('chai').should()
+expect = require('chai').expect
 modulePath = require('path').join __dirname, '../../../app/js/RequestParser'
 tk = require("timekeeper")
 
@@ -16,10 +17,12 @@ describe "RequestParser", ->
 			compile:
 				token: "token-123"
 				options:
+					imageName: "basicImageName/here:2017-1"
 					compiler: "pdflatex"
 					timeout:  42
 				resources: []
-		@RequestParser = SandboxedModule.require modulePath
+		@RequestParser = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @settings = {}
 
 	afterEach ->
 		tk.reset()
@@ -57,6 +60,28 @@ describe "RequestParser", ->
 		it "should set the compiler to pdflatex by default", ->
 			@data.compiler.should.equal "pdflatex"
 
+	describe "with imageName set", ->
+		beforeEach ->
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "should set the imageName", ->
+			@data.imageName.should.equal "basicImageName/here:2017-1"
+
+	describe "with flags set", ->
+		beforeEach ->
+			@validRequest.compile.options.flags = ["-file-line-error"]
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "should set the flags attribute", ->
+			expect(@data.flags).to.deep.equal ["-file-line-error"]
+
+	describe "with flags not specified", ->
+		beforeEach ->
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "it should have an empty flags list", ->
+			expect(@data.flags).to.deep.equal []
+
 	describe "without a timeout specified", ->
 		beforeEach ->
 			delete @validRequest.compile.options.timeout
@@ -206,11 +231,49 @@ describe "RequestParser", ->
 
 	describe "with a root resource path that needs escaping", ->
 		beforeEach ->
-			@validRequest.compile.rootResourcePath = "`rm -rf foo`.tex"
+			@badPath = "`rm -rf foo`.tex"
+			@goodPath = "rm -rf foo.tex"
+			@validRequest.compile.rootResourcePath = @badPath
+			@validRequest.compile.resources.push {
+				path: @badPath
+				date: "12:00 01/02/03"
+				content: "Hello world"
+			}
 			@RequestParser.parse @validRequest, @callback
 			@data = @callback.args[0][1]
 
 		it "should return the escaped resource", ->
-			@data.rootResourcePath.should.equal "rm -rf foo.tex"
+			@data.rootResourcePath.should.equal @goodPath
 
+		it "should also escape the resource path", ->
+			@data.resources[0].path.should.equal @goodPath
 
+	describe "with a root resource path that has a relative path", ->
+		beforeEach ->
+			@validRequest.compile.rootResourcePath = "foo/../../bar.tex"
+			@RequestParser.parse @validRequest, @callback
+			@data = @callback.args[0][1]
+
+		it "should return an error", ->
+			@callback.calledWith("relative path in root resource")
+				.should.equal true
+
+	describe "with a root resource path that has unescaped + relative path", ->
+		beforeEach ->
+			@validRequest.compile.rootResourcePath = "foo/#../bar.tex"
+			@RequestParser.parse @validRequest, @callback
+			@data = @callback.args[0][1]
+
+		it "should return an error", ->
+			@callback.calledWith("relative path in root resource")
+				.should.equal true
+
+	describe "with an unknown syncType", ->
+		beforeEach ->
+			@validRequest.compile.options.syncType = "unexpected"
+			@RequestParser.parse @validRequest, @callback
+			@data = @callback.args[0][1]
+
+		it "should return an error", ->
+			@callback.calledWith("syncType attribute should be one of: full, incremental")
+				.should.equal true

test/unit/coffee/ResourceStateManagerTests.coffee

@@ -0,0 +1,109 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+should = require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/ResourceStateManager'
+Path = require "path"
+Errors = require "../../../app/js/Errors"
+
+describe "ResourceStateManager", ->
+	beforeEach ->
+		@ResourceStateManager = SandboxedModule.require modulePath, requires:
+			"fs": @fs = {}
+			"logger-sharelatex": {log: sinon.stub(), err: sinon.stub()}
+			"./SafeReader": @SafeReader = {}
+		@basePath = "/path/to/write/files/to"
+		@resources = [
+			{path: "resource-1-mock"}
+			{path: "resource-2-mock"}
+			{path: "resource-3-mock"}
+		]
+		@state = "1234567890"
+		@resourceFileName = "#{@basePath}/.project-sync-state"
+		@resourceFileContents = "#{@resources[0].path}\n#{@resources[1].path}\n#{@resources[2].path}\nstateHash:#{@state}"
+		@callback = sinon.stub()
+
+	describe "saveProjectState", ->
+		beforeEach ->
+			@fs.writeFile = sinon.stub().callsArg(2)
+
+		describe "when the state is specified", ->
+			beforeEach ->
+				@ResourceStateManager.saveProjectState(@state, @resources, @basePath, @callback)
+
+			it "should write the resource list to disk", ->
+				@fs.writeFile
+					.calledWith(@resourceFileName, @resourceFileContents)
+					.should.equal true
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "when the state is undefined", ->
+			beforeEach ->
+				@state = undefined
+				@fs.unlink = sinon.stub().callsArg(1)
+				@ResourceStateManager.saveProjectState(@state, @resources, @basePath, @callback)
+
+			it "should unlink the resource file", ->
+				@fs.unlink
+					.calledWith(@resourceFileName)
+					.should.equal true
+
+			it "should not write the resource list to disk", ->
+				@fs.writeFile.called.should.equal false
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+	describe "checkProjectStateMatches", ->
+
+		describe "when the state matches", ->
+			beforeEach ->
+				@SafeReader.readFile = sinon.stub().callsArgWith(3, null, @resourceFileContents)
+				@ResourceStateManager.checkProjectStateMatches(@state, @basePath, @callback)
+
+			it "should read the resource file", ->
+				@SafeReader.readFile
+					.calledWith(@resourceFileName)
+					.should.equal true
+
+			it "should call the callback with the results", ->
+				@callback.calledWithMatch(null, @resources).should.equal true
+
+		describe "when the state does not match", ->
+			beforeEach ->
+				@SafeReader.readFile = sinon.stub().callsArgWith(3, null, @resourceFileContents)
+				@ResourceStateManager.checkProjectStateMatches("not-the-original-state", @basePath, @callback)
+
+			it "should call the callback with an error", ->
+				error = new Errors.FilesOutOfSyncError("invalid state for incremental update")
+				@callback.calledWith(error).should.equal true
+
+	describe "checkResourceFiles", ->
+		describe "when all the files are present", ->
+			beforeEach ->
+				@allFiles = [ @resources[0].path, @resources[1].path, @resources[2].path]
+				@ResourceStateManager.checkResourceFiles(@resources, @allFiles, @basePath, @callback)
+
+			it "should call the callback", ->
+				@callback.calledWithExactly().should.equal true
+
+		describe "when there is a missing file", ->
+			beforeEach ->
+				@allFiles = [ @resources[0].path, @resources[1].path]
+				@fs.stat = sinon.stub().callsArgWith(1, new Error())
+				@ResourceStateManager.checkResourceFiles(@resources, @allFiles, @basePath, @callback)
+
+			it "should call the callback with an error", ->
+				error = new Errors.FilesOutOfSyncError("resource files missing in incremental update")
+				@callback.calledWith(error).should.equal true
+
+		describe "when a resource contains a relative path", ->
+			beforeEach ->
+				@resources[0].path = "../foo/bar.tex"
+				@allFiles = [ @resources[0].path, @resources[1].path, @resources[2].path]
+				@ResourceStateManager.checkResourceFiles(@resources, @allFiles, @basePath, @callback)
+
+			it "should call the callback with an error", ->
+				@callback.calledWith(new Error("relative path in resource file list")).should.equal true
+