Initial open source commit

app/coffee/CommandRunner.coffee

@@ -0,0 +1,12 @@
+spawn = require("child_process").spawn
+logger = require "logger-sharelatex"
+
+module.exports = CommandRunner =
+	run: (project_id, command, directory, timeout, callback = (error) ->) ->
+		command = (arg.replace('$COMPILE_DIR', directory) for arg in command)
+		logger.log project_id: project_id, command: command, directory: directory, "running command"
+		logger.warn "timeouts and sandboxing are not enabled with CommandRunner"
+
+		proc = spawn command[0], command.slice(1), stdio: "inherit", cwd: directory
+		proc.on "close", () ->
+			callback()

app/coffee/CompileController.coffee

@@ -0,0 +1,40 @@
+RequestParser = require "./RequestParser"
+CompileManager = require "./CompileManager"
+Settings = require "settings-sharelatex"
+Metrics = require "./Metrics"
+ProjectPersistenceManager = require "./ProjectPersistenceManager"
+logger = require "logger-sharelatex"
+
+module.exports = CompileController =
+	compile: (req, res, next = (error) ->) ->
+		timer = new Metrics.Timer("compile-request")
+		RequestParser.parse req.body, (error, request) ->
+			return next(error) if error?
+			request.project_id = req.params.project_id
+			ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) ->
+				return next(error) if error?
+				CompileManager.doCompile request, (error, outputFiles = []) ->
+					if error?
+						logger.error err: error, project_id: request.project_id, "error running compile"
+						error = error.message or error
+						status = "failure"
+					else
+						status = "failure"
+						for file in outputFiles
+							if file.type == "pdf"
+								status = "success"
+
+					timer.done()
+					res.send JSON.stringify {
+						compile:
+							status: status
+							error:  error
+							outputFiles: outputFiles.map (file) ->
+								url: "#{Settings.apis.clsi.url}/project/#{request.project_id}/output/#{file.path}"
+								type: file.type
+					}
+		
+	clearCache: (req, res, next = (error) ->) ->
+		ProjectPersistenceManager.clearProject req.params.project_id, (error) ->
+			return next(error) if error?
+			res.send 204 # No content

app/coffee/CompileManager.coffee

@@ -0,0 +1,39 @@
+ResourceWriter = require "./ResourceWriter"
+LatexRunner = require "./LatexRunner"
+OutputFileFinder = require "./OutputFileFinder"
+Settings = require("settings-sharelatex")
+Path = require "path"
+logger = require "logger-sharelatex"
+Metrics = require "./Metrics"
+rimraf = require "rimraf"
+
+module.exports = CompileManager =
+	doCompile: (request, callback = (error, outputFiles) ->) ->
+		compileDir = Path.join(Settings.path.compilesDir, request.project_id)
+
+		timer = new Metrics.Timer("write-to-disk")
+		logger.log project_id: request.project_id, "starting compile"
+		ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) ->
+			return callback(error) if error?
+			logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "written files to disk"
+			timer.done()
+
+			timer = new Metrics.Timer("run-compile")
+			Metrics.inc("compiles")
+			LatexRunner.runLatex request.project_id, {
+				directory: compileDir
+				mainFile:  request.rootResourcePath
+				compiler:  request.compiler
+				timeout:   request.timeout
+			}, (error) ->
+				return callback(error) if error?
+				logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "done compile"
+				timer.done()
+
+				OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) ->
+					return callback(error) if error?
+					callback null, outputFiles
+	
+	clearProject: (project_id, callback = (error) ->) ->
+		compileDir = Path.join(Settings.compileDir, project_id)
+		rimraf compileDir, callback

app/coffee/LatexRunner.coffee

@@ -0,0 +1,57 @@
+Path = require "path"
+Settings = require "settings-sharelatex"
+logger = require "logger-sharelatex"
+Metrics = require "./Metrics"
+CommandRunner = require(Settings.clsi?.commandRunner or "./CommandRunner")
+
+module.exports = LatexRunner =
+	runLatex: (project_id, options, callback = (error) ->) ->
+		{directory, mainFile, compiler, timeout} = options
+		compiler ||= "pdflatex"
+		timeout  ||= 60000 # milliseconds
+
+		logger.log directory: directory, compiler: compiler, timeout: timeout, mainFile: mainFile, "starting compile"
+
+		# We want to run latexmk on the tex file which we it will automatically
+		# generate from the Rtex file.
+		mainFile = mainFile.replace(/\.Rtex$/, ".tex")
+
+		if compiler == "pdflatex"
+			command = LatexRunner._pdflatexCommand mainFile
+		else if compiler == "latex"
+			command = LatexRunner._latexCommand mainFile
+		else if compiler == "xelatex"
+			command = LatexRunner._xelatexCommand mainFile
+		else if compiler == "lualatex"
+			command = LatexRunner._lualatexCommand mainFile
+		else
+			return callback new Error("unknown compiler: #{compiler}")
+
+		CommandRunner.run project_id, command, directory, timeout, callback
+
+	_latexmkBaseCommand: [ "latexmk", "-cd", "-f", "-jobname=output", "-auxdir=$COMPILE_DIR", "-outdir=$COMPILE_DIR"]
+
+	_pdflatexCommand: (mainFile) ->
+		LatexRunner._latexmkBaseCommand.concat [
+			"-pdf", "-e", "$pdflatex='pdflatex -interaction=batchmode %O %S'",
+			Path.join("$COMPILE_DIR", mainFile)
+		]
+		
+	_latexCommand: (mainFile) ->
+		LatexRunner._latexmkBaseCommand.concat [
+			"-pdfdvi", "-e", "$latex='latex -interaction=batchmode %O %S'",
+			Path.join("$COMPILE_DIR", mainFile)
+		]
+
+	_xelatexCommand: (mainFile) ->
+		LatexRunner._latexmkBaseCommand.concat [
+			"-xelatex", "-e", "$pdflatex='xelatex -interaction=batchmode %O %S'",
+			Path.join("$COMPILE_DIR", mainFile)
+		]
+
+	_lualatexCommand: (mainFile) ->
+		LatexRunner._latexmkBaseCommand.concat [
+			"-pdf", "-e", "$pdflatex='lualatex -interaction=batchmode %O %S'",
+			Path.join("$COMPILE_DIR", mainFile)
+		]
+

app/coffee/Metrics.coffee

@@ -0,0 +1,23 @@
+StatsD = require('lynx')
+statsd = new StatsD('localhost', 8125, {on_error:->})
+
+buildKey = (key)-> "clsi.#{process.env.NODE_ENV or "testing"}.#{key}"
+
+module.exports =
+	set : (key, value, sampleRate = 1)->
+		statsd.set buildKey(key), value, sampleRate
+
+	inc : (key, sampleRate = 1)->
+		statsd.increment buildKey(key), sampleRate
+
+	Timer : class
+		constructor :(key, sampleRate = 1)->
+			this.start = new Date()
+			this.key = buildKey(key)
+		done:->
+			timeSpan = new Date - this.start
+			statsd.timing(this.key, timeSpan, this.sampleRate)
+
+	gauge : (key, value, sampleRate = 1)->
+		statsd.gauge key, value, sampleRate
+

app/coffee/OutputFileFinder.coffee

@@ -0,0 +1,58 @@
+async = require "async"
+fs = require "fs"
+Path = require "path"
+wrench = require "wrench"
+
+module.exports = OutputFileFinder =
+	findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
+		incomingResources = {}
+		for resource in resources
+			incomingResources[resource.path] = true
+
+		OutputFileFinder._getAllFiles directory, (error, allFiles) ->
+			jobs = []
+			outputFiles = []
+			for file in allFiles
+				do (file) ->
+					jobs.push (callback) ->
+						if incomingResources[file.path]
+							return callback()
+						else
+							OutputFileFinder._isDirectory Path.join(directory, file.path), (error, directory) ->
+								return callback(error) if error?
+								if !directory
+									outputFiles.push file
+								callback()
+
+			async.series jobs, (error) ->
+				return callback(error) if error?
+				callback null, outputFiles
+
+	_isDirectory: (path, callback = (error, directory) ->) ->
+		fs.stat path, (error, stat) ->
+			callback error, stat?.isDirectory()
+
+	_getAllFiles: (directory, _callback = (error, outputFiles) ->) ->
+		callback = (error, outputFiles) ->
+			_callback(error, outputFiles)
+			_callback = () ->
+
+		outputFiles = []
+
+		wrench.readdirRecursive directory, (error, files) =>
+			if error?
+				if error.code == "ENOENT"
+					# Directory doesn't exist, which is not a problem
+					return callback(null, [])
+				else
+					return callback(error)
+
+			# readdirRecursive returns multiple times and finishes with a null response
+			if !files?
+				return callback(null, outputFiles)
+
+			for file in files
+				outputFiles.push
+					path: file
+					type: file.match(/\.([^\.]+)$/)?[1]
+

app/coffee/ProjectPersistenceManager.coffee

@@ -0,0 +1,54 @@
+UrlCache = require "./UrlCache"
+CompileManager = require "./CompileManager"
+db = require "./db"
+async = require "async"
+logger = require "logger-sharelatex"
+
+module.exports = ProjectPersistenceManager =
+	EXPIRY_TIMEOUT: oneDay = 24 * 60 * 60 * 1000 #ms
+
+	markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
+		db.Project.findOrCreate(project_id: project_id)
+			.success(
+				(project) ->
+					project.updateAttributes(lastAccessed: new Date())
+						.success(() -> callback())
+						.error callback
+			)
+			.error callback
+
+	clearExpiredProjects: (callback = (error) ->) ->
+		ProjectPersistenceManager._findExpiredProjectIds (error, project_ids) ->
+			return callback(error) if error?
+			logger.log project_ids: project_ids, "clearing expired projects"
+			jobs = for project_id in (project_ids or [])
+				do (project_id) ->
+					(callback) ->
+						ProjectPersistenceManager.clearProject project_id, (err) ->
+							if err?
+								logger.error err: err, project_id: project_id, "error clearing project"
+							callback()
+			async.series jobs, callback
+
+	clearProject: (project_id, callback = (error) ->) ->
+		logger.log project_id: project_id, "clearing project"
+		CompileManager.clearProject project_id, (error) ->
+			return callback(error) if error?
+			UrlCache.clearProject project_id, (error) ->
+				return callback(error) if error?
+				ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
+					return callback(error) if error?
+					callback()
+
+	_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
+		db.Project.destroy(project_id: project_id)
+			.success(() -> callback())
+			.error callback
+
+	_findExpiredProjectIds: (callback = (error, project_ids) ->) ->
+		db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)])
+			.success(
+				(projects) ->
+					callback null, projects.map((project) -> project.project_id)
+			)
+			.error callback

app/coffee/RequestParser.coffee

@@ -0,0 +1,74 @@
+module.exports = RequestParser =
+	VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
+	MAX_TIMEOUT: 60
+
+	parse: (body, callback = (error, data) ->) ->
+		response = {}
+
+		if !body.compile?
+			return callback "top level object should have a compile attribute"
+
+		compile = body.compile
+		compile.options ||= {}
+		
+		try
+			response.compiler = @_parseAttribute "compiler",
+				compile.options.compiler,
+				validValues: @VALID_COMPILERS
+				default: "pdflatex"
+				type: "string"
+			response.timeout = @_parseAttribute "timeout",
+				compile.options.timeout
+				default: RequestParser.MAX_TIMEOUT
+				type: "number"
+
+			if response.timeout > RequestParser.MAX_TIMEOUT
+				response.timeout = RequestParser.MAX_TIMEOUT
+			response.timeout = response.timeout * 1000 # milliseconds
+
+			response.resources = (@_parseResource(resource) for resource in (compile.resources or []))
+			response.rootResourcePath = @_parseAttribute "rootResourcePath",
+				compile.rootResourcePath
+				default: "main.tex"
+				type: "string"
+		catch error
+			return callback error
+
+		callback null, response
+
+	_parseResource: (resource) ->
+		if !resource.path? or typeof resource.path != "string"
+			throw "all resources should have a path attribute"
+
+		if resource.modified?
+			modified = new Date(resource.modified)
+			if isNaN(modified.getTime())
+				throw "resource modified date could not be understood: #{resource.modified}"
+
+		if !resource.url? and !resource.content?
+		   	throw "all resources should have either a url or content attribute"
+		if resource.content? and typeof resource.content != "string"
+			throw "content attribute should be a string"
+		if resource.url? and typeof resource.url != "string"
+			throw "url attribute should be a string"
+
+		return {
+			path: resource.path
+			modified: modified
+			url: resource.url
+			content: resource.content
+		}
+
+	_parseAttribute: (name, attribute, options) ->
+		if attribute?
+			if options.validValues?
+				if options.validValues.indexOf(attribute) == -1
+					throw "#{name} attribute should be one of: #{options.validValues.join(", ")}"
+			if options.type?
+				if typeof attribute != options.type
+					throw "#{name} attribute should be a #{options.type}"
+		else
+			return options.default if options.default?
+			throw "Default not implemented"
+		return attribute
+

app/coffee/ResourceWriter.coffee

@@ -0,0 +1,68 @@
+UrlCache = require "./UrlCache"
+Path = require "path"
+fs = require "fs"
+async = require "async"
+mkdirp = require "mkdirp"
+OutputFileFinder = require "./OutputFileFinder"
+Metrics = require "./Metrics"
+
+module.exports = ResourceWriter =
+	syncResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
+		@_removeExtraneousFiles resources, basePath, (error) =>
+			return callback(error) if error?
+			jobs = for resource in resources
+				do (resource) =>
+					(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
+			async.series jobs, callback
+
+	_removeExtraneousFiles: (resources, basePath, _callback = (error) ->) ->
+		timer = new Metrics.Timer("unlink-output-files")
+		callback = (error) ->
+			timer.done()
+			_callback(error)
+
+		OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles) ->
+			return callback(error) if error?
+
+			jobs = []
+			for file in outputFiles or []
+				do (file) ->
+					path = file.path
+					should_delete = true
+					if path.match(/^output\./) or path.match(/\.aux$/)
+						should_delete = false
+					if path == "output.pdf" or path == "output.dvi" or path == "output.log"
+						should_delete = true
+					if should_delete
+						jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
+
+			async.series jobs, callback
+
+	_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
+		fs.stat path, (error, stat) ->
+			return callback(error) if error?
+			if stat.isFile()
+				fs.unlink path, callback
+			else
+				callback()
+
+	_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
+		path = Path.normalize(Path.join(basePath, resource.path))
+		if (path.slice(0, basePath.length) != basePath)
+			return callback new Error("resource path is outside root directory")
+
+		mkdirp Path.dirname(path), (error) ->
+			return callback(error) if error?
+			# TODO: Don't overwrite file if it hasn't been modified
+			if resource.url?
+				UrlCache.downloadUrlToFile(
+					project_id,
+					resource.url,
+					path,
+					resource.modified,
+					callback
+				)
+			else
+				fs.writeFile path, resource.content, callback
+
+		

app/coffee/UrlCache.coffee

@@ -0,0 +1,113 @@
+db = require("./db")
+UrlFetcher = require("./UrlFetcher")
+Settings = require("settings-sharelatex")
+crypto = require("crypto")
+fs = require("fs")
+logger = require "logger-sharelatex"
+async = require "async"
+
+module.exports = UrlCache =
+	downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
+		UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
+			return callback(error) if error?
+			UrlCache._copyFile(pathToCachedUrl, destPath, callback)
+
+	clearProject: (project_id, callback = (error) ->) ->
+		UrlCache._findAllUrlsInProject project_id, (error, urls) ->
+			logger.log project_id: project_id, url_count: urls.length, "clearing project URLs"
+			return callback(error) if error?
+			jobs = for url in (urls or [])
+				do (url) ->
+					(callback) ->
+						UrlCache._clearUrlFromCache project_id, url, (error) ->
+							if error?
+								logger.error err: error, project_id: project_id, url: url, "error clearing project URL"
+							callback()
+			async.series jobs, callback
+
+	_ensureUrlIsInCache: (project_id, url, lastModified, callback = (error, pathOnDisk) ->) ->
+		if lastModified?
+			# MYSQL only stores dates to an accuracy of a second but the incoming lastModified might have milliseconds.
+			# So round down to seconds
+			lastModified = new Date(Math.floor(lastModified.getTime() / 1000) * 1000)
+		UrlCache._doesUrlNeedDownloading project_id, url, lastModified, (error, needsDownloading) =>
+			return callback(error) if error?
+			if needsDownloading
+				logger.log url: url, lastModified: lastModified, "downloading URL"
+				UrlFetcher.pipeUrlToFile url, UrlCache._cacheFilePathForUrl(project_id, url), (error) =>
+					return callback(error) if error?
+					UrlCache._updateOrCreateUrlDetails project_id, url, lastModified, (error) =>
+						return callback(error) if error?
+						callback null, UrlCache._cacheFilePathForUrl(project_id, url)
+			else
+				logger.log url: url, lastModified: lastModified, "URL is up to date in cache"
+				callback null, UrlCache._cacheFilePathForUrl(project_id, url)
+	
+	_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
+		if !lastModified?
+			return callback null, true
+
+		UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
+			return callback(error) if error?
+			if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
+				return callback null, true
+			else
+				return callback null, false
+
+	_cacheFileNameForUrl: (project_id, url) ->
+		project_id + ":" + crypto.createHash("md5").update(url).digest("hex")
+
+	_cacheFilePathForUrl: (project_id, url) ->
+		"#{Settings.path.clsiCacheDir}/#{UrlCache._cacheFileNameForUrl(project_id, url)}"
+
+	_copyFile: (from, to, _callback = (error) ->) ->
+		callbackOnce = (error) ->
+			_callback(error)
+			_callback = () ->
+		writeStream = fs.createWriteStream(to)
+		readStream = fs.createReadStream(from)
+		writeStream.on "error", callbackOnce
+		readStream.on "error", callbackOnce
+		writeStream.on "close", () -> callbackOnce()
+		readStream.pipe(writeStream)
+
+	_clearUrlFromCache: (project_id, url, callback = (error) ->) ->
+		UrlCache._clearUrlDetails project_id, url, (error) ->
+			return callback(error) if error?
+			UrlCache._deleteUrlCacheFromDisk project_id, url, (error) ->
+				return callback(error) if error?
+				callback null
+
+	_deleteUrlCacheFromDisk: (project_id, url, callback = (error) ->) ->
+		fs.unlink UrlCache._cacheFilePathForUrl(project_id, url), callback
+
+	_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
+		db.UrlCache.find(where: { url: url, project_id: project_id })
+			.success((urlDetails) -> callback null, urlDetails)
+			.error callback
+
+	_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
+		db.UrlCache.findOrCreate(url: url, project_id: project_id)
+			.success(
+				(urlDetails) ->
+					urlDetails.updateAttributes(lastModified: lastModified)
+						.success(() -> callback())
+						.error(callback)
+			)
+			.error callback
+
+	_clearUrlDetails: (project_id, url, callback = (error) ->) ->
+		db.UrlCache.destroy(url: url, project_id: project_id)
+			.success(() -> callback null)
+			.error callback
+
+	_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
+		db.UrlCache.findAll(where: { project_id: project_id })
+			.success(
+				(urlEntries) ->
+					callback null, urlEntries.map((entry) -> entry.url)
+			)
+			.error callback
+
+		
+		

app/coffee/UrlFetcher.coffee

@@ -0,0 +1,23 @@
+request = require("request").defaults(jar: false)
+fs = require("fs")
+
+module.exports = UrlFetcher =
+	pipeUrlToFile: (url, filePath, _callback = (error) ->) ->
+		callbackOnce = (error) ->
+			_callback(error)
+			_callback = () ->
+
+		urlStream = request.get(url)
+		fileStream = fs.createWriteStream(filePath)
+
+		urlStream.on "response", (res) ->
+			if res.statusCode >= 200 and res.statusCode < 300
+				urlStream.pipe(fileStream)
+			else
+				callbackOnce(new Error("URL returned non-success status code: #{res.statusCode}"))
+
+		urlStream.on "error", (error) ->
+			callbackOnce(error or new Error("Something went wrong downloading the URL"))
+
+		urlStream.on "end", () ->
+			callbackOnce()

app/coffee/db.coffee

@@ -0,0 +1,24 @@
+Sequelize = require("sequelize")
+Settings = require("settings-sharelatex")
+
+sequelize = new Sequelize(
+	Settings.mysql.clsi.database,
+	Settings.mysql.clsi.username,
+	Settings.mysql.clsi.password,
+	Settings.mysql.clsi
+)
+
+module.exports =
+	UrlCache: sequelize.define("UrlCache", {
+		url: Sequelize.STRING
+		project_id: Sequelize.STRING
+		lastModified: Sequelize.DATE
+	})
+
+	Project: sequelize.define("Project", {
+		project_id: Sequelize.STRING
+		lastAccessed: Sequelize.DATE
+	})
+
+	sync: () -> sequelize.sync()
+