Release version 0.1.3

remove accidental inclusion of , and add null char \x00

Gruntfile.coffee

@@ -50,6 +50,7 @@ module.exports = (grunt) ->
 			unit:
 				options:
 					reporter: "spec"
+					grep: grunt.option("grep")
 				src: ["test/unit/js/**/*.js"]
 			acceptance:
 				options:

app.coffee

@@ -4,6 +4,9 @@ logger = require "logger-sharelatex"
 logger.initialize("clsi")
 smokeTest = require "smoke-test-sharelatex"
 
+Path = require "path"
+fs = require "fs"
+
 Metrics = require "metrics-sharelatex"
 Metrics.initialize("clsi")
 Metrics.open_sockets.monitor(logger)
@@ -21,7 +24,7 @@ app.use Metrics.http.monitor(logger)
 # Compile requests can take longer than the default two
 # minutes (including file download time), so bump up the 
 # timeout a bit.
-TIMEOUT = threeMinutes = 3 * 60 * 1000
+TIMEOUT = 6 * 60 * 1000
 app.use (req, res, next) ->
 	req.setTimeout TIMEOUT
 	res.setTimeout TIMEOUT
@@ -33,8 +36,21 @@ app.delete "/project/:project_id", CompileController.clearCache
 app.get  "/project/:project_id/sync/code", CompileController.syncFromCode
 app.get  "/project/:project_id/sync/pdf", CompileController.syncFromPdf
 
-staticServer = express.static(Settings.path.compilesDir)
-app.get "/project/:project_id/output/*", (req, res, next) ->
+staticServer = express.static Settings.path.compilesDir, setHeaders: (res, path, stat) ->
+	if Path.basename(path) == "output.pdf"
+		res.set("Content-Type", "application/pdf")
+		# Calculate an etag in the same way as nginx
+		# https://github.com/tj/send/issues/65
+		etag = (path, stat) ->
+			'"' + Math.ceil(+stat.mtime / 1000).toString(16) +
+			'-' + Number(stat.size).toString(16) + '"'
+		res.set("Etag", etag(path, stat))
+	else
+		# Force plain treatment of other file types to prevent hosting of HTTP/JS files
+		# that could be used in same-origin/XSS attacks.
+		res.set("Content-Type", "text/plain")
+
+app.get "/project/:project_id/output/*", require("./app/js/SymlinkCheckerMiddlewear"), (req, res, next) ->
 	req.url = "/#{req.params.project_id}/#{req.params[0]}"
 	staticServer(req, res, next)
 
@@ -62,7 +78,7 @@ app.get "/health_check", (req, res)->
 
 app.use (error, req, res, next) ->
 	logger.error err: error, "server error"
-	res.send 500
+	res.send error?.statusCode || 500
 
 app.listen port = (Settings.internal?.clsi?.port or 3013), host = (Settings.internal?.clsi?.host or "localhost"), (error) ->
 	logger.log "CLSI listening on #{host}:#{port}"

app/coffee/OutputFileFinder.coffee

@@ -1,7 +1,8 @@
 async = require "async"
 fs = require "fs"
 Path = require "path"
-wrench = require "wrench"
+spawn = require("child_process").spawn
+logger = require "logger-sharelatex"
 
 module.exports = OutputFileFinder =
 	findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
@@ -9,50 +10,48 @@ module.exports = OutputFileFinder =
 		for resource in resources
 			incomingResources[resource.path] = true
 			
-		OutputFileFinder._getAllFiles directory, (error, allFiles) ->
+		logger.log directory: directory, "getting output files"
+
+		OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
+			return callback(error) if error?
 			jobs = []
 			outputFiles = []
 			for file in allFiles
 				do (file) ->
 					jobs.push (callback) ->
-						if incomingResources[file.path]
+						if incomingResources[file]
 							return callback()
 						else
-							OutputFileFinder._isDirectory Path.join(directory, file.path), (error, directory) ->
-								return callback(error) if error?
-								if !directory
-									outputFiles.push file
+							outputFiles.push {
+								path: file
+								type: file.match(/\.([^\.]+)$/)?[1]
+							}
 							callback()
 
 			async.series jobs, (error) ->
 				return callback(error) if error?
 				callback null, outputFiles
 
-	_isDirectory: (path, callback = (error, directory) ->) ->
-		fs.stat path, (error, stat) ->
-			callback error, stat?.isDirectory()
-
-	_getAllFiles: (directory, _callback = (error, outputFiles) ->) ->
-		callback = (error, outputFiles) ->
-			_callback(error, outputFiles)
+	_getAllFiles: (directory, _callback = (error, fileList) ->) ->
+		callback = (error, fileList) ->
+			_callback(error, fileList)
 			_callback = () ->
 				
-		outputFiles = []
+		args = [directory, "-type", "f"]
+		logger.log args: args, "running find command"
 
-		wrench.readdirRecursive directory, (error, files) =>
-			if error?
-				if error.code == "ENOENT"
-					# Directory doesn't exist, which is not a problem
-					return callback(null, [])
-				else
-					return callback(error)
-
-			# readdirRecursive returns multiple times and finishes with a null response
-			if !files?
-				return callback(null, outputFiles)
-
-			for file in files
-				outputFiles.push
-					path: file
-					type: file.match(/\.([^\.]+)$/)?[1]
+		proc = spawn("find", args)
+		stdout = ""
+		proc.stdout.on "data", (chunk) ->
+			stdout += chunk.toString()	
+		proc.on "error", callback	
+		proc.on "close", (code) ->
+			if code != 0
+				logger.warn {directory, code}, "find returned error, directory likely doesn't exist"
+				return callback null, []
+			fileList = stdout.trim().split("\n")
+			fileList = fileList.map (file) ->
+				# Strip leading directory
+				path = Path.relative(directory, file)
+			return callback null, fileList
 

app/coffee/RequestParser.coffee

@@ -1,6 +1,6 @@
 module.exports = RequestParser =
 	VALID_COMPILERS: ["pdflatex", "latex", "xelatex", "lualatex"]
-	MAX_TIMEOUT: 60
+	MAX_TIMEOUT: 300
 
 	parse: (body, callback = (error, data) ->) ->
 		response = {}
@@ -27,10 +27,12 @@ module.exports = RequestParser =
 			response.timeout = response.timeout * 1000 # milliseconds
 
 			response.resources = (@_parseResource(resource) for resource in (compile.resources or []))
-			response.rootResourcePath = @_parseAttribute "rootResourcePath",
+
+			rootResourcePath = @_parseAttribute "rootResourcePath",
 				compile.rootResourcePath
 				default: "main.tex"
 				type: "string"
+			response.rootResourcePath = RequestParser._sanitizePath(rootResourcePath)
 		catch error
 			return callback error
 
@@ -72,3 +74,6 @@ module.exports = RequestParser =
 			throw "Default not implemented"
 		return attribute
 
+	_sanitizePath: (path) ->
+		# See http://php.net/manual/en/function.escapeshellcmd.php
+		path.replace(/[\#\&\;\`\|\*\?\~\<\>\^\(\)\[\]\{\}\$\\\x0A\xFF\x00]/g, "")

app/coffee/SymlinkCheckerMiddlewear.coffee

@@ -0,0 +1,17 @@
+Path = require("path")
+fs = require("fs")
+Settings = require("settings-sharelatex")
+logger = require("logger-sharelatex")
+
+
+module.exports = (req, res, next)->
+	basePath = Path.resolve("#{Settings.path.compilesDir}/#{req.params.project_id}")
+	requestedFsPath = Path.normalize("#{basePath}/#{req.params[0]}")
+	fs.realpath requestedFsPath, (err, realFsPath)->
+		if err?
+			return res.send(500)
+		else if requestedFsPath != realFsPath
+			logger.warn requestedFsPath:requestedFsPath, realFsPath:realFsPath, path: req.params[0], project_id: req.params.project_id, "trying to access a different file (symlink), aborting"
+			return res.send(404)
+		else
+			return next()

package.json

@@ -1,7 +1,7 @@
 {
   "name": "node-clsi",
   "description": "A Node.js implementation of the CLSI LaTeX web-API",
-  "version": "0.1.0",
+  "version": "0.1.3",
   "repository": {
     "type": "git",
     "url": "https://github.com/sharelatex/clsi-sharelatex.git"
@@ -16,7 +16,7 @@
     "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
     "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.0.0",
-    "sequelize": "~2.0.0-beta.2",
+    "sequelize": "2.0.0-beta.2",
     "wrench": "~1.5.4",
     "smoke-test-sharelatex": "git+https://github.com/sharelatex/smoke-test-sharelatex.git#v1.0.0",
     "sqlite3": "~2.2.0",

test/acceptance/fixtures/examples/markdown-included/chapters/chapter1.md

@@ -1,17 +0,0 @@
-Section Title
--------------
-
-* List item one
-* List item two
-
-: Sample grid table.
-
-+---------------+---------------+--------------------+
-| Fruit         | Price         | Advantages         |
-+===============+===============+====================+
-| Bananas       | $1.34         | - built-in wrapper |
-|               |               | - bright color     |
-+---------------+---------------+--------------------+
-| Oranges       | $2.10         | - cures scurvy     |
-|               |               | - tasty            |
-+---------------+---------------+--------------------+

test/acceptance/fixtures/examples/markdown-included/main.tex

@@ -1,9 +0,0 @@
-\documentclass{article}
-\usepackage{longtable}
-\usepackage{booktabs, multicol, multirow}
-
-\begin{document}
-
-\input{chapters/chapter1}
-
-\end{document}

test/acceptance/fixtures/examples/markdown-standalone/main.md

@@ -1,23 +0,0 @@
-% Title
-% Author
-% Date
-
-Chapter title
-=============
-
-Section Title
--------------
-
-Hello world. Have a nice table:
-
-: Sample grid table.
-
-+---------------+---------------+--------------------+
-| Fruit         | Price         | Advantages         |
-+===============+===============+====================+
-| Bananas       | $1.34         | - built-in wrapper |
-|               |               | - bright color     |
-+---------------+---------------+--------------------+
-| Oranges       | $2.10         | - cures scurvy     |
-|               |               | - tasty            |
-+---------------+---------------+--------------------+

test/unit/coffee/OutputFileFinderTests.coffee

@@ -4,29 +4,26 @@ require('chai').should()
 modulePath = require('path').join __dirname, '../../../app/js/OutputFileFinder'
 path = require "path"
 expect = require("chai").expect
+EventEmitter = require("events").EventEmitter
 
 describe "OutputFileFinder", ->
 	beforeEach ->
 		@OutputFileFinder = SandboxedModule.require modulePath, requires:
 			"fs": @fs = {}
-			"wrench": @wrench = {}
+			"child_process": spawn: @spawn = sinon.stub()
+			"logger-sharelatex": { log: sinon.stub(), warn: sinon.stub() }
 		@directory = "/test/dir"
 		@callback = sinon.stub()
 
 	describe "findOutputFiles", ->
 		beforeEach ->
 			@resource_path = "resource/path.tex"
-			@output_paths   = ["output.pdf", "extra", "extra/file.tex"]
+			@output_paths   = ["output.pdf", "extra/file.tex"]
+			@all_paths = @output_paths.concat [@resource_path]
 			@resources = [
 				path: @resource_path = "resource/path.tex"
 			]
-			@OutputFileFinder._isDirectory = (dirPath, callback = (error, directory) ->) =>
-				callback null, dirPath == path.join(@directory, "extra")
-
-			@wrench.readdirRecursive = (dir, callback) =>
-				callback(null, [@resource_path].concat(@output_paths))
-				callback(null, null)
-			sinon.spy @wrench, "readdirRecursive"
+			@OutputFileFinder._getAllFiles = sinon.stub().callsArgWith(1, null, @all_paths)
 			@OutputFileFinder.findOutputFiles @resources, @directory, (error, @outputFiles) =>
 
 		it "should only return the output files, not directories or resource paths", ->
@@ -38,4 +35,34 @@ describe "OutputFileFinder", ->
 				type: "tex"
 			}]
 			
+	describe "_getAllFiles", ->
+		beforeEach ->
+			@proc = new EventEmitter()
+			@proc.stdout = new EventEmitter()
+			@spawn.returns @proc
+			@directory = "/base/dir"
+			@OutputFileFinder._getAllFiles @directory, @callback
 			
+		describe "successfully", ->
+			beforeEach ->
+				@proc.stdout.emit(
+					"data",
+					["/base/dir/main.tex", "/base/dir/chapters/chapter1.tex"].join("\n") + "\n"
+				)
+				@proc.emit "close", 0
+				
+			it "should call the callback with the relative file paths", ->
+				@callback.calledWith(
+					null,
+					["main.tex", "chapters/chapter1.tex"]
+				).should.equal true
+
+		describe "when the directory doesn't exist", ->
+			beforeEach ->
+				@proc.emit "close", 1
+				
+			it "should call the callback with a blank array", ->
+				@callback.calledWith(
+					null,
+					[]
+				).should.equal true

test/unit/coffee/RequestParserTests.coffee

@@ -204,6 +204,13 @@ describe "RequestParser", ->
 			@callback.calledWith("rootResourcePath attribute should be a string")
 				.should.equal true
 
-
+	describe "with a root resource path that needs escaping", ->
+		beforeEach ->
+			@validRequest.compile.rootResourcePath = "`rm -rf foo`.tex"
+			@RequestParser.parse @validRequest, @callback
+			@data = @callback.args[0][1]
+			
+		it "should return the escaped resource", ->
+			@data.rootResourcePath.should.equal "rm -rf foo.tex"
 		
 

test/unit/coffee/SymlinkCheckerMiddlewearTests.coffee

@@ -0,0 +1,60 @@
+should = require('chai').should()
+SandboxedModule = require('sandboxed-module')
+assert = require('assert')
+path = require('path')
+sinon = require('sinon')
+modulePath = path.join __dirname, "../../../app/js/SymlinkCheckerMiddlewear"
+expect = require("chai").expect
+
+describe "SymlinkCheckerMiddlewear", ->
+
+	beforeEach ->
+
+		@settings = 
+			path:
+				compilesDir: "/compiles/here"
+
+		@fs = {}
+		@SymlinkCheckerMiddlewear = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex":@settings
+			"logger-sharelatex": 
+				log:->
+				warn:->
+			"fs":@fs
+		@req = 
+			params:
+				project_id:"12345"
+
+		@res = {}
+		@req.params[0]= "output.pdf"
+
+
+	describe "sending a normal file through", ->
+		beforeEach ->
+			@fs.realpath = sinon.stub().callsArgWith(1, null, "#{@settings.path.compilesDir}/#{@req.params.project_id}/output.pdf")
+
+		it "should call next", (done)->
+			@SymlinkCheckerMiddlewear @req, @res, done
+
+
+	describe "with a symlink file", ->
+		beforeEach ->
+			@fs.realpath = sinon.stub().callsArgWith(1, null, "/etc/#{@req.params.project_id}/output.pdf")
+
+		it "should send a 404", (done)->
+			@res.send = (resCode)->
+				resCode.should.equal 404
+				done()
+			@SymlinkCheckerMiddlewear @req, @res
+
+	describe "with an error from fs.realpath", ->
+
+		beforeEach ->
+			@fs.realpath = sinon.stub().callsArgWith(1, "error")
+
+		it "should send a 500", (done)->
+			@res.send = (resCode)->
+				resCode.should.equal 500
+				done()
+			@SymlinkCheckerMiddlewear @req, @res
+