Initial open source commit

test/unit/coffee/CompileControllerTests.coffee

@@ -0,0 +1,92 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/CompileController'
+tk = require("timekeeper")
+
+describe "CompileController", ->
+	beforeEach ->
+		@CompileController = SandboxedModule.require modulePath, requires:
+			"./CompileManager": @CompileManager = {}
+			"./RequestParser": @RequestParser = {}
+			"settings-sharelatex": @Settings =
+				apis:
+					clsi:
+						url: "http://clsi.example.com"
+			"./ProjectPersistenceManager": @ProjectPersistenceManager = {}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+		@Settings.externalUrl = "http://www.example.com"
+		@req = {}
+		@res = {}
+
+	describe "compile", ->
+		beforeEach ->
+			@req.body = {
+				compile: "mock-body"
+			}
+			@req.params =
+				project_id: @project_id = "project-id-123"
+			@request = {
+				compile: "mock-parsed-request"
+			}
+			@request_with_project_id =
+				compile: @request.compile
+				project_id: @project_id
+			@output_files = [{
+				path: "output.pdf"
+				type: "pdf"
+			}, {
+				path: "output.log"
+				type: "log"
+			}]
+			@RequestParser.parse = sinon.stub().callsArgWith(1, null, @request)
+			@ProjectPersistenceManager.markProjectAsJustAccessed = sinon.stub().callsArg(1)
+			@res.send = sinon.stub()
+
+		describe "successfully", ->
+			beforeEach ->
+				@CompileManager.doCompile = sinon.stub().callsArgWith(1, null, @output_files)
+				@CompileController.compile @req, @res
+
+			it "should parse the request", ->
+				@RequestParser.parse
+					.calledWith(@req.body)
+					.should.equal true
+
+			it "should run the compile for the specified project", ->
+				@CompileManager.doCompile
+					.calledWith(@request_with_project_id)
+					.should.equal true
+
+			it "should mark the project as accessed", ->
+				@ProjectPersistenceManager.markProjectAsJustAccessed
+					.calledWith(@project_id)
+					.should.equal true
+
+			it "should return the JSON response", ->
+				@res.send
+					.calledWith(JSON.stringify
+						compile:
+							status: "success"
+							error: null
+							outputFiles: @output_files.map (file) =>
+								url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/output/#{file.path}"
+								type: file.type
+					)
+					.should.equal true
+			
+		describe "with an error", ->
+			beforeEach ->
+				@CompileManager.doCompile = sinon.stub().callsArgWith(1, new Error(@message = "error message"), null)
+				@CompileController.compile @req, @res
+		
+			it "should return the JSON response with the error", ->
+				@res.send
+					.calledWith(JSON.stringify
+						compile:
+							status: "failure"
+							error:  @message
+							outputFiles: []
+					)
+					.should.equal true
+

test/unit/coffee/CompileManagerTests.coffee

@@ -0,0 +1,73 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/CompileManager'
+tk = require("timekeeper")
+
+describe "CompileManager", ->
+	beforeEach ->
+		@CompileManager = SandboxedModule.require modulePath, requires:
+			"./LatexRunner": @LatexRunner = {}
+			"./ResourceWriter": @ResourceWriter = {}
+			"./OutputFileFinder": @OutputFileFinder = {}
+			"settings-sharelatex": @Settings = { path: compilesDir: "/compiles/dir" }
+			"logger-sharelatex": @logger = { log: sinon.stub() }
+			"rimraf": @rimraf = sinon.stub().callsArg(1)
+		@callback = sinon.stub()
+
+	describe "doCompile", ->
+		beforeEach ->
+			@output_files = [{
+				path: "output.log"
+				type: "log"
+			}, {
+				path: "output.pdf"
+				type: "pdf"
+			}]
+			@request =
+				resources: @resources = "mock-resources"
+				rootResourcePath: @rootResourcePath = "main.tex"
+				project_id: @project_id = "project-id-123"
+				compiler: @compiler = "pdflatex"
+				timeout: @timeout = 42000
+			@Settings.compileDir = "compiles"
+			@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}"
+			@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3)
+			@LatexRunner.runLatex = sinon.stub().callsArg(2)
+			@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
+			@CompileManager.doCompile @request, @callback
+
+		it "should write the resources to disk", ->
+			@ResourceWriter.syncResourcesToDisk
+				.calledWith(@project_id, @resources, @compileDir)
+				.should.equal true
+
+		it "should run LaTeX", ->
+			@LatexRunner.runLatex
+				.calledWith(@project_id, {
+					directory: @compileDir
+					mainFile:  @rootResourcePath
+					compiler:  @compiler
+					timeout:   @timeout
+				})
+				.should.equal true
+
+		it "should find the output files", ->
+			@OutputFileFinder.findOutputFiles
+				.calledWith(@resources, @compileDir)
+				.should.equal true
+
+		it "should return the output files", ->
+			@callback.calledWith(null, @output_files).should.equal true
+
+	describe "clearProject", ->
+		beforeEach ->
+			@Settings.compileDir = "compiles"
+			@CompileManager.clearProject @project_id, @callback
+
+		it "should remove the project directory", ->
+			@rimraf.calledWith("#{@Settings.compileDir}/#{@project_id}")
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true

test/unit/coffee/LatexRunnerTests.coffee

@@ -0,0 +1,56 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/LatexRunner'
+Path = require "path"
+
+describe "LatexRunner", ->
+	beforeEach ->
+		@LatexRunner = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @Settings =
+				docker:
+					socketPath: "/var/run/docker.sock"
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
+			"./Metrics":
+				Timer: class Timer
+					done: () ->
+			"./CommandRunner": @CommandRunner = {}
+
+		@directory = "/local/compile/directory"
+		@mainFile  = "main-file.tex"
+		@compiler  = "pdflatex"
+		@callback  = sinon.stub()
+		@project_id = "project-id-123"
+
+	describe "runLatex", ->
+		beforeEach ->
+			@CommandRunner.run = sinon.stub().callsArg(4)
+
+		describe "normally", ->
+			beforeEach ->
+				@LatexRunner.runLatex @project_id,
+					directory: @directory
+					mainFile:  @mainFile
+					compiler:  @compiler
+					timeout:   @timeout = 42000
+					@callback
+
+			it "should run the latex command", ->
+				@CommandRunner.run
+					.calledWith(@project_id, sinon.match.any, @directory, @timeout)
+					.should.equal true
+
+		describe "with an .Rtex main file", ->
+			beforeEach ->
+				@LatexRunner.runLatex @project_id,
+					directory: @directory
+					mainFile:  "main-file.Rtex"
+					compiler:  @compiler
+					timeout:   @timeout = 42000
+					@callback
+
+			it "should run the latex command on the equivalent .tex file", ->
+				command = @CommandRunner.run.args[0][1]
+				mainFile = command.slice(-1)[0]
+				mainFile.should.equal "$COMPILE_DIR/main-file.tex"
+

test/unit/coffee/OutputFileFinderTests.coffee

@@ -0,0 +1,41 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/OutputFileFinder'
+path = require "path"
+expect = require("chai").expect
+
+describe "OutputFileFinder", ->
+	beforeEach ->
+		@OutputFileFinder = SandboxedModule.require modulePath, requires:
+			"fs": @fs = {}
+			"wrench": @wrench = {}
+		@directory = "/test/dir"
+		@callback = sinon.stub()
+
+	describe "findOutputFiles", ->
+		beforeEach ->
+			@resource_path = "resource/path.tex"
+			@output_paths   = ["output.pdf", "extra", "extra/file.tex"]
+			@resources = [
+				path: @resource_path = "resource/path.tex"
+			]
+			@OutputFileFinder._isDirectory = (dirPath, callback = (error, directory) ->) =>
+				callback null, dirPath == path.join(@directory, "extra")
+
+			@wrench.readdirRecursive = (dir, callback) =>
+				callback(null, [@resource_path].concat(@output_paths))
+				callback(null, null)
+			sinon.spy @wrench, "readdirRecursive"
+			@OutputFileFinder.findOutputFiles @resources, @directory, (error, @outputFiles) =>
+
+		it "should only return the output files, not directories or resource paths", ->
+			expect(@outputFiles).to.deep.equal [{
+				path: "output.pdf"
+				type: "pdf"
+			}, {
+				path: "extra/file.tex",
+				type: "tex"
+			}]
+
+	

test/unit/coffee/ProjectPersistenceManagerTests.coffee

@@ -0,0 +1,60 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/ProjectPersistenceManager'
+tk = require("timekeeper")
+
+describe "ProjectPersistenceManager", ->
+	beforeEach ->
+		@ProjectPersistenceManager = SandboxedModule.require modulePath, requires:
+			"./UrlCache": @UrlCache = {}
+			"./CompileManager": @CompileManager = {}
+			"logger-sharelatex": @logger = { log: sinon.stub() }
+			"./db": @db = {}
+		@callback = sinon.stub()
+		@project_id = "project-id-123"
+
+	describe "clearExpiredProjects", ->
+		beforeEach ->
+			@project_ids = [
+				"project-id-1"
+				"project-id-2"
+			]
+			@ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids)
+			@ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1)
+			@ProjectPersistenceManager.clearExpiredProjects @callback
+
+		it "should clear each expired project", ->
+			for project_id in @project_ids
+				@ProjectPersistenceManager.clearProject
+					.calledWith(project_id)
+					.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "clearProject", ->
+		beforeEach ->
+			@ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1)
+			@UrlCache.clearProject = sinon.stub().callsArg(1)
+			@CompileManager.clearProject = sinon.stub().callsArg(1)
+			@ProjectPersistenceManager.clearProject @project_id, @callback
+
+		it "should clear the project from the database", ->
+			@ProjectPersistenceManager._clearProjectFromDatabase
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should clear all the cached Urls for the project", ->
+			@UrlCache.clearProject
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should clear the project compile folder", ->
+			@CompileManager.clearProject
+				.calledWith(@project_id)
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+			

test/unit/coffee/RequestParserTests.coffee

@@ -0,0 +1,209 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/RequestParser'
+tk = require("timekeeper")
+
+describe "RequestParser", ->
+	beforeEach ->
+		tk.freeze()
+		@callback = sinon.stub()
+		@validResource =
+			path: "main.tex"
+			date: "12:00 01/02/03"
+			content: "Hello world"
+		@validRequest =
+			compile:
+				token: "token-123"
+				options:
+					compiler: "pdflatex"
+					timeout:  42
+				resources: []
+		@RequestParser = SandboxedModule.require modulePath
+	
+	afterEach ->
+		tk.reset()
+
+	describe "without a top level object", ->
+		beforeEach ->
+			@RequestParser.parse [], @callback
+
+		it "should return an error", ->
+			@callback.calledWith("top level object should have a compile attribute")
+				.should.equal true
+
+	describe "without a compile attribute", ->
+		beforeEach ->
+			@RequestParser.parse {}, @callback
+
+		it "should return an error", ->
+			@callback.calledWith("top level object should have a compile attribute")
+				.should.equal true
+
+	describe "without a valid compiler", ->
+		beforeEach ->
+			@validRequest.compile.options.compiler = "not-a-compiler"
+			@RequestParser.parse @validRequest, @callback
+
+		it "should return an error", ->
+			@callback.calledWith("compiler attribute should be one of: pdflatex, latex, xelatex, lualatex")
+				.should.equal true
+
+	describe "without a compiler specified", ->
+		beforeEach ->
+			delete @validRequest.compile.options.compiler
+			@RequestParser.parse @validRequest, (error, @data) =>
+		
+		it "should set the compiler to pdflatex by default", ->
+			@data.compiler.should.equal "pdflatex"
+
+	describe "without a timeout specified", ->
+		beforeEach ->
+			delete @validRequest.compile.options.timeout
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "should set the timeout to MAX_TIMEOUT", ->
+			@data.timeout.should.equal @RequestParser.MAX_TIMEOUT * 1000
+
+	describe "with a timeout larger than the maximum", ->
+		beforeEach ->
+			@validRequest.compile.options.timeout = @RequestParser.MAX_TIMEOUT + 1
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "should set the timeout to MAX_TIMEOUT", ->
+			@data.timeout.should.equal @RequestParser.MAX_TIMEOUT * 1000
+
+	describe "with a timeout", ->
+		beforeEach ->
+			@RequestParser.parse @validRequest, (error, @data) =>
+
+		it "should set the timeout (in milliseconds)", ->
+			@data.timeout.should.equal @validRequest.compile.options.timeout * 1000
+	
+	describe "with a resource without a path", ->
+		beforeEach ->
+			delete @validResource.path
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse @validRequest, @callback
+
+		it "should return an error", ->
+			@callback.calledWith("all resources should have a path attribute")
+				.should.equal true
+
+	describe "with a resource with a path", ->
+		beforeEach ->
+			@validResource.path = @path = "test.tex"
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse @validRequest, @callback
+			@data = @callback.args[0][1]
+
+		it "should return the path in the parsed response", ->
+			@data.resources[0].path.should.equal @path
+
+	describe "with a resource with a malformed modified date", ->
+		beforeEach ->
+			@validResource.modified = "not-a-date"
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse @validRequest, @callback
+
+		it "should return an error", ->
+			@callback
+				.calledWith(
+					"resource modified date could not be understood: "+
+					@validResource.modified
+				)
+				.should.equal true
+
+	describe "with a resource with a valid date", ->
+		beforeEach ->
+			@date = "12:00 01/02/03"
+			@validResource.modified = @date
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse @validRequest, @callback
+			@data = @callback.args[0][1]
+
+		it "should return the date as a Javascript Date object", ->
+			(@data.resources[0].modified instanceof Date).should.equal true
+			@data.resources[0].modified.getTime().should.equal Date.parse(@date)
+
+	describe "with a resource without either a content or URL attribute", ->
+		beforeEach ->
+			delete @validResource.url
+			delete @validResource.content
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse @validRequest, @callback
+
+		it "should return an error", ->
+			@callback.calledWith("all resources should have either a url or content attribute")
+				.should.equal true
+
+	describe "with a resource where the content is not a string", ->
+		beforeEach ->
+			@validResource.content = []
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse (@validRequest), @callback
+
+		it "should return an error", ->
+			@callback.calledWith("content attribute should be a string")
+				.should.equal true
+
+	describe "with a resource where the url is not a string", ->
+		beforeEach ->
+			@validResource.url = []
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse (@validRequest), @callback
+
+		it "should return an error", ->
+			@callback.calledWith("url attribute should be a string")
+				.should.equal true
+
+	describe "with a resource with a url", ->
+		beforeEach ->
+			@validResource.url = @url = "www.example.com"
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse (@validRequest), @callback
+			@data = @callback.args[0][1]
+
+		it "should return the url in the parsed response", ->
+			@data.resources[0].url.should.equal @url
+		
+	describe "with a resource with a content attribute", ->
+		beforeEach ->
+			@validResource.content = @content = "Hello world"
+			@validRequest.compile.resources.push @validResource
+			@RequestParser.parse (@validRequest), @callback
+			@data = @callback.args[0][1]
+
+		it "should return the content in the parsed response", ->
+			@data.resources[0].content.should.equal @content
+		
+	describe "without a root resource path", ->
+		beforeEach ->
+			delete @validRequest.compile.rootResourcePath
+			@RequestParser.parse (@validRequest), @callback
+			@data = @callback.args[0][1]
+
+		it "should set the root resource path to 'main.tex' by default", ->
+			@data.rootResourcePath.should.equal "main.tex"
+
+	describe "with a root resource path", ->
+		beforeEach ->
+			@validRequest.compile.rootResourcePath = @path = "test.tex"
+			@RequestParser.parse (@validRequest), @callback
+			@data = @callback.args[0][1]
+
+		it "should return the root resource path in the parsed response", ->
+			@data.rootResourcePath.should.equal @path
+
+	describe "with a root resource path that is not a string", ->
+		beforeEach ->
+			@validRequest.compile.rootResourcePath = []
+			@RequestParser.parse (@validRequest), @callback
+
+		it "should return an error", ->
+			@callback.calledWith("rootResourcePath attribute should be a string")
+				.should.equal true
+
+
+		
+

test/unit/coffee/ResourceWriterTests.coffee

@@ -0,0 +1,152 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/ResourceWriter'
+path = require "path"
+
+describe "ResourceWriter", ->
+	beforeEach ->
+		@ResourceWriter = SandboxedModule.require modulePath, requires:
+			"fs": @fs = {}
+			"wrench": @wrench = {}
+			"./UrlCache" : @UrlCache = {}
+			"mkdirp" : @mkdirp = sinon.stub().callsArg(1)
+			"./OutputFileFinder": @OutputFileFinder = {}
+			"./Metrics": @Metrics =
+				Timer: class Timer
+					done: sinon.stub()
+		@project_id = "project-id-123"
+		@basePath = "/path/to/write/files/to"
+		@callback = sinon.stub()
+
+	describe "syncResourcesToDisk", ->
+		beforeEach ->
+			@resources = [
+				"resource-1-mock"
+				"resource-2-mock"
+				"resource-3-mock"
+			]
+			@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
+			@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
+			@ResourceWriter.syncResourcesToDisk(@project_id, @resources, @basePath, @callback)
+
+		it "should remove old files", ->
+			@ResourceWriter._removeExtraneousFiles
+				.calledWith(@resources, @basePath)
+				.should.equal true
+
+		it "should write each resource to disk", ->
+			for resource in @resources
+				@ResourceWriter._writeResourceToDisk
+					.calledWith(@project_id, resource, @basePath)
+					.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "_removeExtraneousFiles", ->
+		beforeEach ->
+			@output_files = [{
+				path: "output.pdf"
+				type: "pdf"
+			}, {
+				path: "extra/file.tex"
+				type: "tex"
+			}, {
+				path: "extra.aux"
+				type: "aux"
+			}]
+			@resources = "mock-resources"
+			@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
+			@ResourceWriter._deleteFileIfNotDirectory = sinon.stub().callsArg(1)
+			@ResourceWriter._removeExtraneousFiles(@resources, @basePath, @callback)
+
+		it "should find the existing output files", ->
+			@OutputFileFinder.findOutputFiles
+				.calledWith(@resources, @basePath)
+				.should.equal true
+
+		it "should delete the output files", ->
+			@ResourceWriter._deleteFileIfNotDirectory
+				.calledWith(path.join(@basePath, "output.pdf"))
+				.should.equal true
+
+		it "should delete the extra files", ->
+			@ResourceWriter._deleteFileIfNotDirectory
+				.calledWith(path.join(@basePath, "extra/file.tex"))
+				.should.equal true
+
+		it "should not delete the extra aux files", ->
+			@ResourceWriter._deleteFileIfNotDirectory
+				.calledWith(path.join(@basePath, "extra.aux"))
+				.should.equal false
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+		it "should time the request", ->
+			@Metrics.Timer::done.called.should.equal true
+
+	describe "_writeResourceToDisk", ->
+		describe "with a url based resource", ->
+			beforeEach ->
+				@resource =
+					path: "main.tex"
+					url: "http://www.example.com/main.tex"
+					modified: Date.now()
+				@UrlCache.downloadUrlToFile = sinon.stub().callsArg(4)
+				@ResourceWriter._writeResourceToDisk(@project_id, @resource, @basePath, @callback)
+
+			it "should ensure the directory exists", ->
+				@mkdirp
+					.calledWith(path.dirname(path.join(@basePath, @resource.path)))
+					.should.equal true
+
+			it "should write the URL from the cache", ->
+				@UrlCache.downloadUrlToFile
+					.calledWith(@project_id, @resource.url, path.join(@basePath, @resource.path), @resource.modified)
+					.should.equal true
+			
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "with a content based resource", ->
+			beforeEach ->
+				@resource =
+					path: "main.tex"
+					content: "Hello world"
+				@fs.writeFile = sinon.stub().callsArg(2)
+				@ResourceWriter._writeResourceToDisk(@project_id, @resource, @basePath, @callback)
+
+			it "should ensure the directory exists", ->
+				@mkdirp
+					.calledWith(path.dirname(path.join(@basePath, @resource.path)))
+					.should.equal true
+
+			it "should write the contents to disk", ->
+				@fs.writeFile
+					.calledWith(path.join(@basePath, @resource.path), @resource.content)
+					.should.equal true
+				
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "with a file path that breaks out of the root folder", ->
+			beforeEach ->
+				@resource =
+					path: "../../main.tex"
+					content: "Hello world"
+				@fs.writeFile = sinon.stub().callsArg(2)
+				@ResourceWriter._writeResourceToDisk(@project_id, @resource, @basePath, @callback)
+
+			it "should not write to disk", ->
+				@fs.writeFile.called.should.equal false
+
+			it "should return an error", ->
+				@callback
+					.calledWith(new Error("resource path is outside root directory"))
+					.should.equal true
+			
+			
+
+			

test/unit/coffee/UrlCacheTests.coffee

@@ -0,0 +1,200 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/UrlCache'
+EventEmitter = require("events").EventEmitter
+
+describe "UrlCache", ->
+	beforeEach ->
+		@callback = sinon.stub()
+		@url = "www.example.com/file"
+		@project_id = "project-id-123"
+		@UrlCache = SandboxedModule.require modulePath, requires:
+			"./db" : {}
+			"./UrlFetcher" : @UrlFetcher = {}
+			"logger-sharelatex": @logger = {log: sinon.stub()}
+			"settings-sharelatex": @Settings = { path: clsiCacheDir: "/cache/dir" }
+			"fs": @fs = {}
+	
+	describe "_doesUrlNeedDownloading", ->
+		beforeEach ->
+			@lastModified = new Date()
+			@lastModifiedRoundedToSeconds = new Date(Math.floor(@lastModified.getTime() / 1000) * 1000)
+
+		describe "when URL does not exist in cache", ->
+			beforeEach ->
+				@UrlCache._findUrlDetails = sinon.stub().callsArgWith(2, null, null)
+				@UrlCache._doesUrlNeedDownloading(@project_id, @url, @lastModified, @callback)
+
+			it "should return the callback with true", ->
+				@callback.calledWith(null, true).should.equal true
+
+		describe "when URL does exist in cache", ->
+			beforeEach ->
+				@urlDetails = {}
+				@UrlCache._findUrlDetails = sinon.stub().callsArgWith(2, null, @urlDetails)
+
+			describe "when the modified date is more recent than the cached modified date", ->
+				beforeEach ->
+					@urlDetails.lastModified = new Date(@lastModified.getTime() - 1000)
+					@UrlCache._doesUrlNeedDownloading(@project_id, @url, @lastModified, @callback)
+
+				it "should get the url details", ->
+					@UrlCache._findUrlDetails
+						.calledWith(@project_id, @url)
+						.should.equal true
+
+				it "should return the callback with true", ->
+					@callback.calledWith(null, true).should.equal true
+
+			describe "when the cached modified date is more recent than the modified date", ->
+				beforeEach ->
+					@urlDetails.lastModified = new Date(@lastModified.getTime() + 1000)
+					@UrlCache._doesUrlNeedDownloading(@project_id, @url, @lastModified, @callback)
+
+				it "should return the callback with false", ->
+					@callback.calledWith(null, false).should.equal true
+
+			describe "when the cached modified date is equal to the modified date", ->
+				beforeEach ->
+					@urlDetails.lastModified = @lastModified
+					@UrlCache._doesUrlNeedDownloading(@project_id, @url, @lastModified, @callback)
+
+				it "should return the callback with false", ->
+					@callback.calledWith(null, false).should.equal true
+
+			describe "when the provided modified date does not exist", ->
+				beforeEach ->
+					@lastModified = null
+					@UrlCache._doesUrlNeedDownloading(@project_id, @url, @lastModified, @callback)
+
+				it "should return the callback with true", ->
+					@callback.calledWith(null, true).should.equal true
+
+			describe "when the URL does not have a modified date", ->
+				beforeEach ->
+					@urlDetails.lastModified = null
+					@UrlCache._doesUrlNeedDownloading(@project_id, @url, @lastModified, @callback)
+
+				it "should return the callback with true", ->
+					@callback.calledWith(null, true).should.equal true
+
+	describe "_ensureUrlIsInCache", ->
+		beforeEach ->
+			@UrlFetcher.pipeUrlToFile = sinon.stub().callsArg(2)
+			@UrlCache._updateOrCreateUrlDetails = sinon.stub().callsArg(3)
+			
+		describe "when the URL needs updating", ->
+			beforeEach ->
+				@UrlCache._doesUrlNeedDownloading = sinon.stub().callsArgWith(3, null, true)
+				@UrlCache._ensureUrlIsInCache(@project_id, @url, @lastModified, @callback)
+
+			it "should check that the url needs downloading", ->
+				@UrlCache._doesUrlNeedDownloading
+					.calledWith(@project_id, @url, @lastModifiedRoundedToSeconds)
+					.should.equal true
+
+			it "should download the URL to the cache file", ->
+				@UrlFetcher.pipeUrlToFile
+					.calledWith(@url, @UrlCache._cacheFilePathForUrl(@project_id, @url))
+					.should.equal true
+				
+
+			it "should update the database entry", ->
+				@UrlCache._updateOrCreateUrlDetails
+					.calledWith(@project_id, @url, @lastModifiedRoundedToSeconds)
+					.should.equal true
+
+			it "should return the callback with the cache file path", ->
+				@callback
+					.calledWith(null, @UrlCache._cacheFilePathForUrl(@project_id, @url))
+					.should.equal true
+
+		describe "when the URL does not need updating", ->
+			beforeEach ->
+				@UrlCache._doesUrlNeedDownloading = sinon.stub().callsArgWith(3, null, false)
+				@UrlCache._ensureUrlIsInCache(@project_id, @url, @lastModified, @callback)
+				
+			it "should not download the URL to the cache file", ->
+				@UrlFetcher.pipeUrlToFile
+					.called.should.equal false
+
+			it "should return the callback with the cache file path", ->
+				@callback
+					.calledWith(null, @UrlCache._cacheFilePathForUrl(@project_id, @url))
+					.should.equal true
+
+	describe "downloadUrlToFile", ->
+		beforeEach ->
+			@cachePath = "path/to/cached/url"
+			@destPath = "path/to/destination"
+			@UrlCache._copyFile = sinon.stub().callsArg(2)
+			@UrlCache._ensureUrlIsInCache = sinon.stub().callsArgWith(3, null, @cachePath)
+			@UrlCache.downloadUrlToFile(@project_id, @url, @destPath, @lastModified, @callback)
+
+		it "should ensure the URL is downloaded and updated in the cache", ->
+			@UrlCache._ensureUrlIsInCache
+				.calledWith(@project_id, @url, @lastModified)
+				.should.equal true
+
+		it "should copy the file to the new location", ->
+			@UrlCache._copyFile
+				.calledWith(@cachePath, @destPath)
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "_deleteUrlCacheFromDisk", ->
+		beforeEach ->
+			@fs.unlink = sinon.stub().callsArg(1)
+			@UrlCache._deleteUrlCacheFromDisk(@project_id, @url, @callback)
+
+		it "should delete the cache file", ->
+			@fs.unlink
+				.calledWith(@UrlCache._cacheFilePathForUrl(@project_id, @url))
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "_clearUrlFromCache", ->
+		beforeEach ->
+			@UrlCache._deleteUrlCacheFromDisk = sinon.stub().callsArg(2)
+			@UrlCache._clearUrlDetails = sinon.stub().callsArg(2)
+			@UrlCache._clearUrlFromCache @project_id, @url, @callback
+
+		it "should delete the file on disk", ->
+			@UrlCache._deleteUrlCacheFromDisk
+				.calledWith(@project_id, @url)
+				.should.equal true
+
+		it "should clear the entry in the database", ->
+			@UrlCache._clearUrlDetails
+				.calledWith(@project_id, @url)
+				.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+
+	describe "clearProject", ->
+		beforeEach ->
+			@urls = [
+				"www.example.com/file1"
+				"www.example.com/file2"
+			]
+			@UrlCache._findAllUrlsInProject = sinon.stub().callsArgWith(1, null, @urls)
+			@UrlCache._clearUrlFromCache = sinon.stub().callsArg(2)
+			@UrlCache.clearProject @project_id, @callback
+
+		it "should clear the cache for each url in the project", ->
+			for url in @urls
+				@UrlCache._clearUrlFromCache
+					.calledWith(@project_id, url)
+					.should.equal true
+
+		it "should call the callback", ->
+			@callback.called.should.equal true
+			
+				
+

test/unit/coffee/UrlFetcherTests.coffee

@@ -0,0 +1,74 @@
+SandboxedModule = require('sandboxed-module')
+sinon = require('sinon')
+require('chai').should()
+modulePath = require('path').join __dirname, '../../../app/js/UrlFetcher'
+EventEmitter = require("events").EventEmitter
+
+describe "UrlFetcher", ->
+	beforeEach ->
+		@callback = sinon.stub()
+		@url = "www.example.com/file"
+		@UrlFetcher = SandboxedModule.require modulePath, requires:
+			request: defaults: @defaults = sinon.stub().returns(@request = {})
+			fs: @fs = {}
+
+	it "should turn off the cookie jar in request", ->
+		@defaults.calledWith(jar: false)
+			.should.equal true
+		
+	describe "_pipeUrlToFile", ->
+		beforeEach ->
+			@path = "/path/to/file/on/disk"
+			@request.get = sinon.stub().returns(@urlStream = new EventEmitter)
+			@urlStream.pipe = sinon.stub()
+			@fs.createWriteStream = sinon.stub().returns(@fileStream = "write-stream-stub")
+			@UrlFetcher.pipeUrlToFile(@url, @path, @callback)
+
+		it "should request the URL", ->
+			@request.get
+				.calledWith(@url)
+				.should.equal true
+
+		it "should open the file for writing", ->
+			@fs.createWriteStream
+				.calledWith(@path)
+				.should.equal true
+
+		describe "successfully", ->
+			beforeEach ->
+				@res = statusCode: 200
+				@urlStream.emit "response", @res
+				@urlStream.emit "end"
+
+			it "should pipe the URL to the file", ->
+				@urlStream.pipe
+					.calledWith(@fileStream)
+					.should.equal true
+		
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "with non success status code", ->
+			beforeEach ->
+				@res = statusCode: 404
+				@urlStream.emit "response", @res
+				@urlStream.emit "end"
+
+			it "should call the callback with an error", ->
+				@callback
+					.calledWith(new Error("URL returned non-success status code: 404"))
+					.should.equal true
+
+		describe "with error", ->
+			beforeEach ->
+				@urlStream.emit "error", @error = new Error("something went wrong")
+
+			it "should call the callback with the error", ->
+				@callback
+					.calledWith(@error)
+					.should.equal true
+
+			it "should only call the callback once, even if end is called", ->
+				@urlStream.emit "end"
+				@callback.calledOnce.should.equal true
+