From 8c42a353e16f31cf088c20878f2ccd7b7ccc28c8 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:25:13 +0100 Subject: [PATCH 1/4] put the build id in the output file urls the url attribute will now give the preferred location for accessing the output file, without the url having to be constructed by the web client --- app/coffee/CompileController.coffee | 5 ++++- test/unit/coffee/CompileControllerTests.coffee | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/coffee/CompileController.coffee b/app/coffee/CompileController.coffee index 5cc9a5e..478d91b 100644 --- a/app/coffee/CompileController.coffee +++ b/app/coffee/CompileController.coffee @@ -33,7 +33,10 @@ module.exports = CompileController = status: status error: error?.message or error outputFiles: outputFiles.map (file) -> - url: "#{Settings.apis.clsi.url}/project/#{request.project_id}/output/#{file.path}" + url: + "#{Settings.apis.clsi.url}/project/#{request.project_id}" + + (if file.build? then "/build/#{file.build}" else "") + + "/output/#{file.path}" path: file.path type: file.type build: file.build diff --git a/test/unit/coffee/CompileControllerTests.coffee b/test/unit/coffee/CompileControllerTests.coffee index d8d6e86..7693799 100644 --- a/test/unit/coffee/CompileControllerTests.coffee +++ b/test/unit/coffee/CompileControllerTests.coffee @@ -75,7 +75,7 @@ describe "CompileController", -> status: "success" error: null outputFiles: @output_files.map (file) => - url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/output/#{file.path}" + url: "#{@Settings.apis.clsi.url}/project/#{@project_id}/build/#{file.build}/output/#{file.path}" path: file.path type: file.type build: file.build From 226e6c87b1b06c7da505700e667b4506a37c8947 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:29:26 +0100 Subject: [PATCH 2/4] add per-user routes and methods --- app.coffee | 12 ++++- app/coffee/CompileController.coffee | 11 ++-- app/coffee/CompileManager.coffee | 50 ++++++++++++------- .../unit/coffee/CompileControllerTests.coffee | 12 ++--- test/unit/coffee/CompileManagerTests.coffee | 25 +++++----- 5 files changed, 68 insertions(+), 42 deletions(-) diff --git a/app.coffee b/app.coffee index 9f54f4a..cfe8ebc 100644 --- a/app.coffee +++ b/app.coffee @@ -42,8 +42,8 @@ app.param 'project_id', (req, res, next, project_id) -> else next new Error("invalid project id") -app.param 'user_id', (req, res, next, project_id) -> - if project_id?.match /^[a-zA-Z0-9_-]+$/ +app.param 'user_id', (req, res, next, user_id) -> + if user_id?.match /^[0-9a-f]{24}$/ next() else next new Error("invalid user id") @@ -63,6 +63,14 @@ app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf app.get "/project/:project_id/wordcount", CompileController.wordcount app.get "/project/:project_id/status", CompileController.status +# Per-user containers +app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile +# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache + +app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode +app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf +app.get "/project/:project_id/user/:user_id/wordcount", CompileController.wordcount + ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks" # create a static server which does not allow access to any symlinks diff --git a/app/coffee/CompileController.coffee b/app/coffee/CompileController.coffee index 478d91b..7e1f078 100644 --- a/app/coffee/CompileController.coffee +++ b/app/coffee/CompileController.coffee @@ -11,6 +11,7 @@ module.exports = CompileController = RequestParser.parse req.body, (error, request) -> return next(error) if error? request.project_id = req.params.project_id + request.user_id = req.params.user_id if req.params.user_id? ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) -> return next(error) if error? CompileManager.doCompile request, (error, outputFiles = []) -> @@ -35,6 +36,7 @@ module.exports = CompileController = outputFiles: outputFiles.map (file) -> url: "#{Settings.apis.clsi.url}/project/#{request.project_id}" + + (if request.user_id? then "/user/#{request.user_id}" else "") + (if file.build? then "/build/#{file.build}" else "") + "/output/#{file.path}" path: file.path @@ -52,8 +54,9 @@ module.exports = CompileController = line = parseInt(req.query.line, 10) column = parseInt(req.query.column, 10) project_id = req.params.project_id + user_id = req.params.user_id - CompileManager.syncFromCode project_id, file, line, column, (error, pdfPositions) -> + CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) -> return next(error) if error? res.send JSON.stringify { pdf: pdfPositions @@ -64,8 +67,9 @@ module.exports = CompileController = h = parseFloat(req.query.h) v = parseFloat(req.query.v) project_id = req.params.project_id + user_id = req.params.user_id - CompileManager.syncFromPdf project_id, page, h, v, (error, codePositions) -> + CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) -> return next(error) if error? res.send JSON.stringify { code: codePositions @@ -74,10 +78,11 @@ module.exports = CompileController = wordcount: (req, res, next = (error) ->) -> file = req.query.file || "main.tex" project_id = req.params.project_id + user_id = req.params.user_id image = req.query.image logger.log {image, file, project_id}, "word count request" - CompileManager.wordcount project_id, file, image, (error, result) -> + CompileManager.wordcount project_id, user_id, file, image, (error, result) -> return next(error) if error? res.send JSON.stringify { texcount: result diff --git a/app/coffee/CompileManager.coffee b/app/coffee/CompileManager.coffee index 41377ab..cd332a4 100644 --- a/app/coffee/CompileManager.coffee +++ b/app/coffee/CompileManager.coffee @@ -15,17 +15,23 @@ commandRunner = Settings.clsi?.commandRunner or "./CommandRunner" logger.info commandRunner:commandRunner, "selecting command runner for clsi" CommandRunner = require(commandRunner) +getCompileName = (project_id, user_id) -> + if user_id? then "#{project_id}-#{user_id}" else project_id + +getCompileDir = (project_id, user_id) -> + Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)) + module.exports = CompileManager = doCompile: (request, callback = (error, outputFiles) ->) -> - compileDir = Path.join(Settings.path.compilesDir, request.project_id) + compileDir = getCompileDir(request.project_id, request.user_id) timer = new Metrics.Timer("write-to-disk") - logger.log project_id: request.project_id, "starting doCompile" + logger.log project_id: request.project_id, user_id: request.user_id, "starting compile" ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) -> if error? - logger.err err:error, project_id: request.project_id, "error writing resources to disk" + logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk" return callback(error) - logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "written files to disk" + logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk" timer.done() injectDraftModeIfRequired = (callback) -> @@ -42,7 +48,8 @@ module.exports = CompileManager = tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test Metrics.inc("compiles") Metrics.inc("compiles-with-image.#{tag}") - LatexRunner.runLatex request.project_id, { + compileName = getCompileName(request.project_id, request.user_id) + LatexRunner.runLatex compileName, { directory: compileDir mainFile: request.rootResourcePath compiler: request.compiler @@ -58,7 +65,7 @@ module.exports = CompileManager = loadavg = os.loadavg?() Metrics.gauge("load-avg", loadavg[0]) if loadavg? ts = timer.done() - logger.log {project_id: request.project_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" + logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" if stats?["latex-runs"] > 0 Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]) if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0 @@ -106,24 +113,28 @@ module.exports = CompileManager = else callback(null, true) # directory exists - syncFromCode: (project_id, file_name, line, column, callback = (error, pdfPositions) ->) -> + syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) -> # If LaTeX was run in a virtual environment, the file path that synctex expects # might not match the file path on the host. The .synctex.gz file however, will be accessed # wherever it is on the host. - base_dir = Settings.path.synctexBaseDir(project_id) + compileName = getCompileName(project_id, user_id) + base_dir = Settings.path.synctexBaseDir(compileName) file_path = base_dir + "/" + file_name - synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf") + compileDir = getCompileDir(project_id, user_id) + synctex_path = Path.join(compileDir, "output.pdf") CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) -> return callback(error) if error? - logger.log project_id: project_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output" + logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output" callback null, CompileManager._parseSynctexFromCodeOutput(stdout) - syncFromPdf: (project_id, page, h, v, callback = (error, filePositions) ->) -> - base_dir = Settings.path.synctexBaseDir(project_id) - synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf") + syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) -> + compileName = getCompileName(project_id, user_id) + base_dir = Settings.path.synctexBaseDir(compileName) + compileDir = getCompileDir(project_id, user_id) + synctex_path = Path.join(compileDir, "output.pdf") CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) -> return callback(error) if error? - logger.log project_id: project_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" + logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir) _runSynctex: (args, callback = (error, stdout) ->) -> @@ -162,19 +173,20 @@ module.exports = CompileManager = } return results - wordcount: (project_id, file_name, image, callback = (error, pdfPositions) ->) -> - logger.log project_id:project_id, file_name:file_name, image:image, "running wordcount" + wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) -> + logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount" file_path = "$COMPILE_DIR/" + file_name command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"] - directory = Path.join(Settings.path.compilesDir, project_id) + directory = getCompileDir(project_id, user_id) timeout = 10 * 1000 + compileName = getCompileName(project_id, user_id) - CommandRunner.run project_id, command, directory, image, timeout, (error) -> + CommandRunner.run compileName, command, directory, image, timeout, (error) -> return callback(error) if error? try stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") catch err - logger.err err:err, command:command, directory:directory, project_id:project_id, "error reading word count output" + logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output" return callback(err) callback null, CompileManager._parseWordcountFromOutput(stdout) diff --git a/test/unit/coffee/CompileControllerTests.coffee b/test/unit/coffee/CompileControllerTests.coffee index 7693799..1fc6a99 100644 --- a/test/unit/coffee/CompileControllerTests.coffee +++ b/test/unit/coffee/CompileControllerTests.coffee @@ -146,12 +146,12 @@ describe "CompileController", -> column: @column.toString() @res.send = sinon.stub() - @CompileManager.syncFromCode = sinon.stub().callsArgWith(4, null, @pdfPositions = ["mock-positions"]) + @CompileManager.syncFromCode = sinon.stub().callsArgWith(5, null, @pdfPositions = ["mock-positions"]) @CompileController.syncFromCode @req, @res, @next it "should find the corresponding location in the PDF", -> @CompileManager.syncFromCode - .calledWith(@project_id, @file, @line, @column) + .calledWith(@project_id, undefined, @file, @line, @column) .should.equal true it "should return the positions", -> @@ -175,12 +175,12 @@ describe "CompileController", -> v: @v.toString() @res.send = sinon.stub() - @CompileManager.syncFromPdf = sinon.stub().callsArgWith(4, null, @codePositions = ["mock-positions"]) + @CompileManager.syncFromPdf = sinon.stub().callsArgWith(5, null, @codePositions = ["mock-positions"]) @CompileController.syncFromPdf @req, @res, @next it "should find the corresponding location in the code", -> @CompileManager.syncFromPdf - .calledWith(@project_id, @page, @h, @v) + .calledWith(@project_id, undefined, @page, @h, @v) .should.equal true it "should return the positions", -> @@ -201,12 +201,12 @@ describe "CompileController", -> image: @image = "example.com/image" @res.send = sinon.stub() - @CompileManager.wordcount = sinon.stub().callsArgWith(3, null, @texcount = ["mock-texcount"]) + @CompileManager.wordcount = sinon.stub().callsArgWith(4, null, @texcount = ["mock-texcount"]) @CompileController.wordcount @req, @res, @next it "should return the word count of a file", -> @CompileManager.wordcount - .calledWith(@project_id, @file, @image) + .calledWith(@project_id, undefined, @file, @image) .should.equal true it "should return the texcount info", -> diff --git a/test/unit/coffee/CompileManagerTests.coffee b/test/unit/coffee/CompileManagerTests.coffee index 55f5cc5..0ba62b9 100644 --- a/test/unit/coffee/CompileManagerTests.coffee +++ b/test/unit/coffee/CompileManagerTests.coffee @@ -43,11 +43,12 @@ describe "CompileManager", -> resources: @resources = "mock-resources" rootResourcePath: @rootResourcePath = "main.tex" project_id: @project_id = "project-id-123" + user_id: @user_id = "1234" compiler: @compiler = "pdflatex" timeout: @timeout = 42000 imageName: @image = "example.com/image" @Settings.compileDir = "compiles" - @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}" + @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3) @LatexRunner.runLatex = sinon.stub().callsArg(2) @OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files) @@ -65,7 +66,7 @@ describe "CompileManager", -> it "should run LaTeX", -> @LatexRunner.runLatex - .calledWith(@project_id, { + .calledWith("#{@project_id}-#{@user_id}", { directory: @compileDir mainFile: @rootResourcePath compiler: @compiler @@ -150,17 +151,17 @@ describe "CompileManager", -> @column = 3 @file_name = "main.tex" @child_process.execFile = sinon.stub() - @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}" + @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" describe "syncFromCode", -> beforeEach -> @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n", "") - @CompileManager.syncFromCode @project_id, @file_name, @line, @column, @callback + @CompileManager.syncFromCode @project_id, @user_id, @file_name, @line, @column, @callback it "should execute the synctex binary", -> bin_path = Path.resolve(__dirname + "/../../../bin/synctex") - synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf" - file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}" + synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf" + file_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}" @child_process.execFile .calledWith(bin_path, ["code", synctex_path, file_path, @line, @column], timeout: 10000) .should.equal true @@ -178,12 +179,12 @@ describe "CompileManager", -> describe "syncFromPdf", -> beforeEach -> - @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}\t#{@line}\t#{@column}\n", "") - @CompileManager.syncFromPdf @project_id, @page, @h, @v, @callback + @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n", "") + @CompileManager.syncFromPdf @project_id, @user_id, @page, @h, @v, @callback it "should execute the synctex binary", -> bin_path = Path.resolve(__dirname + "/../../../bin/synctex") - synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf" + synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf" @child_process.execFile .calledWith(bin_path, ["pdf", synctex_path, @page, @h, @v], timeout: 10000) .should.equal true @@ -209,15 +210,15 @@ describe "CompileManager", -> @Settings.path.compilesDir = "/local/compile/directory" @image = "example.com/image" - @CompileManager.wordcount @project_id, @file_name, @image, @callback + @CompileManager.wordcount @project_id, @user_id, @file_name, @image, @callback it "should run the texcount command", -> - @directory = "#{@Settings.path.compilesDir}/#{@project_id}" + @directory = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @file_path = "$COMPILE_DIR/#{@file_name}" @command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"] @CommandRunner.run - .calledWith(@project_id, @command, @directory, @image, @timeout) + .calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout) .should.equal true it "should call the callback with the parsed output", -> From 0887fe3a7289b474ee12d5e421aa09c70137ab1d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:31:44 +0100 Subject: [PATCH 3/4] add per-user routes for clearing cache and extend expiry methods this adds separate functionality for clearing the cache (assets and database) and the project compile directory for a specific user --- app.coffee | 2 +- app/coffee/CompileController.coffee | 2 +- app/coffee/CompileManager.coffee | 27 ++++++++++++++-- app/coffee/ProjectPersistenceManager.coffee | 32 ++++++++++++------- test/unit/coffee/CompileManagerTests.coffee | 10 +++--- .../ProjectPersistenceManagerTests.coffee | 12 ++++--- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/app.coffee b/app.coffee index cfe8ebc..6a8d73f 100644 --- a/app.coffee +++ b/app.coffee @@ -65,7 +65,7 @@ app.get "/project/:project_id/status", CompileController.status # Per-user containers app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile -# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache +app.delete "/project/:project_id/user/:user_id", CompileController.clearCache app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf diff --git a/app/coffee/CompileController.coffee b/app/coffee/CompileController.coffee index 7e1f078..d4dddd4 100644 --- a/app/coffee/CompileController.coffee +++ b/app/coffee/CompileController.coffee @@ -45,7 +45,7 @@ module.exports = CompileController = } clearCache: (req, res, next = (error) ->) -> - ProjectPersistenceManager.clearProject req.params.project_id, (error) -> + ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) -> return next(error) if error? res.sendStatus(204) # No content diff --git a/app/coffee/CompileManager.coffee b/app/coffee/CompileManager.coffee index cd332a4..bb93dbd 100644 --- a/app/coffee/CompileManager.coffee +++ b/app/coffee/CompileManager.coffee @@ -9,7 +9,9 @@ Metrics = require "./Metrics" child_process = require "child_process" DraftModeManager = require "./DraftModeManager" fs = require("fs") +fse = require "fs-extra" os = require("os") +async = require "async" commandRunner = Settings.clsi?.commandRunner or "./CommandRunner" logger.info commandRunner:commandRunner, "selecting command runner for clsi" @@ -76,12 +78,12 @@ module.exports = CompileManager = OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> callback null, newOutputFiles - clearProject: (project_id, _callback = (error) ->) -> + clearProject: (project_id, user_id, _callback = (error) ->) -> callback = (error) -> _callback(error) _callback = () -> - compileDir = Path.join(Settings.path.compilesDir, project_id) + compileDir = getCompileDir(project_id, user_id) CompileManager._checkDirectory compileDir, (err, exists) -> return callback(err) if err? @@ -100,6 +102,27 @@ module.exports = CompileManager = else return callback(new Error("rm -r #{compileDir} failed: #{stderr}")) + _findAllDirs: (callback = (error, allDirs) ->) -> + root = Settings.path.compilesDir + fs.readdir root, (err, files) -> + return callback(err) if err? + allDirs = (Path.join(root, file) for file in files) + callback(null, allDirs) + + clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) -> + now = Date.now() + # action for each directory + expireIfNeeded = (checkDir, cb) -> + fs.stat checkDir, (err, stats) -> + return cb() if err? # ignore errors checking directory + age = now - stats.mtime + hasExpired = (age > max_cache_age_ms) + if hasExpired then fse.remove(checkDir, cb) else cb() + # iterate over all project directories + CompileManager._findAllDirs (error, allDirs) -> + return callback() if error? + async.eachSeries allDirs, expireIfNeeded, callback + _checkDirectory: (compileDir, callback = (error, exists) ->) -> fs.lstat compileDir, (err, stats) -> if err?.code is 'ENOENT' diff --git a/app/coffee/ProjectPersistenceManager.coffee b/app/coffee/ProjectPersistenceManager.coffee index f70f43c..200d977 100644 --- a/app/coffee/ProjectPersistenceManager.coffee +++ b/app/coffee/ProjectPersistenceManager.coffee @@ -27,21 +27,30 @@ module.exports = ProjectPersistenceManager = jobs = for project_id in (project_ids or []) do (project_id) -> (callback) -> - ProjectPersistenceManager.clearProject project_id, (err) -> + ProjectPersistenceManager.clearProjectFromCache project_id, (err) -> if err? logger.error err: err, project_id: project_id, "error clearing project" callback() - async.series jobs, callback - - clearProject: (project_id, callback = (error) ->) -> - logger.log project_id: project_id, "clearing project" - CompileManager.clearProject project_id, (error) -> - return callback(error) if error? - UrlCache.clearProject project_id, (error) -> + async.series jobs, (error) -> return callback(error) if error? - ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> - return callback(error) if error? - callback() + CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) -> + callback() # ignore any errors from deleting directories + + clearProject: (project_id, user_id, callback = (error) ->) -> + logger.log project_id: project_id, "clearing project for user" + CompileManager.clearProject project_id, user_id, (error) -> + return callback(error) if error? + ProjectPersistenceManager.clearProjectFromCache project_id, (error) -> + return callback(error) if error? + callback() + + clearProjectFromCache: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "clearing project from cache" + UrlCache.clearProject project_id, (error) -> + return callback(error) if error? + ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> + return callback(error) if error? + callback() _clearProjectFromDatabase: (project_id, callback = (error) ->) -> db.Project.destroy(where: {project_id: project_id}) @@ -54,5 +63,4 @@ module.exports = ProjectPersistenceManager = callback null, projects.map((project) -> project.project_id) ).error callback - logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout" diff --git a/test/unit/coffee/CompileManagerTests.coffee b/test/unit/coffee/CompileManagerTests.coffee index 0ba62b9..611ed11 100644 --- a/test/unit/coffee/CompileManagerTests.coffee +++ b/test/unit/coffee/CompileManagerTests.coffee @@ -105,12 +105,12 @@ describe "CompileManager", -> @proc.stdout = new EventEmitter() @proc.stderr = new EventEmitter() @child_process.spawn = sinon.stub().returns(@proc) - @CompileManager.clearProject @project_id, @callback + @CompileManager.clearProject @project_id, @user_id, @callback @proc.emit "close", 0 it "should remove the project directory", -> @child_process.spawn - .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"]) + .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"]) .should.equal true it "should call the callback", -> @@ -124,13 +124,13 @@ describe "CompileManager", -> @proc.stdout = new EventEmitter() @proc.stderr = new EventEmitter() @child_process.spawn = sinon.stub().returns(@proc) - @CompileManager.clearProject @project_id, @callback + @CompileManager.clearProject @project_id, @user_id, @callback @proc.stderr.emit "data", @error = "oops" @proc.emit "close", 1 it "should remove the project directory", -> @child_process.spawn - .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"]) + .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"]) .should.equal true it "should call the callback with an error from the stderr", -> @@ -138,7 +138,7 @@ describe "CompileManager", -> .calledWith(new Error()) .should.equal true - @callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id} failed: #{@error}" + @callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id}-#{@user_id} failed: #{@error}" describe "syncing", -> beforeEach -> diff --git a/test/unit/coffee/ProjectPersistenceManagerTests.coffee b/test/unit/coffee/ProjectPersistenceManagerTests.coffee index f8c78ef..69bfd4f 100644 --- a/test/unit/coffee/ProjectPersistenceManagerTests.coffee +++ b/test/unit/coffee/ProjectPersistenceManagerTests.coffee @@ -13,6 +13,7 @@ describe "ProjectPersistenceManager", -> "./db": @db = {} @callback = sinon.stub() @project_id = "project-id-123" + @user_id = "1234" describe "clearExpiredProjects", -> beforeEach -> @@ -21,12 +22,13 @@ describe "ProjectPersistenceManager", -> "project-id-2" ] @ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids) - @ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1) + @ProjectPersistenceManager.clearProjectFromCache = sinon.stub().callsArg(1) + @CompileManager.clearExpiredProjects = sinon.stub().callsArg(1) @ProjectPersistenceManager.clearExpiredProjects @callback it "should clear each expired project", -> for project_id in @project_ids - @ProjectPersistenceManager.clearProject + @ProjectPersistenceManager.clearProjectFromCache .calledWith(project_id) .should.equal true @@ -37,8 +39,8 @@ describe "ProjectPersistenceManager", -> beforeEach -> @ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1) @UrlCache.clearProject = sinon.stub().callsArg(1) - @CompileManager.clearProject = sinon.stub().callsArg(1) - @ProjectPersistenceManager.clearProject @project_id, @callback + @CompileManager.clearProject = sinon.stub().callsArg(2) + @ProjectPersistenceManager.clearProject @project_id, @user_id, @callback it "should clear the project from the database", -> @ProjectPersistenceManager._clearProjectFromDatabase @@ -52,7 +54,7 @@ describe "ProjectPersistenceManager", -> it "should clear the project compile folder", -> @CompileManager.clearProject - .calledWith(@project_id) + .calledWith(@project_id, @user_id) .should.equal true it "should call the callback", -> From 6e017ecaf10edd02f8d1d3d5fef5c40d80c8ba6a Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 31 May 2016 16:12:55 +0100 Subject: [PATCH 4/4] log user_id when clearing project --- app/coffee/ProjectPersistenceManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/coffee/ProjectPersistenceManager.coffee b/app/coffee/ProjectPersistenceManager.coffee index 200d977..403043f 100644 --- a/app/coffee/ProjectPersistenceManager.coffee +++ b/app/coffee/ProjectPersistenceManager.coffee @@ -37,7 +37,7 @@ module.exports = ProjectPersistenceManager = callback() # ignore any errors from deleting directories clearProject: (project_id, user_id, callback = (error) ->) -> - logger.log project_id: project_id, "clearing project for user" + logger.log project_id: project_id, user_id:user_id, "clearing project for user" CompileManager.clearProject project_id, user_id, (error) -> return callback(error) if error? ProjectPersistenceManager.clearProjectFromCache project_id, (error) ->