diff --git a/app.coffee b/app.coffee index cfe8ebc..6a8d73f 100644 --- a/app.coffee +++ b/app.coffee @@ -65,7 +65,7 @@ app.get "/project/:project_id/status", CompileController.status # Per-user containers app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile -# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache +app.delete "/project/:project_id/user/:user_id", CompileController.clearCache app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf diff --git a/app/coffee/CompileController.coffee b/app/coffee/CompileController.coffee index 7e1f078..d4dddd4 100644 --- a/app/coffee/CompileController.coffee +++ b/app/coffee/CompileController.coffee @@ -45,7 +45,7 @@ module.exports = CompileController = } clearCache: (req, res, next = (error) ->) -> - ProjectPersistenceManager.clearProject req.params.project_id, (error) -> + ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) -> return next(error) if error? res.sendStatus(204) # No content diff --git a/app/coffee/CompileManager.coffee b/app/coffee/CompileManager.coffee index cd332a4..bb93dbd 100644 --- a/app/coffee/CompileManager.coffee +++ b/app/coffee/CompileManager.coffee @@ -9,7 +9,9 @@ Metrics = require "./Metrics" child_process = require "child_process" DraftModeManager = require "./DraftModeManager" fs = require("fs") +fse = require "fs-extra" os = require("os") +async = require "async" commandRunner = Settings.clsi?.commandRunner or "./CommandRunner" logger.info commandRunner:commandRunner, "selecting command runner for clsi" @@ -76,12 +78,12 @@ module.exports = CompileManager = OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) -> callback null, newOutputFiles - clearProject: (project_id, _callback = (error) ->) -> + clearProject: (project_id, user_id, _callback = (error) ->) -> callback = (error) -> _callback(error) _callback = () -> - compileDir = Path.join(Settings.path.compilesDir, project_id) + compileDir = getCompileDir(project_id, user_id) CompileManager._checkDirectory compileDir, (err, exists) -> return callback(err) if err? @@ -100,6 +102,27 @@ module.exports = CompileManager = else return callback(new Error("rm -r #{compileDir} failed: #{stderr}")) + _findAllDirs: (callback = (error, allDirs) ->) -> + root = Settings.path.compilesDir + fs.readdir root, (err, files) -> + return callback(err) if err? + allDirs = (Path.join(root, file) for file in files) + callback(null, allDirs) + + clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) -> + now = Date.now() + # action for each directory + expireIfNeeded = (checkDir, cb) -> + fs.stat checkDir, (err, stats) -> + return cb() if err? # ignore errors checking directory + age = now - stats.mtime + hasExpired = (age > max_cache_age_ms) + if hasExpired then fse.remove(checkDir, cb) else cb() + # iterate over all project directories + CompileManager._findAllDirs (error, allDirs) -> + return callback() if error? + async.eachSeries allDirs, expireIfNeeded, callback + _checkDirectory: (compileDir, callback = (error, exists) ->) -> fs.lstat compileDir, (err, stats) -> if err?.code is 'ENOENT' diff --git a/app/coffee/ProjectPersistenceManager.coffee b/app/coffee/ProjectPersistenceManager.coffee index f70f43c..200d977 100644 --- a/app/coffee/ProjectPersistenceManager.coffee +++ b/app/coffee/ProjectPersistenceManager.coffee @@ -27,21 +27,30 @@ module.exports = ProjectPersistenceManager = jobs = for project_id in (project_ids or []) do (project_id) -> (callback) -> - ProjectPersistenceManager.clearProject project_id, (err) -> + ProjectPersistenceManager.clearProjectFromCache project_id, (err) -> if err? logger.error err: err, project_id: project_id, "error clearing project" callback() - async.series jobs, callback - - clearProject: (project_id, callback = (error) ->) -> - logger.log project_id: project_id, "clearing project" - CompileManager.clearProject project_id, (error) -> - return callback(error) if error? - UrlCache.clearProject project_id, (error) -> + async.series jobs, (error) -> return callback(error) if error? - ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> - return callback(error) if error? - callback() + CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) -> + callback() # ignore any errors from deleting directories + + clearProject: (project_id, user_id, callback = (error) ->) -> + logger.log project_id: project_id, "clearing project for user" + CompileManager.clearProject project_id, user_id, (error) -> + return callback(error) if error? + ProjectPersistenceManager.clearProjectFromCache project_id, (error) -> + return callback(error) if error? + callback() + + clearProjectFromCache: (project_id, callback = (error) ->) -> + logger.log project_id: project_id, "clearing project from cache" + UrlCache.clearProject project_id, (error) -> + return callback(error) if error? + ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) -> + return callback(error) if error? + callback() _clearProjectFromDatabase: (project_id, callback = (error) ->) -> db.Project.destroy(where: {project_id: project_id}) @@ -54,5 +63,4 @@ module.exports = ProjectPersistenceManager = callback null, projects.map((project) -> project.project_id) ).error callback - logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout" diff --git a/test/unit/coffee/CompileManagerTests.coffee b/test/unit/coffee/CompileManagerTests.coffee index 0ba62b9..611ed11 100644 --- a/test/unit/coffee/CompileManagerTests.coffee +++ b/test/unit/coffee/CompileManagerTests.coffee @@ -105,12 +105,12 @@ describe "CompileManager", -> @proc.stdout = new EventEmitter() @proc.stderr = new EventEmitter() @child_process.spawn = sinon.stub().returns(@proc) - @CompileManager.clearProject @project_id, @callback + @CompileManager.clearProject @project_id, @user_id, @callback @proc.emit "close", 0 it "should remove the project directory", -> @child_process.spawn - .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"]) + .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"]) .should.equal true it "should call the callback", -> @@ -124,13 +124,13 @@ describe "CompileManager", -> @proc.stdout = new EventEmitter() @proc.stderr = new EventEmitter() @child_process.spawn = sinon.stub().returns(@proc) - @CompileManager.clearProject @project_id, @callback + @CompileManager.clearProject @project_id, @user_id, @callback @proc.stderr.emit "data", @error = "oops" @proc.emit "close", 1 it "should remove the project directory", -> @child_process.spawn - .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"]) + .calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"]) .should.equal true it "should call the callback with an error from the stderr", -> @@ -138,7 +138,7 @@ describe "CompileManager", -> .calledWith(new Error()) .should.equal true - @callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id} failed: #{@error}" + @callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id}-#{@user_id} failed: #{@error}" describe "syncing", -> beforeEach -> diff --git a/test/unit/coffee/ProjectPersistenceManagerTests.coffee b/test/unit/coffee/ProjectPersistenceManagerTests.coffee index f8c78ef..69bfd4f 100644 --- a/test/unit/coffee/ProjectPersistenceManagerTests.coffee +++ b/test/unit/coffee/ProjectPersistenceManagerTests.coffee @@ -13,6 +13,7 @@ describe "ProjectPersistenceManager", -> "./db": @db = {} @callback = sinon.stub() @project_id = "project-id-123" + @user_id = "1234" describe "clearExpiredProjects", -> beforeEach -> @@ -21,12 +22,13 @@ describe "ProjectPersistenceManager", -> "project-id-2" ] @ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids) - @ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1) + @ProjectPersistenceManager.clearProjectFromCache = sinon.stub().callsArg(1) + @CompileManager.clearExpiredProjects = sinon.stub().callsArg(1) @ProjectPersistenceManager.clearExpiredProjects @callback it "should clear each expired project", -> for project_id in @project_ids - @ProjectPersistenceManager.clearProject + @ProjectPersistenceManager.clearProjectFromCache .calledWith(project_id) .should.equal true @@ -37,8 +39,8 @@ describe "ProjectPersistenceManager", -> beforeEach -> @ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1) @UrlCache.clearProject = sinon.stub().callsArg(1) - @CompileManager.clearProject = sinon.stub().callsArg(1) - @ProjectPersistenceManager.clearProject @project_id, @callback + @CompileManager.clearProject = sinon.stub().callsArg(2) + @ProjectPersistenceManager.clearProject @project_id, @user_id, @callback it "should clear the project from the database", -> @ProjectPersistenceManager._clearProjectFromDatabase @@ -52,7 +54,7 @@ describe "ProjectPersistenceManager", -> it "should clear the project compile folder", -> @CompileManager.clearProject - .calledWith(@project_id) + .calledWith(@project_id, @user_id) .should.equal true it "should call the callback", ->