From 627bed428e2ac7abdcedfcf7a389fa0742099d68 Mon Sep 17 00:00:00 2001
From: Henry Oswald <henry.oswald@sharelatex.com>
Date: Mon, 30 Jul 2018 16:22:04 +0100
Subject: [PATCH] added a queue with 1 concurency to db queries

---
 app/coffee/ProjectPersistenceManager.coffee |  3 +
 app/coffee/UrlCache.coffee                  | 68 +++++++++++++--------
 2 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/app/coffee/ProjectPersistenceManager.coffee b/app/coffee/ProjectPersistenceManager.coffee
index 403043f..17bead2 100644
--- a/app/coffee/ProjectPersistenceManager.coffee
+++ b/app/coffee/ProjectPersistenceManager.coffee
@@ -11,6 +11,7 @@ module.exports = ProjectPersistenceManager =
 	EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5
 
 	markProjectAsJustAccessed: (project_id, callback = (error) ->) ->
+		console.log("markProjectAsJustAccessed")
 		db.Project.findOrCreate(where: {project_id: project_id})
 			.spread(
 				(project, created) ->
@@ -53,11 +54,13 @@ module.exports = ProjectPersistenceManager =
 				callback()
 
 	_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
+		console.log("_clearProjectFromDatabase")
 		db.Project.destroy(where: {project_id: project_id})
 			.then(() -> callback())
 			.error callback
 
 	_findExpiredProjectIds: (callback = (error, project_ids) ->) ->
+		console.log("_findExpiredProjectIds")
 		db.Project.findAll(where: ["lastAccessed < ?", new Date(Date.now() - ProjectPersistenceManager.EXPIRY_TIMEOUT)])
 			.then((projects) ->
 				callback null, projects.map((project) -> project.project_id)
diff --git a/app/coffee/UrlCache.coffee b/app/coffee/UrlCache.coffee
index 06e6012..e5fc81c 100644
--- a/app/coffee/UrlCache.coffee
+++ b/app/coffee/UrlCache.coffee
@@ -6,6 +6,15 @@ fs = require("fs")
 logger = require "logger-sharelatex"
 async = require "async"
 
+queue = async.queue((task, cb)->
+	console.log("running task")
+	task(cb)
+, 1)
+
+console.log("hi there queue")
+queue.drain = ()->
+    console.log('HI all items have been processed')
+
 module.exports = UrlCache =
 	downloadUrlToFile: (project_id, url, destPath, lastModified, callback = (error) ->) ->
 		UrlCache._ensureUrlIsInCache project_id, url, lastModified, (error, pathToCachedUrl) =>
@@ -51,8 +60,9 @@ module.exports = UrlCache =
 	_doesUrlNeedDownloading: (project_id, url, lastModified, callback = (error, needsDownloading) ->) ->
 		if !lastModified?
 			return callback null, true
-
+		console.log "about to get _findUrlDetails"
 		UrlCache._findUrlDetails project_id, url, (error, urlDetails) ->
+			console.log error, urlDetails, "_findUrlDetails result"
 			return callback(error) if error?
 			if !urlDetails? or !urlDetails.lastModified? or urlDetails.lastModified.getTime() < lastModified.getTime()
 				return callback null, true
@@ -94,36 +104,44 @@ module.exports = UrlCache =
 				return callback()
 
 	_findUrlDetails: (project_id, url, callback = (error, urlDetails) ->) ->
-		console.log("_findUrlDetails")
-		db.UrlCache.find(where: { url: url, project_id: project_id })
-			.then((urlDetails) -> callback null, urlDetails)
-			.error callback
+		job = (cb)->
+			db.UrlCache.find(where: { url: url, project_id: project_id })
+				.then((urlDetails) -> cb null, urlDetails)
+				.error cb
+		queue.push job, callback
 
 	_updateOrCreateUrlDetails: (project_id, url, lastModified, callback = (error) ->) ->
-		console.log("_updateOrCreateUrlDetails")
-		db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
-			.spread(
-				(urlDetails, created) ->
-					urlDetails.updateAttributes(lastModified: lastModified)
-						.then(() -> callback())
-						.error(callback)
-			)
-			.error callback
+		job = (cb)->
+			console.log("_updateOrCreateUrlDetails")
+			db.UrlCache.findOrCreate(where: {url: url, project_id: project_id})
+				.spread(
+					(urlDetails, created) ->
+						urlDetails.updateAttributes(lastModified: lastModified)
+							.then(() -> cb())
+							.error(cb)
+				)
+				.error cb
+		queue.push(job, callback)
 
 	_clearUrlDetails: (project_id, url, callback = (error) ->) ->
-		console.log("_clearUrlDetails")
-		db.UrlCache.destroy(where: {url: url, project_id: project_id})
-			.then(() -> callback null)
-			.error callback
+		job = (cb)->
+			console.log("_clearUrlDetails")
+			db.UrlCache.destroy(where: {url: url, project_id: project_id})
+				.then(() -> cb null)
+				.error cb
+		queue.push(job, callback)
+
 
 	_findAllUrlsInProject: (project_id, callback = (error, urls) ->) ->
-		console.log("_findAllUrlsInProject")
-		db.UrlCache.findAll(where: { project_id: project_id })
-			.then(
-				(urlEntries) ->
-					callback null, urlEntries.map((entry) -> entry.url)
-			)
-			.error callback
+		job = (cb)->
+			console.log("_findAllUrlsInProject")
+			db.UrlCache.findAll(where: { project_id: project_id })
+				.then(
+					(urlEntries) ->
+						cb null, urlEntries.map((entry) -> entry.url)
+				)
+				.error cb
+		queue.push(job, callback)