From 6464aefdb4dd4d7ba358f9791a550ca59d53a009 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 3 Jul 2018 16:41:34 +0100 Subject: [PATCH] added filestoreDomainOveride --- app/coffee/UrlFetcher.coffee | 6 +++++ config/settings.defaults.coffee | 1 + test/unit/coffee/UrlFetcherTests.coffee | 35 +++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/app/coffee/UrlFetcher.coffee b/app/coffee/UrlFetcher.coffee index 201306c..58645f0 100644 --- a/app/coffee/UrlFetcher.coffee +++ b/app/coffee/UrlFetcher.coffee @@ -1,6 +1,8 @@ request = require("request").defaults(jar: false) fs = require("fs") logger = require "logger-sharelatex" +settings = require("settings-sharelatex") +URL = require('url'); oneMinute = 60 * 1000 @@ -11,6 +13,9 @@ module.exports = UrlFetcher = _callback(error) _callback = () -> + if settings.filestoreDomainOveride? + p = URL.parse(url).path + url = "#{settings.filestoreDomainOveride}#{p}" timeoutHandler = setTimeout () -> timeoutHandler = null logger.error url:url, filePath: filePath, "Timed out downloading file to cache" @@ -31,6 +36,7 @@ module.exports = UrlFetcher = logger.log url:url, filePath: filePath, "finished downloading file into cache" urlStream.on "response", (res) -> + console.log if res.statusCode >= 200 and res.statusCode < 300 fileStream = fs.createWriteStream(filePath) diff --git a/config/settings.defaults.coffee b/config/settings.defaults.coffee index a64941f..081ded8 100644 --- a/config/settings.defaults.coffee +++ b/config/settings.defaults.coffee @@ -32,6 +32,7 @@ module.exports = smokeTest: process.env["SMOKE_TEST"] or false project_cache_length_ms: 1000 * 60 * 60 * 24 parallelFileDownloads:1 + filestoreDomainOveride: process.env["FILESTORE_DOMAIN_OVERRIDE"] if process.env["DOCKER_RUNNER"] diff --git a/test/unit/coffee/UrlFetcherTests.coffee b/test/unit/coffee/UrlFetcherTests.coffee index 4bd161b..e91720e 100644 --- a/test/unit/coffee/UrlFetcherTests.coffee +++ b/test/unit/coffee/UrlFetcherTests.coffee @@ -7,16 +7,47 @@ EventEmitter = require("events").EventEmitter describe "UrlFetcher", -> beforeEach -> @callback = sinon.stub() - @url = "www.example.com/file" + @url = "https://www.example.com/file/here?query=string" @UrlFetcher = SandboxedModule.require modulePath, requires: request: defaults: @defaults = sinon.stub().returns(@request = {}) fs: @fs = {} "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } + "settings-sharelatex": @settings = {} it "should turn off the cookie jar in request", -> @defaults.calledWith(jar: false) .should.equal true - + + describe "rewrite url domain if filestoreDomainOveride is set", -> + beforeEach -> + @path = "/path/to/file/on/disk" + @request.get = sinon.stub().returns(@urlStream = new EventEmitter) + @urlStream.pipe = sinon.stub() + @urlStream.pause = sinon.stub() + @urlStream.resume = sinon.stub() + @fs.createWriteStream = sinon.stub().returns(@fileStream = new EventEmitter) + @fs.unlink = (file, callback) -> callback() + + it "should use the normal domain when override not set", (done)-> + @UrlFetcher.pipeUrlToFile @url, @path, => + @request.get.args[0][0].url.should.equal @url + done() + @res = statusCode: 200 + @urlStream.emit "response", @res + @urlStream.emit "end" + @fileStream.emit "finish" + + + it "should use override domain when filestoreDomainOveride is set", (done)-> + @settings.filestoreDomainOveride = "192.11.11.11" + @UrlFetcher.pipeUrlToFile @url, @path, => + @request.get.args[0][0].url.should.equal "192.11.11.11/file/here?query=string" + done() + @res = statusCode: 200 + @urlStream.emit "response", @res + @urlStream.emit "end" + @fileStream.emit "finish" + describe "pipeUrlToFile", -> beforeEach (done)-> @path = "/path/to/file/on/disk"