From 2e5e040475518753606a1ff8945a949cac2ebac6 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 20 Mar 2020 13:37:58 +0000 Subject: [PATCH 1/3] limit clsi lifespan via health checks and PROCESS_LIFE_SPAN_LIMIT_MS --- app.js | 44 ++++++++++++++++--------------------- config/settings.defaults.js | 9 ++++---- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/app.js b/app.js index c03fcd8..ec67140 100644 --- a/app.js +++ b/app.js @@ -22,7 +22,6 @@ const ContentTypeMapper = require('./app/js/ContentTypeMapper') const Errors = require('./app/js/Errors') const Path = require('path') -const fs = require('fs') Metrics.open_sockets.monitor(logger) Metrics.memory.monitor(logger) @@ -208,23 +207,35 @@ const resCacher = { setContentType: 'application/json' } +const startupTime = Date.now() +const checkIfProcessIsTooOld = function() { + if ( + Settings.processLifespanLimitMs && + startupTime + Settings.processLifespanLimitMs < Date.now() + ) { + logger.log('shutting down, process is too old') + resCacher.send = function() {} + resCacher.statusCode = 500 + resCacher.body = { processToOld: true } + } +} + if (Settings.smokeTest) { - let runSmokeTest - ;(runSmokeTest = function() { + const runSmokeTest = function() { + checkIfProcessIsTooOld() logger.log('running smoke tests') smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))( {}, resCacher ) return setTimeout(runSmokeTest, 30 * 1000) - })() + } + runSmokeTest() } app.get('/health_check', function(req, res) { - res.contentType(resCacher != null ? resCacher.setContentType : undefined) - return res - .status(resCacher != null ? resCacher.code : undefined) - .send(resCacher != null ? resCacher.body : undefined) + res.contentType(resCacher.setContentType) + return res.status(resCacher.code).send(resCacher.body) }) app.get('/smoke_test_force', (req, res) => @@ -234,23 +245,6 @@ app.get('/smoke_test_force', (req, res) => ) ) -const profiler = require('v8-profiler-node8') -app.get('/profile', function(req, res) { - const time = parseInt(req.query.time || '1000') - profiler.startProfiling('test') - return setTimeout(function() { - const profile = profiler.stopProfiling('test') - return res.json(profile) - }, time) -}) - -app.get('/heapdump', (req, res) => - require('heapdump').writeSnapshot( - `/tmp/${Date.now()}.clsi.heapsnapshot`, - (err, filename) => res.send(filename) - ) -) - app.use(function(error, req, res, next) { if (error instanceof Errors.NotFoundError) { logger.warn({ err: error, url: req.url }, 'not found error') diff --git a/config/settings.defaults.js b/config/settings.defaults.js index b0fd0cb..fb7384b 100644 --- a/config/settings.defaults.js +++ b/config/settings.defaults.js @@ -22,6 +22,9 @@ module.exports = { compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb', + processLifespanLimitMs: + process.env.PROCESS_LIFE_SPAN_LIMIT_MS || 60 * 60 * 24 * 1000 * 2, + path: { compilesDir: Path.resolve(__dirname + '/../compiles'), clsiCacheDir: Path.resolve(__dirname + '/../cache'), @@ -65,8 +68,7 @@ if (process.env.DOCKER_RUNNER) { dockerRunner: process.env.DOCKER_RUNNER === 'true', docker: { image: - process.env.TEXLIVE_IMAGE || - 'quay.io/sharelatex/texlive-full:2017.1', + process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1', env: { HOME: '/tmp' }, @@ -93,8 +95,7 @@ if (process.env.DOCKER_RUNNER) { module.exports.path.synctexBaseDir = () => '/compile' - module.exports.path.sandboxedCompilesHostDir = - process.env.COMPILES_HOST_DIR + module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR module.exports.path.synctexBinHostPath = process.env.SYNCTEX_BIN_HOST_PATH } From 3513748f732e5eb7b0408c3055412c238497fb8b Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 9 Apr 2020 14:11:04 +0100 Subject: [PATCH 2/3] add variance into shutdown time to avoid stampeed --- app.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/app.js b/app.js index ec67140..f96d019 100644 --- a/app.js +++ b/app.js @@ -208,27 +208,34 @@ const resCacher = { } const startupTime = Date.now() -const checkIfProcessIsTooOld = function() { +const checkIfProcessIsTooOld = function(cont) { + if (typeof Settings.processLifespanLimitMs === 'string') { + Settings.processLifespanLimitMs = parseInt(Settings.processLifespanLimitMs) + Settings.processLifespanLimitMs += + Settings.processLifespanLimitMs * (Math.random() / 10) + } if ( Settings.processLifespanLimitMs && startupTime + Settings.processLifespanLimitMs < Date.now() ) { logger.log('shutting down, process is too old') resCacher.send = function() {} - resCacher.statusCode = 500 + resCacher.code = 500 resCacher.body = { processToOld: true } + } else { + cont() } } if (Settings.smokeTest) { const runSmokeTest = function() { - checkIfProcessIsTooOld() - logger.log('running smoke tests') - smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))( - {}, - resCacher - ) - return setTimeout(runSmokeTest, 30 * 1000) + checkIfProcessIsTooOld(function() { + logger.log('running smoke tests') + smokeTest.run( + require.resolve(__dirname + '/test/smoke/js/SmokeTests.js') + )({}, resCacher) + return setTimeout(runSmokeTest, 30 * 1000) + }) } runSmokeTest() } From 557dc47e30c4cbcc8017a761c37dfae0aada2e61 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 23 Apr 2020 11:32:33 +0100 Subject: [PATCH 3/3] cleanup the shutdown code a bit --- app.js | 19 +++++++++---------- config/settings.defaults.js | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/app.js b/app.js index f96d019..80b00ef 100644 --- a/app.js +++ b/app.js @@ -207,17 +207,16 @@ const resCacher = { setContentType: 'application/json' } -const startupTime = Date.now() +let shutdownTime +if (Settings.processLifespanLimitMs) { + Settings.processLifespanLimitMs += + Settings.processLifespanLimitMs * (Math.random() / 10) + shutdownTime = Date.now() + Settings.processLifespanLimitMs + logger.info('Lifespan limited to ', shutdownTime) +} + const checkIfProcessIsTooOld = function(cont) { - if (typeof Settings.processLifespanLimitMs === 'string') { - Settings.processLifespanLimitMs = parseInt(Settings.processLifespanLimitMs) - Settings.processLifespanLimitMs += - Settings.processLifespanLimitMs * (Math.random() / 10) - } - if ( - Settings.processLifespanLimitMs && - startupTime + Settings.processLifespanLimitMs < Date.now() - ) { + if (shutdownTime && shutdownTime < Date.now()) { logger.log('shutting down, process is too old') resCacher.send = function() {} resCacher.code = 500 diff --git a/config/settings.defaults.js b/config/settings.defaults.js index fb7384b..e5df062 100644 --- a/config/settings.defaults.js +++ b/config/settings.defaults.js @@ -23,7 +23,7 @@ module.exports = { compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb', processLifespanLimitMs: - process.env.PROCESS_LIFE_SPAN_LIMIT_MS || 60 * 60 * 24 * 1000 * 2, + parseInt(process.env.PROCESS_LIFE_SPAN_LIMIT_MS) || 60 * 60 * 24 * 1000 * 2, path: { compilesDir: Path.resolve(__dirname + '/../compiles'),