Merge pull request #162 from overleaf/ta-jpa-epipe-retry

[DockerRunner] retry container inspect on EPIPE
This commit is contained in:
Henry Oswald authored and GitHub committed 2020-05-19 11:15:25 +01:00
commit fec359afac
2 files changed
+83 -15

No files matched your search

+23 -13
View File
@@ -26,6 +26,7 @@ const LockManager = require('./DockerLockManager')
const fs = require('fs')
const Path = require('path')
const _ = require('underscore')
const metrics = require('metrics-sharelatex')
logger.info('using docker runner')
@@ -411,19 +412,28 @@ module.exports = DockerRunner = {
})
}
)
return container.inspect(function(error, stats) {
if ((error != null ? error.statusCode : undefined) === 404) {
return createAndStartContainer()
} else if (error != null) {
logger.err(
{ container_name: name, error },
'unable to inspect container to start'
)
return callback(error)
} else {
return startExistingContainer()
}
})
var inspectContainer = (isRetry) =>
container.inspect(function(error, stats) {
if ((error != null ? error.statusCode : undefined) === 404) {
return createAndStartContainer()
} else if (error != null) {
if (error.message.match(/EPIPE/)) {
if (!isRetry) {
metrics.inc('container-inspect-epipe-retry')
return inspectContainer(true)
}
metrics.inc('container-inspect-epipe-error')
}
logger.err(
{ container_name: name, error },
'unable to inspect container to start'
)
return callback(error)
} else {
return startExistingContainer()
}
})
inspectContainer(false)
},
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {