Merge pull request #180 from overleaf/bg-add-compile-groups

add compile groups support

app/js/CompileManager.js

@@ -199,7 +199,8 @@ module.exports = CompileManager = {
               timeout: request.timeout,
               image: request.imageName,
               flags: request.flags,
-              environment: env
+              environment: env,
+              compileGroup: request.compileGroup
             },
             function(error, output, stats, timings) {
               // request was for validation only
@@ -510,6 +511,7 @@ module.exports = CompileManager = {
     const directory = getCompileDir(project_id, user_id)
     const timeout = 60 * 1000 // increased to allow for large projects
     const compileName = getCompileName(project_id, user_id)
+    const compileGroup = 'synctex'
     CompileManager._checkFileExists(directory, 'output.synctex.gz', error => {
       if (error) {
         return callback(error)
@@ -521,6 +523,7 @@ module.exports = CompileManager = {
         Settings.clsi != null ? Settings.clsi.docker.image : undefined,
         timeout,
         {},
+        compileGroup,
         function(error, output) {
           if (error != null) {
             logger.err(
@@ -585,6 +588,7 @@ module.exports = CompileManager = {
     const compileDir = getCompileDir(project_id, user_id)
     const timeout = 60 * 1000
     const compileName = getCompileName(project_id, user_id)
+    const compileGroup = 'wordcount'
     return fse.ensureDir(compileDir, function(error) {
       if (error != null) {
         logger.err(
@@ -600,6 +604,7 @@ module.exports = CompileManager = {
         image,
         timeout,
         {},
+        compileGroup,
         function(error) {
           if (error != null) {
             return callback(error)

app/js/DockerRunner.js

@@ -44,7 +44,16 @@ module.exports = DockerRunner = {
   ERR_EXITED: new Error('exited'),
   ERR_TIMED_OUT: new Error('container timed out'),
 
-  run(project_id, command, directory, image, timeout, environment, callback) {
+  run(
+    project_id,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    callback
+  ) {
     let name
     if (callback == null) {
       callback = function(error, output) {}
@@ -87,7 +96,8 @@ module.exports = DockerRunner = {
       image,
       volumes,
       timeout,
-      environment
+      environment,
+      compileGroup
     )
     const fingerprint = DockerRunner._fingerprintContainer(options)
     options.name = name = `project-${project_id}-${fingerprint}`
@@ -223,7 +233,14 @@ module.exports = DockerRunner = {
     )
   },
 
-  _getContainerOptions(command, image, volumes, timeout, environment) {
+  _getContainerOptions(
+    command,
+    image,
+    volumes,
+    timeout,
+    environment,
+    compileGroup
+  ) {
     let m, year
     let key, value, hostVol, dockerVol
     const timeoutInSeconds = timeout / 1000
@@ -310,6 +327,23 @@ module.exports = DockerRunner = {
       options.HostConfig.Runtime = Settings.clsi.docker.runtime
     }
 
+    if (Settings.clsi.docker.Readonly) {
+      options.HostConfig.ReadonlyRootfs = true
+      options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' }
+    }
+
+    // Allow per-compile group overriding of individual settings
+    if (
+      Settings.clsi.docker.compileGroupConfig &&
+      Settings.clsi.docker.compileGroupConfig[compileGroup]
+    ) {
+      const override = Settings.clsi.docker.compileGroupConfig[compileGroup]
+      let key
+      for (key in override) {
+        _.set(options, key, override[key])
+      }
+    }
+
     return options
   },
 

app/js/LatexRunner.js

@@ -36,7 +36,8 @@ module.exports = LatexRunner = {
       timeout,
       image,
       environment,
-      flags
+      flags,
+      compileGroup
     } = options
     if (!compiler) {
       compiler = 'pdflatex'
@@ -46,7 +47,15 @@ module.exports = LatexRunner = {
     } // milliseconds
 
     logger.log(
-      { directory, compiler, timeout, mainFile, environment, flags },
+      {
+        directory,
+        compiler,
+        timeout,
+        mainFile,
+        environment,
+        flags,
+        compileGroup
+      },
       'starting compile'
     )
 
@@ -79,6 +88,7 @@ module.exports = LatexRunner = {
       image,
       timeout,
       environment,
+      compileGroup,
       function(error, output) {
         delete ProcessTable[id]
         if (error != null) {

app/js/LocalCommandRunner.js

@@ -20,7 +20,16 @@ const logger = require('logger-sharelatex')
 logger.info('using standard command runner')
 
 module.exports = CommandRunner = {
-  run(project_id, command, directory, image, timeout, environment, callback) {
+  run(
+    project_id,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    callback
+  ) {
     let key, value
     if (callback == null) {
       callback = function(error) {}

app/js/RequestParser.js

@@ -74,7 +74,17 @@ module.exports = RequestParser = {
         default: [],
         type: 'object'
       })
-
+      if (settings.allowedCompileGroups) {
+        response.compileGroup = this._parseAttribute(
+          'compileGroup',
+          compile.options.compileGroup,
+          {
+            validValues: settings.allowedCompileGroups,
+            default: '',
+            type: 'string'
+          }
+        )
+      }
       // The syncType specifies whether the request contains all
       // resources (full) or only those resources to be updated
       // in-place (incremental).

config/settings.defaults.js

@@ -63,6 +63,17 @@ module.exports = {
   }
 }
 
+if (process.env.ALLOWED_COMPILE_GROUPS) {
+  try {
+    module.exports.allowedCompileGroups = process.env.ALLOWED_COMPILE_GROUPS.split(
+      ' '
+    )
+  } catch (error) {
+    console.error(error, 'could not apply allowed compile group setting')
+    process.exit(1)
+  }
+}
+
 if (process.env.DOCKER_RUNNER) {
   let seccompProfilePath
   module.exports.clsi = {
@@ -82,6 +93,29 @@ if (process.env.DOCKER_RUNNER) {
     checkProjectsIntervalMs: 10 * 60 * 1000
   }
 
+  try {
+    // Override individual docker settings using path-based keys, e.g.:
+    // compileGroupDockerConfigs = {
+    //    priority: { 'HostConfig.CpuShares': 100 }
+    //    beta: { 'dotted.path.here', 'value'}
+    // }
+    const compileGroupConfig = JSON.parse(
+      process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}'
+    )
+    // Automatically clean up wordcount and synctex containers
+    const defaultCompileGroupConfig = {
+      wordcount: { 'HostConfig.AutoRemove': true },
+      synctex: { 'HostConfig.AutoRemove': true }
+    }
+    module.exports.clsi.docker.compileGroupConfig = Object.assign(
+      defaultCompileGroupConfig,
+      compileGroupConfig
+    )
+  } catch (error) {
+    console.error(error, 'could not apply compile group docker configs')
+    process.exit(1)
+  }
+
   try {
     seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
     module.exports.clsi.docker.seccomp_profile = JSON.stringify(

test/unit/js/CompileManagerTests.js

@@ -160,7 +160,8 @@ describe('CompileManager', function() {
         compiler: (this.compiler = 'pdflatex'),
         timeout: (this.timeout = 42000),
         imageName: (this.image = 'example.com/image'),
-        flags: (this.flags = ['-file-line-error'])
+        flags: (this.flags = ['-file-line-error']),
+        compileGroup: (this.compileGroup = 'compile-group')
       }
       this.env = {}
       this.Settings.compileDir = 'compiles'
@@ -199,7 +200,8 @@ describe('CompileManager', function() {
             timeout: this.timeout,
             image: this.image,
             flags: this.flags,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
@@ -253,7 +255,8 @@ describe('CompileManager', function() {
               CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16',
               CHKTEX_EXIT_ON_ERROR: 1,
               CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000'
-            }
+            },
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
@@ -275,7 +278,8 @@ describe('CompileManager', function() {
             timeout: this.timeout,
             image: this.image,
             flags: this.flags,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           })
           .should.equal(true)
       })
@@ -384,7 +388,7 @@ describe('CompileManager', function() {
         this.stdout = `NODE\t${this.page}\t${this.h}\t${this.v}\t${this.width}\t${this.height}\n`
         this.CommandRunner.run = sinon
           .stub()
-          .callsArgWith(6, null, { stdout: this.stdout })
+          .callsArgWith(7, null, { stdout: this.stdout })
         return this.CompileManager.syncFromCode(
           this.project_id,
           this.user_id,
@@ -443,7 +447,7 @@ describe('CompileManager', function() {
         this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}\t${this.line}\t${this.column}\n`
         this.CommandRunner.run = sinon
           .stub()
-          .callsArgWith(6, null, { stdout: this.stdout })
+          .callsArgWith(7, null, { stdout: this.stdout })
         return this.CompileManager.syncFromPdf(
           this.project_id,
           this.user_id,
@@ -485,7 +489,7 @@ describe('CompileManager', function() {
 
   return describe('wordcount', function() {
     beforeEach(function() {
-      this.CommandRunner.run = sinon.stub().callsArg(6)
+      this.CommandRunner.run = sinon.stub().callsArg(7)
       this.fs.readFile = sinon
         .stub()
         .callsArgWith(

test/unit/js/DockerRunnerTests.js

@@ -86,6 +86,7 @@ describe('DockerRunner', function() {
     this.project_id = 'project-id-123'
     this.volumes = { '/local/compile/directory': '/compile' }
     this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
+    this.compileGroup = 'compile-group'
     return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
   })
 
@@ -122,6 +123,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           (err, output) => {
             this.callback(err, output)
             return done()
@@ -171,6 +173,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -219,6 +222,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -252,6 +256,7 @@ describe('DockerRunner', function() {
           null,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -281,6 +286,7 @@ describe('DockerRunner', function() {
           this.image,
           this.timeout,
           this.env,
+          this.compileGroup,
           this.callback
         )
       })
@@ -292,6 +298,64 @@ describe('DockerRunner', function() {
     })
   })
 
+  describe('run with _getOptions', function() {
+    beforeEach(function(done) {
+      // this.DockerRunner._getContainerOptions = sinon
+      //   .stub()
+      //   .returns((this.options = { mockoptions: 'foo' }))
+      this.DockerRunner._fingerprintContainer = sinon
+        .stub()
+        .returns((this.fingerprint = 'fingerprint'))
+
+      this.name = `project-${this.project_id}-${this.fingerprint}`
+
+      this.command = ['mock', 'command', '--outdir=$COMPILE_DIR']
+      this.command_with_dir = ['mock', 'command', '--outdir=/compile']
+      this.timeout = 42000
+      return done()
+    })
+
+    describe('when a compile group config is set', function() {
+      beforeEach(function() {
+        this.Settings.clsi.docker.compileGroupConfig = {
+          'compile-group': {
+            'HostConfig.newProperty': 'new-property'
+          },
+          'other-group': { otherProperty: 'other-property' }
+        }
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+        return this.DockerRunner.run(
+          this.project_id,
+          this.command,
+          this.directory,
+          this.image,
+          this.timeout,
+          this.env,
+          this.compileGroup,
+          this.callback
+        )
+      })
+
+      it('should set the docker options for the compile group', function() {
+        const options = this.DockerRunner._runAndWaitForContainer.lastCall
+          .args[0]
+        return expect(options.HostConfig).to.deep.include({
+          Binds: ['/local/compile/directory:/compile:rw'],
+          LogConfig: { Type: 'none', Config: {} },
+          CapDrop: 'ALL',
+          SecurityOpt: ['no-new-privileges'],
+          newProperty: 'new-property'
+        })
+      })
+
+      return it('should call the callback', function() {
+        return this.callback.calledWith(null, this.output).should.equal(true)
+      })
+    })
+  })
+
   describe('_runAndWaitForContainer', function() {
     beforeEach(function() {
       this.options = { mockoptions: 'foo', name: (this.name = 'mock-name') }

test/unit/js/LatexRunnerTests.js

@@ -48,6 +48,7 @@ describe('LatexRunner', function() {
     this.mainFile = 'main-file.tex'
     this.compiler = 'pdflatex'
     this.image = 'example.com/image'
+    this.compileGroup = 'compile-group'
     this.callback = sinon.stub()
     this.project_id = 'project-id-123'
     return (this.env = { foo: '123' })
@@ -55,7 +56,7 @@ describe('LatexRunner', function() {
 
   return describe('runLatex', function() {
     beforeEach(function() {
-      return (this.CommandRunner.run = sinon.stub().callsArgWith(6, null, {
+      return (this.CommandRunner.run = sinon.stub().callsArgWith(7, null, {
         stdout: 'this is stdout',
         stderr: 'this is stderr'
       }))
@@ -71,7 +72,8 @@ describe('LatexRunner', function() {
             compiler: this.compiler,
             timeout: (this.timeout = 42000),
             image: this.image,
-            environment: this.env
+            environment: this.env,
+            compileGroup: this.compileGroup
           },
           this.callback
         )
@@ -85,7 +87,8 @@ describe('LatexRunner', function() {
             this.directory,
             this.image,
             this.timeout,
-            this.env
+            this.env,
+            this.compileGroup
           )
           .should.equal(true)
       })