From 5b5fd2f5dff3c64c2adb83f68be0d6371cd35b3a Mon Sep 17 00:00:00 2001
From: Brian Gough <brian.gough@overleaf.com>
Date: Thu, 7 May 2020 10:30:14 +0100
Subject: [PATCH] set encoding when reading from streams

using .toString() works most of the time but can lead to utf8 characters being
broken across chunk boundaries.

https://nodejs.org/api/stream.html#stream_readable_setencoding_encoding
---
 app/js/CompileManager.js              | 2 +-
 app/js/LocalCommandRunner.js          | 2 +-
 app/js/OutputFileFinder.js            | 2 +-
 app/js/OutputFileOptimiser.js         | 2 +-
 test/unit/js/CompileManagerTests.js   | 2 ++
 test/unit/js/OutputFileFinderTests.js | 1 +
 6 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/app/js/CompileManager.js b/app/js/CompileManager.js
index 3bf54bc..dd62f43 100644
--- a/app/js/CompileManager.js
+++ b/app/js/CompileManager.js
@@ -334,7 +334,7 @@ module.exports = CompileManager = {
       proc.on('error', callback)
 
       let stderr = ''
-      proc.stderr.on('data', chunk => (stderr += chunk.toString()))
+      proc.stderr.setEncoding('utf8').on('data', chunk => (stderr += chunk))
 
       return proc.on('close', function(code) {
         if (code === 0) {
diff --git a/app/js/LocalCommandRunner.js b/app/js/LocalCommandRunner.js
index 61ecd88..ccaf507 100644
--- a/app/js/LocalCommandRunner.js
+++ b/app/js/LocalCommandRunner.js
@@ -46,7 +46,7 @@ module.exports = CommandRunner = {
     const proc = spawn(command[0], command.slice(1), { cwd: directory, env })
 
     let stdout = ''
-    proc.stdout.on('data', data => (stdout += data))
+    proc.stdout.setEncoding('utf8').on('data', data => (stdout += data))
 
     proc.on('error', function(err) {
       logger.err(
diff --git a/app/js/OutputFileFinder.js b/app/js/OutputFileFinder.js
index 50012b5..8319978 100644
--- a/app/js/OutputFileFinder.js
+++ b/app/js/OutputFileFinder.js
@@ -87,7 +87,7 @@ module.exports = OutputFileFinder = {
 
     const proc = spawn('find', args)
     let stdout = ''
-    proc.stdout.on('data', chunk => (stdout += chunk.toString()))
+    proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
     proc.on('error', callback)
     return proc.on('close', function(code) {
       if (code !== 0) {
diff --git a/app/js/OutputFileOptimiser.js b/app/js/OutputFileOptimiser.js
index c0b8cc1..41ba7b4 100644
--- a/app/js/OutputFileOptimiser.js
+++ b/app/js/OutputFileOptimiser.js
@@ -78,7 +78,7 @@ module.exports = OutputFileOptimiser = {
     const timer = new Metrics.Timer('qpdf')
     const proc = spawn('qpdf', args)
     let stdout = ''
-    proc.stdout.on('data', chunk => (stdout += chunk.toString()))
+    proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
     callback = _.once(callback) // avoid double call back for error and close event
     proc.on('error', function(err) {
       logger.warn({ err, args }, 'qpdf failed')
diff --git a/test/unit/js/CompileManagerTests.js b/test/unit/js/CompileManagerTests.js
index 180f6f3..74a0a47 100644
--- a/test/unit/js/CompileManagerTests.js
+++ b/test/unit/js/CompileManagerTests.js
@@ -294,6 +294,7 @@ describe('CompileManager', function() {
         this.proc = new EventEmitter()
         this.proc.stdout = new EventEmitter()
         this.proc.stderr = new EventEmitter()
+        this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr)
         this.child_process.spawn = sinon.stub().returns(this.proc)
         this.CompileManager.clearProject(
           this.project_id,
@@ -328,6 +329,7 @@ describe('CompileManager', function() {
         this.proc = new EventEmitter()
         this.proc.stdout = new EventEmitter()
         this.proc.stderr = new EventEmitter()
+        this.proc.stderr.setEncoding = sinon.stub().returns(this.proc.stderr)
         this.child_process.spawn = sinon.stub().returns(this.proc)
         this.CompileManager.clearProject(
           this.project_id,
diff --git a/test/unit/js/OutputFileFinderTests.js b/test/unit/js/OutputFileFinderTests.js
index e5f9904..ee591b4 100644
--- a/test/unit/js/OutputFileFinderTests.js
+++ b/test/unit/js/OutputFileFinderTests.js
@@ -70,6 +70,7 @@ describe('OutputFileFinder', function() {
     beforeEach(function() {
       this.proc = new EventEmitter()
       this.proc.stdout = new EventEmitter()
+      this.proc.stdout.setEncoding = sinon.stub().returns(this.proc.stdout)
       this.spawn.returns(this.proc)
       this.directory = '/base/dir'
       return this.OutputFileFinder._getAllFiles(this.directory, this.callback)