log any missing files

This commit is contained in:
Brian Gough
2017-09-07 13:51:36 +01:00
parent 0fac2655f7
commit acab9d45a0
4 changed files with 33 additions and 13 deletions

View File

@@ -5,7 +5,7 @@ spawn = require("child_process").spawn
logger = require "logger-sharelatex" logger = require "logger-sharelatex"
module.exports = OutputFileFinder = module.exports = OutputFileFinder =
findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) -> findOutputFiles: (resources, directory, callback = (error, outputFiles, allFiles) ->) ->
incomingResources = {} incomingResources = {}
for resource in resources for resource in resources
incomingResources[resource.path] = true incomingResources[resource.path] = true
@@ -16,7 +16,6 @@ module.exports = OutputFileFinder =
if error? if error?
logger.err err:error, "error finding all output files" logger.err err:error, "error finding all output files"
return callback(error) return callback(error)
jobs = []
outputFiles = [] outputFiles = []
for file in allFiles for file in allFiles
if !incomingResources[file] if !incomingResources[file]
@@ -24,7 +23,7 @@ module.exports = OutputFileFinder =
path: file path: file
type: file.match(/\.([^\.]+)$/)?[1] type: file.match(/\.([^\.]+)$/)?[1]
} }
callback null, outputFiles callback null, outputFiles, allFiles
_getAllFiles: (directory, _callback = (error, fileList) ->) -> _getAllFiles: (directory, _callback = (error, fileList) ->) ->
callback = (error, fileList) -> callback = (error, fileList) ->

View File

@@ -49,3 +49,15 @@ module.exports = ResourceStateManager =
else else
resources = ({path: path} for path in resourceList) resources = ({path: path} for path in resourceList)
callback(null, resources) callback(null, resources)
checkResourceFiles: (resources, allFiles, directory, callback = (error) ->) ->
# check if any of the input files are not present in list of files
seenFile = {}
for file in allFiles
seenFile[file] = true
missingFiles = (resource.path for resource in resources when not seenFile[resource.path])
if missingFiles.length > 0
logger.err missingFiles:missingFiles, dir:directory, allFiles:allFiles, resources:resources, "missing input files for project"
return callback new Errors.FilesOutOfSyncError("resource files missing in incremental update")
else
callback()

View File

@@ -18,7 +18,9 @@ module.exports = ResourceWriter =
logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync" logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
ResourceStateManager.checkProjectStateHashMatches request.syncState, basePath, (error, resourceList) -> ResourceStateManager.checkProjectStateHashMatches request.syncState, basePath, (error, resourceList) ->
return callback(error) if error? return callback(error) if error?
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error) => ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) ->
return callback(error) if error?
ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) ->
return callback(error) if error? return callback(error) if error?
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) -> ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
return callback(error) if error? return callback(error) if error?
@@ -60,13 +62,13 @@ module.exports = ResourceWriter =
else else
return callback() return callback()
_removeExtraneousFiles: (resources, basePath, _callback = (error) ->) -> _removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) ->
timer = new Metrics.Timer("unlink-output-files") timer = new Metrics.Timer("unlink-output-files")
callback = (error) -> callback = (error, result...) ->
timer.done() timer.done()
_callback(error) _callback(error, result...)
OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles) -> OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) ->
return callback(error) if error? return callback(error) if error?
jobs = [] jobs = []
@@ -85,7 +87,9 @@ module.exports = ResourceWriter =
if should_delete if should_delete
jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
async.series jobs, callback async.series jobs, (error) ->
return callback(error) if error?
callback(null, outputFiles, allFiles)
_deleteFileIfNotDirectory: (path, callback = (error) ->) -> _deleteFileIfNotDirectory: (path, callback = (error) ->) ->
fs.stat path, (error, stat) -> fs.stat path, (error, stat) ->

View File

@@ -32,7 +32,6 @@ describe "ResourceWriter", ->
] ]
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3) @ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2) @ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArg(2)
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3) @ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
@ResourceWriter.syncResourcesToDisk({ @ResourceWriter.syncResourcesToDisk({
project_id: @project_id project_id: @project_id
@@ -65,9 +64,10 @@ describe "ResourceWriter", ->
"resource-1-mock" "resource-1-mock"
] ]
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3) @ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2) @ResourceWriter._removeExtraneousFiles = sinon.stub().callsArgWith(2, null, @outputFiles = [], @allFiles = [])
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArgWith(2, null, @resources) @ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArgWith(2, null, @resources)
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3) @ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
@ResourceStateManager.checkResourceFiles = sinon.stub().callsArg(3)
@ResourceWriter.syncResourcesToDisk({ @ResourceWriter.syncResourcesToDisk({
project_id: @project_id, project_id: @project_id,
syncType: "incremental", syncType: "incremental",
@@ -85,6 +85,11 @@ describe "ResourceWriter", ->
.calledWith(@resources, @basePath) .calledWith(@resources, @basePath)
.should.equal true .should.equal true
it "should check each resource exists", ->
@ResourceStateManager.checkResourceFiles
.calledWith(@resources, @allFiles, @basePath)
.should.equal true
it "should write each resource to disk", -> it "should write each resource to disk", ->
for resource in @resources for resource in @resources
@ResourceWriter._writeResourceToDisk @ResourceWriter._writeResourceToDisk