log any missing files
This commit is contained in:
@@ -5,7 +5,7 @@ spawn = require("child_process").spawn
|
|||||||
logger = require "logger-sharelatex"
|
logger = require "logger-sharelatex"
|
||||||
|
|
||||||
module.exports = OutputFileFinder =
|
module.exports = OutputFileFinder =
|
||||||
findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
|
findOutputFiles: (resources, directory, callback = (error, outputFiles, allFiles) ->) ->
|
||||||
incomingResources = {}
|
incomingResources = {}
|
||||||
for resource in resources
|
for resource in resources
|
||||||
incomingResources[resource.path] = true
|
incomingResources[resource.path] = true
|
||||||
@@ -16,7 +16,6 @@ module.exports = OutputFileFinder =
|
|||||||
if error?
|
if error?
|
||||||
logger.err err:error, "error finding all output files"
|
logger.err err:error, "error finding all output files"
|
||||||
return callback(error)
|
return callback(error)
|
||||||
jobs = []
|
|
||||||
outputFiles = []
|
outputFiles = []
|
||||||
for file in allFiles
|
for file in allFiles
|
||||||
if !incomingResources[file]
|
if !incomingResources[file]
|
||||||
@@ -24,7 +23,7 @@ module.exports = OutputFileFinder =
|
|||||||
path: file
|
path: file
|
||||||
type: file.match(/\.([^\.]+)$/)?[1]
|
type: file.match(/\.([^\.]+)$/)?[1]
|
||||||
}
|
}
|
||||||
callback null, outputFiles
|
callback null, outputFiles, allFiles
|
||||||
|
|
||||||
_getAllFiles: (directory, _callback = (error, fileList) ->) ->
|
_getAllFiles: (directory, _callback = (error, fileList) ->) ->
|
||||||
callback = (error, fileList) ->
|
callback = (error, fileList) ->
|
||||||
|
|||||||
@@ -49,3 +49,15 @@ module.exports = ResourceStateManager =
|
|||||||
else
|
else
|
||||||
resources = ({path: path} for path in resourceList)
|
resources = ({path: path} for path in resourceList)
|
||||||
callback(null, resources)
|
callback(null, resources)
|
||||||
|
|
||||||
|
checkResourceFiles: (resources, allFiles, directory, callback = (error) ->) ->
|
||||||
|
# check if any of the input files are not present in list of files
|
||||||
|
seenFile = {}
|
||||||
|
for file in allFiles
|
||||||
|
seenFile[file] = true
|
||||||
|
missingFiles = (resource.path for resource in resources when not seenFile[resource.path])
|
||||||
|
if missingFiles.length > 0
|
||||||
|
logger.err missingFiles:missingFiles, dir:directory, allFiles:allFiles, resources:resources, "missing input files for project"
|
||||||
|
return callback new Errors.FilesOutOfSyncError("resource files missing in incremental update")
|
||||||
|
else
|
||||||
|
callback()
|
||||||
|
|||||||
@@ -18,7 +18,9 @@ module.exports = ResourceWriter =
|
|||||||
logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
|
logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
|
||||||
ResourceStateManager.checkProjectStateHashMatches request.syncState, basePath, (error, resourceList) ->
|
ResourceStateManager.checkProjectStateHashMatches request.syncState, basePath, (error, resourceList) ->
|
||||||
return callback(error) if error?
|
return callback(error) if error?
|
||||||
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error) =>
|
ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) ->
|
||||||
|
return callback(error) if error?
|
||||||
|
ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) ->
|
||||||
return callback(error) if error?
|
return callback(error) if error?
|
||||||
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
|
ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
|
||||||
return callback(error) if error?
|
return callback(error) if error?
|
||||||
@@ -60,13 +62,13 @@ module.exports = ResourceWriter =
|
|||||||
else
|
else
|
||||||
return callback()
|
return callback()
|
||||||
|
|
||||||
_removeExtraneousFiles: (resources, basePath, _callback = (error) ->) ->
|
_removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) ->
|
||||||
timer = new Metrics.Timer("unlink-output-files")
|
timer = new Metrics.Timer("unlink-output-files")
|
||||||
callback = (error) ->
|
callback = (error, result...) ->
|
||||||
timer.done()
|
timer.done()
|
||||||
_callback(error)
|
_callback(error, result...)
|
||||||
|
|
||||||
OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles) ->
|
OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) ->
|
||||||
return callback(error) if error?
|
return callback(error) if error?
|
||||||
|
|
||||||
jobs = []
|
jobs = []
|
||||||
@@ -85,7 +87,9 @@ module.exports = ResourceWriter =
|
|||||||
if should_delete
|
if should_delete
|
||||||
jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
|
jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
|
||||||
|
|
||||||
async.series jobs, callback
|
async.series jobs, (error) ->
|
||||||
|
return callback(error) if error?
|
||||||
|
callback(null, outputFiles, allFiles)
|
||||||
|
|
||||||
_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
|
_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
|
||||||
fs.stat path, (error, stat) ->
|
fs.stat path, (error, stat) ->
|
||||||
|
|||||||
@@ -32,7 +32,6 @@ describe "ResourceWriter", ->
|
|||||||
]
|
]
|
||||||
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
|
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
|
||||||
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
|
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
|
||||||
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArg(2)
|
|
||||||
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
|
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
|
||||||
@ResourceWriter.syncResourcesToDisk({
|
@ResourceWriter.syncResourcesToDisk({
|
||||||
project_id: @project_id
|
project_id: @project_id
|
||||||
@@ -65,9 +64,10 @@ describe "ResourceWriter", ->
|
|||||||
"resource-1-mock"
|
"resource-1-mock"
|
||||||
]
|
]
|
||||||
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
|
@ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3)
|
||||||
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArg(2)
|
@ResourceWriter._removeExtraneousFiles = sinon.stub().callsArgWith(2, null, @outputFiles = [], @allFiles = [])
|
||||||
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArgWith(2, null, @resources)
|
@ResourceStateManager.checkProjectStateHashMatches = sinon.stub().callsArgWith(2, null, @resources)
|
||||||
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
|
@ResourceStateManager.saveProjectStateHash = sinon.stub().callsArg(3)
|
||||||
|
@ResourceStateManager.checkResourceFiles = sinon.stub().callsArg(3)
|
||||||
@ResourceWriter.syncResourcesToDisk({
|
@ResourceWriter.syncResourcesToDisk({
|
||||||
project_id: @project_id,
|
project_id: @project_id,
|
||||||
syncType: "incremental",
|
syncType: "incremental",
|
||||||
@@ -85,6 +85,11 @@ describe "ResourceWriter", ->
|
|||||||
.calledWith(@resources, @basePath)
|
.calledWith(@resources, @basePath)
|
||||||
.should.equal true
|
.should.equal true
|
||||||
|
|
||||||
|
it "should check each resource exists", ->
|
||||||
|
@ResourceStateManager.checkResourceFiles
|
||||||
|
.calledWith(@resources, @allFiles, @basePath)
|
||||||
|
.should.equal true
|
||||||
|
|
||||||
it "should write each resource to disk", ->
|
it "should write each resource to disk", ->
|
||||||
for resource in @resources
|
for resource in @resources
|
||||||
@ResourceWriter._writeResourceToDisk
|
@ResourceWriter._writeResourceToDisk
|
||||||
|
|||||||
Reference in New Issue
Block a user