archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Tweaks to management functions

Browse Source
This commit is contained in:
Francesco Siddi
2016-01-08 14:20:07 +01:00
parent f9f411366d
commit ff174439c3
2 changed files with 14 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pillar/application/__init__.py
View File

@@ -251,7 +251,9 @@ def update_file_name(item):
def _update_name(item, file_id): def _update_name(item, file_id):
files_collection = app.data.driver.db['files'] files_collection = app.data.driver.db['files']
f = files_collection.find_one({'_id': file_id}) f = files_collection.find_one({'_id': file_id})
if f and f['backend'] == 'gcs': status = item['properties']['status']
if f and f['backend'] == 'gcs' and status != 'processing':
# Process only files that are on GCS and that are not processing
try: try:
storage = GoogleCloudStorageBucket(str(item['project'])) storage = GoogleCloudStorageBucket(str(item['project']))
blob = storage.Get(f['file_path'], to_dict=False) blob = storage.Get(f['file_path'], to_dict=False)

23
pillar/manage.py
View File

@@ -463,7 +463,7 @@ def files_verify_project():
"""Verify for missing or conflicting node/file ids""" """Verify for missing or conflicting node/file ids"""
nodes_collection = app.data.driver.db['nodes'] nodes_collection = app.data.driver.db['nodes']
files_collection = app.data.driver.db['files'] files_collection = app.data.driver.db['files']
issues = dict(missing=[], conflicting=[]) issues = dict(missing=[], conflicting=[], processing=[])
def _parse_file(item, file_id): def _parse_file(item, file_id):
f = files_collection.find_one({'_id': file_id}) f = files_collection.find_one({'_id': file_id})
@@ -471,6 +471,9 @@ def files_verify_project():
if 'project' in item and 'project' in f: if 'project' in item and 'project' in f:
if item['project'] != f['project']: if item['project'] != f['project']:
issues['conflicting'].append(item['_id']) issues['conflicting'].append(item['_id'])
if 'status' in item['properties'] \
and item['properties']['status'] == 'processing':
issues['processing'].append(item['_id'])
else: else:
issues['missing'].append( issues['missing'].append(
"{0} missing {1}".format(item['_id'], file_id)) "{0} missing {1}".format(item['_id'], file_id))
@@ -483,17 +486,13 @@ def files_verify_project():
for f in item['properties']['files']: for f in item['properties']['files']:
_parse_file(item, f['file']) _parse_file(item, f['file'])
if issues: print "==="
print "The following issues were detected:" print "Issues detected:"
if issues['missing']: for k, v in issues.iteritems():
print "Missing:" print "{0}:".format(k)
for i in issues['missing']: for i in v:
print i print i
print "===" print "==="
if issues['conflicting']:
print "Conflicts:"
for i in issues['conflicting']:
print i
if __name__ == '__main__': if __name__ == '__main__':
manager.run() manager.run()