diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 41b4fed9..32f65e4d 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -251,11 +251,24 @@ def update_file_name(item): def _update_name(item, file_id): files_collection = app.data.driver.db['files'] f = files_collection.find_one({'_id': file_id}) - if f['backend'] == 'gcs': - storage = GoogleCloudStorageBucket(str(item['project'])) - blob = storage.Get(f['file_path'], to_dict=False) - storage.update_name(blob, "{0}.{1}".format( - item['name'], f['format'])) + if f and f['backend'] == 'gcs': + try: + storage = GoogleCloudStorageBucket(str(item['project'])) + blob = storage.Get(f['file_path'], to_dict=False) + storage.update_name(blob, "{0}.{1}".format( + item['name'], f['format'])) + try: + # Assign the same name to variations + for v in f['variations']: + blob = storage.Get(v['file_path'], to_dict=False) + storage.update_name(blob, "{0}-{1}.{2}".format( + item['name'], v['size'], v['format'])) + except KeyError: + pass + except AttributeError: + bugsnag.notify(Exception('Missing or conflicting ids detected'), + meta_data={'nodes_info': + {'node_id': item['_id'], 'file_id': file_id}}) # Currently we search for 'file' and 'files' keys in the object properties. # This could become a bit more flexible and realy on a true reference of the diff --git a/pillar/application/utils/gcs.py b/pillar/application/utils/gcs.py index 69316723..f93f0f6c 100644 --- a/pillar/application/utils/gcs.py +++ b/pillar/application/utils/gcs.py @@ -149,4 +149,3 @@ class GoogleCloudStorageBucket(object): """ blob.content_disposition = "attachment; filename={0}".format(name) blob.patch() - diff --git a/pillar/manage.py b/pillar/manage.py index 1290300f..01438738 100644 --- a/pillar/manage.py +++ b/pillar/manage.py @@ -446,5 +446,54 @@ def convert_assets_to_textures(project_id): nodes_collection.remove({'_id': n['_id']}) +@manager.command +def set_attachment_names(): + """Loop through all existing nodes and assign proper ContentDisposition + metadata to referenced files that are using GCS. + """ + from application import update_file_name + nodes_collection = app.data.driver.db['nodes'] + for n in nodes_collection.find(): + print "Updating node {0}".format(n['_id']) + update_file_name(n) + + +@manager.command +def files_verify_project(): + """Verify for missing or conflicting node/file ids""" + nodes_collection = app.data.driver.db['nodes'] + files_collection = app.data.driver.db['files'] + issues = dict(missing=[], conflicting=[]) + + def _parse_file(item, file_id): + f = files_collection.find_one({'_id': file_id}) + if f: + if 'project' in item and 'project' in f: + if item['project'] != f['project']: + issues['conflicting'].append(item['_id']) + else: + issues['missing'].append( + "{0} missing {1}".format(item['_id'], file_id)) + + for item in nodes_collection.find(): + print "Verifying node {0}".format(item['_id']) + if 'file' in item['properties']: + _parse_file(item, item['properties']['file']) + elif 'files' in item['properties']: + for f in item['properties']['files']: + _parse_file(item, f['file']) + + if issues: + print "The following issues were detected:" + if issues['missing']: + print "Missing:" + for i in issues['missing']: + print i + print "===" + if issues['conflicting']: + print "Conflicts:" + for i in issues['conflicting']: + print i + if __name__ == '__main__': manager.run()