Management functions

- set_attachment_names assigns to GCS blobs names based on the node
they belong to
- files_verify_project reports conflicts in file ids

pillar/application/__init__.py

@@ -251,11 +251,24 @@ def update_file_name(item):
     def _update_name(item, file_id):
         files_collection = app.data.driver.db['files']
         f = files_collection.find_one({'_id': file_id})
-        if f['backend'] == 'gcs':
-            storage = GoogleCloudStorageBucket(str(item['project']))
-            blob = storage.Get(f['file_path'], to_dict=False)
-            storage.update_name(blob, "{0}.{1}".format(
-                item['name'], f['format']))
+        if f and f['backend'] == 'gcs':
+            try:
+                storage = GoogleCloudStorageBucket(str(item['project']))
+                blob = storage.Get(f['file_path'], to_dict=False)
+                storage.update_name(blob, "{0}.{1}".format(
+                    item['name'], f['format']))
+                try:
+                    # Assign the same name to variations
+                    for v in f['variations']:
+                        blob = storage.Get(v['file_path'], to_dict=False)
+                        storage.update_name(blob, "{0}-{1}.{2}".format(
+                            item['name'], v['size'], v['format']))
+                except KeyError:
+                    pass
+            except AttributeError:
+                bugsnag.notify(Exception('Missing or conflicting ids detected'),
+                    meta_data={'nodes_info':
+                        {'node_id': item['_id'], 'file_id': file_id}})
 
     # Currently we search for 'file' and 'files' keys in the object properties.
     # This could become a bit more flexible and realy on a true reference of the

pillar/application/utils/gcs.py

@@ -149,4 +149,3 @@ class GoogleCloudStorageBucket(object):
         """
         blob.content_disposition = "attachment; filename={0}".format(name)
         blob.patch()
-

pillar/manage.py

@@ -446,5 +446,54 @@ def convert_assets_to_textures(project_id):
                 nodes_collection.remove({'_id': n['_id']})
 
 
+@manager.command
+def set_attachment_names():
+    """Loop through all existing nodes and assign proper ContentDisposition
+    metadata to referenced files that are using GCS.
+    """
+    from application import update_file_name
+    nodes_collection = app.data.driver.db['nodes']
+    for n in nodes_collection.find():
+        print "Updating node {0}".format(n['_id'])
+        update_file_name(n)
+
+
+@manager.command
+def files_verify_project():
+    """Verify for missing or conflicting node/file ids"""
+    nodes_collection = app.data.driver.db['nodes']
+    files_collection = app.data.driver.db['files']
+    issues = dict(missing=[], conflicting=[])
+
+    def _parse_file(item, file_id):
+        f = files_collection.find_one({'_id': file_id})
+        if f:
+            if 'project' in item and 'project' in f:
+                if item['project'] != f['project']:
+                    issues['conflicting'].append(item['_id'])
+        else:
+            issues['missing'].append(
+                "{0} missing {1}".format(item['_id'], file_id))
+
+    for item in nodes_collection.find():
+        print "Verifying node {0}".format(item['_id'])
+        if 'file' in item['properties']:
+            _parse_file(item, item['properties']['file'])
+        elif 'files' in item['properties']:
+            for f in item['properties']['files']:
+                _parse_file(item, f['file'])
+
+    if issues:
+        print "The following issues were detected:"
+        if issues['missing']:
+            print "Missing:"
+            for i in issues['missing']:
+                print i
+            print "==="
+        if issues['conflicting']:
+            print "Conflicts:"
+            for i in issues['conflicting']:
+                print i
+
 if __name__ == '__main__':
     manager.run()