PEP8 formatting for the entire project.

Conflicts:
	pillar/application/modules/file_storage.py

pillar/application/__init__.py

@@ -15,7 +15,6 @@ from eve import Eve
 from eve.auth import TokenAuth
 from eve.io.mongo import Validator
 
-
 RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
 
 
@@ -63,7 +62,7 @@ class ValidateCustomFields(Validator):
         project = projects_collection.find_one(lookup)
         node_type = next(
             (item for item in project['node_types'] if item.get('name') \
-                and item['name'] == self.document['node_type']), None)
+             and item['name'] == self.document['node_type']), None)
         try:
             value = self.convert_properties(value, node_type['dyn_schema'])
         except Exception, e:
@@ -82,6 +81,7 @@ class ValidateCustomFields(Validator):
             self._error(
                 field, "Error validating properties")
 
+
 # We specify a settings.py file because when running on wsgi we can't detect it
 # automatically. The default path (which works in Docker) can be overridden with
 # an env variable.
@@ -90,6 +90,7 @@ settings_path = os.environ.get(
 app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth)
 
 import config
+
 app.config.from_object(config.Deployment)
 
 # Configure logging
@@ -104,8 +105,8 @@ log.setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO)
 log.info('Pillar starting')
 
 bugsnag.configure(
-  api_key=app.config['BUGSNAG_API_KEY'],
+    api_key=app.config['BUGSNAG_API_KEY'],
-  project_root="/data/git/pillar/pillar",
+    project_root="/data/git/pillar/pillar",
 )
 handle_exceptions(app)
 
@@ -125,8 +126,8 @@ except KeyError:
 # Algolia search
 if 'ALGOLIA_USER' in app.config:
     client = algoliasearch.Client(
-            app.config['ALGOLIA_USER'],
+        app.config['ALGOLIA_USER'],
-            app.config['ALGOLIA_API_KEY'])
+        app.config['ALGOLIA_API_KEY'])
     algolia_index_users = client.init_index(app.config['ALGOLIA_INDEX_USERS'])
     algolia_index_nodes = client.init_index(app.config['ALGOLIA_INDEX_NODES'])
 else:
@@ -157,24 +158,29 @@ def before_returning_item_permissions(response):
     validate_token()
     check_permissions(response, 'GET', append_allowed_methods=True)
 
+
 def before_returning_resource_permissions(response):
     for item in response['_items']:
         validate_token()
         check_permissions(item, 'GET', append_allowed_methods=True)
 
+
 def before_replacing_node(item, original):
     check_permissions(original, 'PUT')
     update_file_name(item)
 
+
 def after_replacing_node(item, original):
     """Push an update to the Algolia index when a node item is updated"""
     algolia_index_node_save(item)
 
+
 def before_inserting_nodes(items):
     """Before inserting a node in the collection we check if the user is allowed
     and we append the project id to it.
     """
     nodes_collection = app.data.driver.db['nodes']
+
     def find_parent_project(node):
         """Recursive function that finds the ultimate parent of a node."""
         if node and 'parent' in node:
@@ -184,6 +190,7 @@ def before_inserting_nodes(items):
             return node
         else:
             return None
+
     for item in items:
         check_permissions(item, 'POST')
         if 'parent' in item and 'project' not in item:
@@ -223,7 +230,8 @@ def after_inserting_nodes(items):
             item['_id'],
             'node',
             context_object_id
-            )
+        )
+
 
 def item_parse_attachments(response):
     """Before returning a response, check if the 'attachments' property is
@@ -252,7 +260,7 @@ def item_parse_attachments(response):
                     size = f['size'] if 'size' in f else 'l'
                     # Get the correc variation from the file
                     thumbnail = next((item for item in f['variations'] if
-                        item['size'] == size), None)
+                                      item['size'] == size), None)
                     l = generate_link(f['backend'], thumbnail['file_path'], str(f['project']))
                     # Build Markdown img string
                     l = '![{0}]({1} "{2}")'.format(slug, l, f['name'])
@@ -266,10 +274,12 @@ def item_parse_attachments(response):
                 else:
                     response[field_name_path[0]] = field_content
 
+
 def resource_parse_attachments(response):
     for item in response['_items']:
         item_parse_attachments(item)
 
+
 def project_node_type_has_method(response):
     """Check for a specific request arg, and check generate the allowed_methods
     list for the required node_type.
@@ -283,7 +293,7 @@ def project_node_type_has_method(response):
         # Look up the node type in the project document
         node_type = next(
             (item for item in response['node_types'] if item.get('name') \
-                and item['name'] == node_type_name), None)
+             and item['name'] == node_type_name), None)
         if not node_type:
             return abort(404)
         # Check permissions and append the allowed_methods to the node_type
@@ -300,6 +310,7 @@ def before_returning_resource_notifications(response):
         if request.args.get('parse'):
             notification_parse(item)
 
+
 app.on_fetched_item_nodes += before_returning_item_permissions
 app.on_fetched_item_nodes += item_parse_attachments
 app.on_fetched_resource_nodes += before_returning_resource_permissions
@@ -316,6 +327,7 @@ app.on_fetched_item_projects += before_returning_item_permissions
 app.on_fetched_item_projects += project_node_type_has_method
 app.on_fetched_resource_projects += before_returning_resource_permissions
 
+
 def post_GET_user(request, payload):
     json_data = json.loads(payload.data)
     # Check if we are querying the users endpoint (instead of the single user)
@@ -325,19 +337,23 @@ def post_GET_user(request, payload):
     #     compute_permissions(json_data['_id'], app.data.driver)
     payload.data = json.dumps(json_data)
 
+
 def after_replacing_user(item, original):
     """Push an update to the Algolia index when a user item is updated"""
     algolia_index_user_save(item)
 
+
 app.on_post_GET_users += post_GET_user
 app.on_replace_users += after_replacing_user
 
+
 def post_POST_files(request, payload):
     """After an file object has been created, we do the necessary processing
     and further update it.
     """
     process_file(request.get_json())
 
+
 app.on_post_POST_files += post_POST_files
 
 
@@ -351,6 +367,7 @@ def before_returning_file(response):
             variation['link'] = generate_link(
                 response['backend'], variation['file_path'], project_id)
 
+
 def before_returning_files(response):
     for item in response['_items']:
         # TODO: add project id to all files
@@ -365,12 +382,15 @@ app.on_fetched_resource_files += before_returning_files
 def before_deleting_file(item):
     delete_file(item)
 
+
 app.on_delete_item_files += before_deleting_file
 
 # The file_storage module needs app to be defined
 from modules.file_storage import file_storage
-#from modules.file_storage.serve import *
+
+# from modules.file_storage.serve import *
 app.register_blueprint(file_storage, url_prefix='/storage')
 # The encoding module (receive notification and report progress)
 from modules.encoding import encoding
+
 app.register_blueprint(encoding, url_prefix='/encoding')

pillar/application/modules/encoding.py

@@ -15,7 +15,7 @@ def zencoder_notifications():
             # If we are in production, look for the Zencoder header secret
             try:
                 notification_secret_request = request.headers[
-                'X-Zencoder-Notification-Secret']
+                    'X-Zencoder-Notification-Secret']
             except KeyError:
                 return abort(401)
             # If the header is found, check it agains the one in the config
@@ -44,7 +44,7 @@ def zencoder_notifications():
                 format = 'mp4' if format == 'mpeg4' else format
                 # Find a variation matching format and resolution
                 variation = next((v for v in f['variations'] if v['format'] == format \
-                    and v['width'] == output['width']), None)
+                                  and v['width'] == output['width']), None)
                 # If found, update with delivered file size
                 # TODO: calculate md5 on the storage
                 if variation:

pillar/application/modules/file_storage.py

@@ -17,12 +17,11 @@ from application.utils.cdn import hash_file_path
 from application.utils.gcs import GoogleCloudStorageBucket
 from application.utils.encoding import Encoder
 
-
 log = logging.getLogger(__name__)
 
 file_storage = Blueprint('file_storage', __name__,
                          template_folder='templates',
-                         static_folder='../../static/storage',)
+                         static_folder='../../static/storage', )
 
 
 @file_storage.route('/gcs/<bucket_name>/<subdir>/')
@@ -50,7 +49,7 @@ def browse_gcs(bucket_name, subdir, file_path=None):
         return jsonify(listing)
 
 
-#@file_storage.route('/build_thumbnails/<path:file_path>')
+# @file_storage.route('/build_thumbnails/<path:file_path>')
 def build_thumbnails(file_path=None, file_id=None):
     """Given a file path or file ObjectId pointing to an image file, fetch it
     and generate a set of predefined variations (using generate_local_thumbnails).
@@ -92,7 +91,7 @@ def build_thumbnails(file_path=None, file_id=None):
             length=thumbnail['length'],
             md5=thumbnail['md5'],
             file_path=basename,
-            )
+        )
         # XXX Inject is_public for size 't' (should be part of the upload),
         # and currently we set it here and then on the fly during blob
         # creation by simply parsing the extension of the filename. This is
@@ -198,10 +197,10 @@ def process_file(src_file):
                 width=src_video_data['res_x'],
                 height=src_video_data['res_y'],
                 content_type="video/{0}".format(v),
-                length=0, # Available after encode
+                length=0,  # Available after encode
-                md5="", # Available after encode
+                md5="",  # Available after encode
                 file_path=filename,
-                )
+            )
             # Append file variation
             src_file['variations'].append(file_variation)
 
@@ -221,7 +220,7 @@ def process_file(src_file):
                             backend=j['backend'])
                         # Add the processing status to the file object
                         r = put_internal('files',
-                            src_file, **{'_id': ObjectId(file_id)})
+                                         src_file, **{'_id': ObjectId(file_id)})
                         pass
                 except KeyError:
                     pass
@@ -268,6 +267,7 @@ def delete_file(file_item):
             pass
         else:
             pass
+
     files_collection = app.data.driver.db['files']
     # Collect children (variations) of the original file
     children = files_collection.find({'parent': file_item['_id']})
@@ -292,7 +292,7 @@ def generate_link(backend, file_path, project_id=None, is_public=False):
             link = None
     elif backend == 'pillar':
         link = url_for('file_storage.index', file_name=file_path, _external=True,
-        _scheme=app.config['SCHEME'])
+                       _scheme=app.config['SCHEME'])
     elif backend == 'cdnsun':
         link = hash_file_path(file_path, None)
     else:

pillar/application/utils/activities.py

@@ -75,7 +75,7 @@ def notification_parse(notification):
         is_read=('is_read' in notification and notification['is_read']),
         is_subscribed=is_subscribed,
         subscription=subscription['_id']
-        )
+    )
     notification.update(updates)
 
 
@@ -112,7 +112,7 @@ def activity_subscribe(user_id, context_object_type, context_object_id):
 
 
 def activity_object_add(actor_user_id, verb, object_type, object_id,
-        context_object_type, context_object_id):
+                        context_object_type, context_object_id):
     """Add a notification object and creates a notification for each user that
     - is not the original author of the post
     - is actively subscribed to the object
@@ -138,7 +138,7 @@ def activity_object_add(actor_user_id, verb, object_type, object_id,
             object=object_id,
             context_object_type=context_object_type,
             context_object=context_object_id
-            )
+        )
 
         activity = post_internal('activities', activity)
         if activity[3] != 201:

pillar/application/utils/algolia.py

@@ -4,6 +4,7 @@ from application import algolia_index_users
 from application import algolia_index_nodes
 from application.modules.file_storage import generate_link
 
+
 def algolia_index_user_save(user):
     # Define accepted roles
     accepted_roles = ['admin', 'subscriber', 'demo']
@@ -29,7 +30,7 @@ def algolia_index_node_save(node):
     if node['node_type'] in accepted_node_types and algolia_index_nodes:
         # If a nodes does not have status published, do not index
         if 'status' in node['properties'] \
-            and node['properties']['status'] != 'published':
+                and node['properties']['status'] != 'published':
             return
 
         projects_collection = app.data.driver.db['projects']
@@ -44,15 +45,15 @@ def algolia_index_node_save(node):
             'project': {
                 '_id': project['_id'],
                 'name': project['name']
-                },
+            },
             'created': node['_created'],
             'updated': node['_updated'],
             'node_type': node['node_type'],
             'user': {
                 '_id': user['_id'],
                 'full_name': user['full_name']
-                },
+            },
-            }
+        }
         if 'description' in node and node['description']:
             node_ob['description'] = node['description']
         if 'picture' in node and node['picture']:
@@ -61,11 +62,11 @@ def algolia_index_node_save(node):
             picture = files_collection.find_one(lookup)
             if picture['backend'] == 'gcs':
                 variation_t = next((item for item in picture['variations'] \
-                    if item['size'] == 't'), None)
+                                    if item['size'] == 't'), None)
                 if variation_t:
                     node_ob['picture'] = generate_link(picture['backend'],
-                        variation_t['file_path'], project_id=str(picture['project']),
+                                                       variation_t['file_path'], project_id=str(picture['project']),
-                        is_public=True)
+                                                       is_public=True)
         # If the node has world permissions, compute the Free permission
         if 'permissions' in node and 'world' in node['permissions']:
             if 'GET' in node['permissions']['world']:

pillar/application/utils/authorization.py

@@ -44,7 +44,7 @@ def check_permissions(resource, method, append_allowed_methods=False):
                 project = projects_collection.find_one(resource['project'])
             node_type = next(
                 (item for item in project['node_types'] if item.get('name') \
-                    and item['name'] == resource['node_type']), None)
+                 and item['name'] == resource['node_type']), None)
             computed_permissions = node_type['permissions']
     else:
         computed_permissions = None

pillar/application/utils/cdn.py

@@ -2,37 +2,38 @@ import datetime
 from hashlib import md5
 from application import app
 
+
 def hash_file_path(file_path, expiry_timestamp=None):
-  if not file_path.startswith('/'):
+    if not file_path.startswith('/'):
         file_path = '/' + file_path;
-  service_domain = app.config['CDN_SERVICE_DOMAIN']
+    service_domain = app.config['CDN_SERVICE_DOMAIN']
-  domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER']
+    domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER']
-  asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \
+    asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \
-    '://' + \
+                '://' + \
-    service_domain + \
+                service_domain + \
-    domain_subfolder + \
+                domain_subfolder + \
-    file_path
+                file_path
 
-  if app.config['CDN_USE_URL_SIGNING']:
+    if app.config['CDN_USE_URL_SIGNING']:
 
-    url_signing_key = app.config['CDN_URL_SIGNING_KEY']
+        url_signing_key = app.config['CDN_URL_SIGNING_KEY']
-    hash_string = domain_subfolder + file_path + url_signing_key;
+        hash_string = domain_subfolder + file_path + url_signing_key;
 
-    if not expiry_timestamp:
+        if not expiry_timestamp:
-      expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=24)
+            expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=24)
-      expiry_timestamp = expiry_timestamp.strftime('%s')
+            expiry_timestamp = expiry_timestamp.strftime('%s')
 
-    hash_string = expiry_timestamp + hash_string;
+        hash_string = expiry_timestamp + hash_string;
 
-    expiry_timestamp = "," + str(expiry_timestamp);
+        expiry_timestamp = "," + str(expiry_timestamp);
 
-    hashed_file_path = md5(hash_string).digest().encode('base64')[:-1]
+        hashed_file_path = md5(hash_string).digest().encode('base64')[:-1]
-    hashed_file_path = hashed_file_path.replace('+', '-')
+        hashed_file_path = hashed_file_path.replace('+', '-')
-    hashed_file_path = hashed_file_path.replace('/', '_')
+        hashed_file_path = hashed_file_path.replace('/', '_')
 
-    asset_url = asset_url + \
+        asset_url = asset_url + \
-      '?secure=' + \
+                    '?secure=' + \
-      hashed_file_path + \
+                    hashed_file_path + \
-      expiry_timestamp
+                    expiry_timestamp
 
-  return asset_url
+    return asset_url

pillar/application/utils/encoding.py

@@ -3,6 +3,7 @@ from zencoder import Zencoder
 from application import encoding_service_client
 from application import app
 
+
 class Encoder:
     """Generic Encoder wrapper. Provides a consistent API, independent from
     the encoding backend enabled.
@@ -26,7 +27,7 @@ class Encoder:
                     'witdh': v['width'],
                     'url': os.path.join(storage_base, v['file_path'])})
             r = encoding_service_client.job.create(file_input, outputs=outputs,
-                options=options)
+                                                   options=options)
             if r.code == 201:
                 return dict(process_id=r.body['id'], backend='zencoder')
             else:

pillar/application/utils/gcs.py

@@ -47,7 +47,7 @@ class GoogleCloudStorageBucket(object):
         files = []
         for f in req:
             filename = os.path.basename(f.name)
-            if filename != '': # Skip own folder name
+            if filename != '':  # Skip own folder name
                 files.append(dict(
                     path=os.path.relpath(f.name, self.subdir),
                     text=filename,
@@ -67,7 +67,7 @@ class GoogleCloudStorageBucket(object):
             name=os.path.basename(os.path.normpath(path)),
             type='group_storage',
             children=files + directories
-            )
+        )
 
         return list_dict
 
@@ -165,8 +165,8 @@ def update_file_name(item):
                     pass
             except AttributeError:
                 bugsnag.notify(Exception('Missing or conflicting ids detected'),
-                    meta_data={'nodes_info':
+                               meta_data={'nodes_info':
-                        {'node_id': item['_id'], 'file_id': file_id}})
+                                              {'node_id': item['_id'], 'file_id': file_id}})
 
     # Currently we search for 'file' and 'files' keys in the object properties.
     # This could become a bit more flexible and realy on a true reference of the

pillar/application/utils/imaging.py

@@ -43,13 +43,13 @@ def generate_local_thumbnails(src, return_image_stats=False):
             format = im.format.lower()
             # Get format
             thumbnails[size] = dict(
-                file_path=dst, # Full path, to be processed before storage
+                file_path=dst,  # Full path, to be processed before storage
                 length=length,
                 width=width,
                 height=height,
                 md5='--',
                 content_type='image/' + format,
-                )
+            )
 
     if return_image_stats:
         return thumbnails
@@ -78,38 +78,38 @@ def resize_and_crop(img_path, modified_path, size, crop_type='middle'):
     # Get current and desired ratio for the images
     img_ratio = img.size[0] / float(img.size[1])
     ratio = size[0] / float(size[1])
-    #The image is scaled/cropped vertically or horizontally depending on the ratio
+    # The image is scaled/cropped vertically or horizontally depending on the ratio
     if ratio > img_ratio:
         img = img.resize((size[0], int(round(size[0] * img.size[1] / img.size[0]))),
-            Image.ANTIALIAS)
+                         Image.ANTIALIAS)
         # Crop in the top, middle or bottom
         if crop_type == 'top':
             box = (0, 0, img.size[0], size[1])
         elif crop_type == 'middle':
             box = (0, int(round((img.size[1] - size[1]) / 2)), img.size[0],
-                int(round((img.size[1] + size[1]) / 2)))
+                   int(round((img.size[1] + size[1]) / 2)))
         elif crop_type == 'bottom':
             box = (0, img.size[1] - size[1], img.size[0], img.size[1])
-        else :
+        else:
             raise ValueError('ERROR: invalid value for crop_type')
         img = img.crop(box)
     elif ratio < img_ratio:
         img = img.resize((int(round(size[1] * img.size[0] / img.size[1])), size[1]),
-            Image.ANTIALIAS)
+                         Image.ANTIALIAS)
         # Crop in the top, middle or bottom
         if crop_type == 'top':
             box = (0, 0, size[0], img.size[1])
         elif crop_type == 'middle':
             box = (int(round((img.size[0] - size[0]) / 2)), 0,
-                int(round((img.size[0] + size[0]) / 2)), img.size[1])
+                   int(round((img.size[0] + size[0]) / 2)), img.size[1])
         elif crop_type == 'bottom':
             box = (img.size[0] - size[0], 0, img.size[0], img.size[1])
-        else :
+        else:
             raise ValueError('ERROR: invalid value for crop_type')
         img = img.crop(box)
-    else :
+    else:
         img = img.resize((size[0], size[1]),
-            Image.ANTIALIAS)
+                         Image.ANTIALIAS)
     # If the scale is the same, we do not need to crop
     img.save(modified_path, "JPEG")
 
@@ -145,7 +145,7 @@ def get_video_data(filepath):
             duration=duration,
             res_x=video_stream['width'],
             res_y=video_stream['height'],
-            )
+        )
         if video_stream['sample_aspect_ratio'] != '1:1':
             print '[warning] Pixel aspect ratio is not square!'
 
@@ -184,8 +184,8 @@ def ffmpeg_encode(src, format, res_y=720):
             '-deadline', 'good',
             '-cpu-used', '0',
             '-vprofile', '0',
-            '-qmax', '51', '-qmin', '11', '-slices', '4','-b:v', '2M',
+            '-qmax', '51', '-qmin', '11', '-slices', '4', '-b:v', '2M',
-            #'-acodec', 'libmp3lame', '-ab', '112k', '-ar', '44100',
+            # '-acodec', 'libmp3lame', '-ab', '112k', '-ar', '44100',
             '-f', 'webm'])
 
     if not os.environ.get('VERBOSE'):
@@ -205,4 +205,3 @@ def ffmpeg_encode(src, format, res_y=720):
         dst = None
     # return path of the encoded video
     return dst
-

pillar/application/utils/storage.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-#import logging
+# import logging
 from application import app
 from application.utils.gcs import GoogleCloudStorageBucket
 
@@ -12,14 +12,14 @@ BIN_RSYNC = app.config['BIN_RSYNC']
 
 def get_sizedata(filepath):
     outdata = dict(
-        size = int(os.stat(filepath).st_size)
+        size=int(os.stat(filepath).st_size)
-        )
+    )
     return outdata
 
 
 def rsync(path, remote_dir=''):
     DRY_RUN = False
-    arguments=['--verbose', '--ignore-existing', '--recursive', '--human-readable']
+    arguments = ['--verbose', '--ignore-existing', '--recursive', '--human-readable']
     logs_path = app.config['CDN_SYNC_LOGS']
     storage_address = app.config['CDN_STORAGE_ADDRESS']
     user = app.config['CDN_STORAGE_USER']
@@ -34,7 +34,7 @@ def rsync(path, remote_dir=''):
             '-e ' + BIN_SSH + ' -i ' + rsa_key_path + ' -o "StrictHostKeyChecking=no"')
     # if known_hosts_path:
     #     folder_arguments.append("-o UserKnownHostsFile " + known_hosts_path)
-    folder_arguments.append("--log-file=" + logs_path  + "/rsync.log")
+    folder_arguments.append("--log-file=" + logs_path + "/rsync.log")
     folder_arguments.append(path)
     folder_arguments.append(user + "@" + storage_address + ":/public/" + remote_dir)
     # print (folder_arguments)
@@ -46,7 +46,7 @@ def rsync(path, remote_dir=''):
     subprocess.Popen(['nohup', BIN_RSYNC] + folder_arguments, stdout=devnull, stderr=devnull)
 
 
-def remote_storage_sync(path): #can be both folder and file
+def remote_storage_sync(path):  # can be both folder and file
     if os.path.isfile(path):
         filename = os.path.split(path)[1]
         rsync(path, filename[:2] + '/')
@@ -62,6 +62,7 @@ def push_to_storage(project_id, full_path, backend='cgs'):
     By default we store items in a Google Cloud Storage bucket named after the
     project id.
     """
+
     def push_single_file(project_id, full_path, backend):
         if backend == 'cgs':
             storage = GoogleCloudStorageBucket(project_id, subdir='_')
@@ -82,4 +83,3 @@ def push_to_storage(project_id, full_path, backend='cgs'):
                     push_single_file(project_id, os.path.join(root, name), backend)
         else:
             raise IOError('ERROR: path not found')
-