From 8d9a23b7b94705cc549974c6fe01ff52f0f66281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 21 Mar 2016 12:25:07 +0100 Subject: [PATCH] PEP8 formatting for the entire project. Conflicts: pillar/application/modules/file_storage.py --- pillar/application/__init__.py | 40 +++++++++++++----- pillar/application/modules/encoding.py | 4 +- pillar/application/modules/file_storage.py | 18 ++++---- pillar/application/utils/activities.py | 6 +-- pillar/application/utils/algolia.py | 15 +++---- pillar/application/utils/authorization.py | 2 +- pillar/application/utils/cdn.py | 49 +++++++++++----------- pillar/application/utils/encoding.py | 3 +- pillar/application/utils/gcs.py | 8 ++-- pillar/application/utils/imaging.py | 29 +++++++------ pillar/application/utils/storage.py | 14 +++---- 11 files changed, 105 insertions(+), 83 deletions(-) diff --git a/pillar/application/__init__.py b/pillar/application/__init__.py index 15b81a02..1eff40e8 100644 --- a/pillar/application/__init__.py +++ b/pillar/application/__init__.py @@ -15,7 +15,6 @@ from eve import Eve from eve.auth import TokenAuth from eve.io.mongo import Validator - RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' @@ -63,7 +62,7 @@ class ValidateCustomFields(Validator): project = projects_collection.find_one(lookup) node_type = next( (item for item in project['node_types'] if item.get('name') \ - and item['name'] == self.document['node_type']), None) + and item['name'] == self.document['node_type']), None) try: value = self.convert_properties(value, node_type['dyn_schema']) except Exception, e: @@ -82,6 +81,7 @@ class ValidateCustomFields(Validator): self._error( field, "Error validating properties") + # We specify a settings.py file because when running on wsgi we can't detect it # automatically. The default path (which works in Docker) can be overridden with # an env variable. @@ -90,6 +90,7 @@ settings_path = os.environ.get( app = Eve(settings=settings_path, validator=ValidateCustomFields, auth=NewAuth) import config + app.config.from_object(config.Deployment) # Configure logging @@ -104,8 +105,8 @@ log.setLevel(logging.DEBUG if app.config['DEBUG'] else logging.INFO) log.info('Pillar starting') bugsnag.configure( - api_key=app.config['BUGSNAG_API_KEY'], - project_root="/data/git/pillar/pillar", + api_key=app.config['BUGSNAG_API_KEY'], + project_root="/data/git/pillar/pillar", ) handle_exceptions(app) @@ -125,8 +126,8 @@ except KeyError: # Algolia search if 'ALGOLIA_USER' in app.config: client = algoliasearch.Client( - app.config['ALGOLIA_USER'], - app.config['ALGOLIA_API_KEY']) + app.config['ALGOLIA_USER'], + app.config['ALGOLIA_API_KEY']) algolia_index_users = client.init_index(app.config['ALGOLIA_INDEX_USERS']) algolia_index_nodes = client.init_index(app.config['ALGOLIA_INDEX_NODES']) else: @@ -157,24 +158,29 @@ def before_returning_item_permissions(response): validate_token() check_permissions(response, 'GET', append_allowed_methods=True) + def before_returning_resource_permissions(response): for item in response['_items']: validate_token() check_permissions(item, 'GET', append_allowed_methods=True) + def before_replacing_node(item, original): check_permissions(original, 'PUT') update_file_name(item) + def after_replacing_node(item, original): """Push an update to the Algolia index when a node item is updated""" algolia_index_node_save(item) + def before_inserting_nodes(items): """Before inserting a node in the collection we check if the user is allowed and we append the project id to it. """ nodes_collection = app.data.driver.db['nodes'] + def find_parent_project(node): """Recursive function that finds the ultimate parent of a node.""" if node and 'parent' in node: @@ -184,6 +190,7 @@ def before_inserting_nodes(items): return node else: return None + for item in items: check_permissions(item, 'POST') if 'parent' in item and 'project' not in item: @@ -223,7 +230,8 @@ def after_inserting_nodes(items): item['_id'], 'node', context_object_id - ) + ) + def item_parse_attachments(response): """Before returning a response, check if the 'attachments' property is @@ -252,7 +260,7 @@ def item_parse_attachments(response): size = f['size'] if 'size' in f else 'l' # Get the correc variation from the file thumbnail = next((item for item in f['variations'] if - item['size'] == size), None) + item['size'] == size), None) l = generate_link(f['backend'], thumbnail['file_path'], str(f['project'])) # Build Markdown img string l = '![{0}]({1} "{2}")'.format(slug, l, f['name']) @@ -266,10 +274,12 @@ def item_parse_attachments(response): else: response[field_name_path[0]] = field_content + def resource_parse_attachments(response): for item in response['_items']: item_parse_attachments(item) + def project_node_type_has_method(response): """Check for a specific request arg, and check generate the allowed_methods list for the required node_type. @@ -283,7 +293,7 @@ def project_node_type_has_method(response): # Look up the node type in the project document node_type = next( (item for item in response['node_types'] if item.get('name') \ - and item['name'] == node_type_name), None) + and item['name'] == node_type_name), None) if not node_type: return abort(404) # Check permissions and append the allowed_methods to the node_type @@ -300,6 +310,7 @@ def before_returning_resource_notifications(response): if request.args.get('parse'): notification_parse(item) + app.on_fetched_item_nodes += before_returning_item_permissions app.on_fetched_item_nodes += item_parse_attachments app.on_fetched_resource_nodes += before_returning_resource_permissions @@ -316,6 +327,7 @@ app.on_fetched_item_projects += before_returning_item_permissions app.on_fetched_item_projects += project_node_type_has_method app.on_fetched_resource_projects += before_returning_resource_permissions + def post_GET_user(request, payload): json_data = json.loads(payload.data) # Check if we are querying the users endpoint (instead of the single user) @@ -325,19 +337,23 @@ def post_GET_user(request, payload): # compute_permissions(json_data['_id'], app.data.driver) payload.data = json.dumps(json_data) + def after_replacing_user(item, original): """Push an update to the Algolia index when a user item is updated""" algolia_index_user_save(item) + app.on_post_GET_users += post_GET_user app.on_replace_users += after_replacing_user + def post_POST_files(request, payload): """After an file object has been created, we do the necessary processing and further update it. """ process_file(request.get_json()) + app.on_post_POST_files += post_POST_files @@ -351,6 +367,7 @@ def before_returning_file(response): variation['link'] = generate_link( response['backend'], variation['file_path'], project_id) + def before_returning_files(response): for item in response['_items']: # TODO: add project id to all files @@ -365,12 +382,15 @@ app.on_fetched_resource_files += before_returning_files def before_deleting_file(item): delete_file(item) + app.on_delete_item_files += before_deleting_file # The file_storage module needs app to be defined from modules.file_storage import file_storage -#from modules.file_storage.serve import * + +# from modules.file_storage.serve import * app.register_blueprint(file_storage, url_prefix='/storage') # The encoding module (receive notification and report progress) from modules.encoding import encoding + app.register_blueprint(encoding, url_prefix='/encoding') diff --git a/pillar/application/modules/encoding.py b/pillar/application/modules/encoding.py index 0f44313b..02a688c3 100644 --- a/pillar/application/modules/encoding.py +++ b/pillar/application/modules/encoding.py @@ -15,7 +15,7 @@ def zencoder_notifications(): # If we are in production, look for the Zencoder header secret try: notification_secret_request = request.headers[ - 'X-Zencoder-Notification-Secret'] + 'X-Zencoder-Notification-Secret'] except KeyError: return abort(401) # If the header is found, check it agains the one in the config @@ -44,7 +44,7 @@ def zencoder_notifications(): format = 'mp4' if format == 'mpeg4' else format # Find a variation matching format and resolution variation = next((v for v in f['variations'] if v['format'] == format \ - and v['width'] == output['width']), None) + and v['width'] == output['width']), None) # If found, update with delivered file size # TODO: calculate md5 on the storage if variation: diff --git a/pillar/application/modules/file_storage.py b/pillar/application/modules/file_storage.py index 65c39ac2..47053a67 100644 --- a/pillar/application/modules/file_storage.py +++ b/pillar/application/modules/file_storage.py @@ -17,12 +17,11 @@ from application.utils.cdn import hash_file_path from application.utils.gcs import GoogleCloudStorageBucket from application.utils.encoding import Encoder - log = logging.getLogger(__name__) file_storage = Blueprint('file_storage', __name__, template_folder='templates', - static_folder='../../static/storage',) + static_folder='../../static/storage', ) @file_storage.route('/gcs///') @@ -50,7 +49,7 @@ def browse_gcs(bucket_name, subdir, file_path=None): return jsonify(listing) -#@file_storage.route('/build_thumbnails/') +# @file_storage.route('/build_thumbnails/') def build_thumbnails(file_path=None, file_id=None): """Given a file path or file ObjectId pointing to an image file, fetch it and generate a set of predefined variations (using generate_local_thumbnails). @@ -92,7 +91,7 @@ def build_thumbnails(file_path=None, file_id=None): length=thumbnail['length'], md5=thumbnail['md5'], file_path=basename, - ) + ) # XXX Inject is_public for size 't' (should be part of the upload), # and currently we set it here and then on the fly during blob # creation by simply parsing the extension of the filename. This is @@ -198,10 +197,10 @@ def process_file(src_file): width=src_video_data['res_x'], height=src_video_data['res_y'], content_type="video/{0}".format(v), - length=0, # Available after encode - md5="", # Available after encode + length=0, # Available after encode + md5="", # Available after encode file_path=filename, - ) + ) # Append file variation src_file['variations'].append(file_variation) @@ -221,7 +220,7 @@ def process_file(src_file): backend=j['backend']) # Add the processing status to the file object r = put_internal('files', - src_file, **{'_id': ObjectId(file_id)}) + src_file, **{'_id': ObjectId(file_id)}) pass except KeyError: pass @@ -268,6 +267,7 @@ def delete_file(file_item): pass else: pass + files_collection = app.data.driver.db['files'] # Collect children (variations) of the original file children = files_collection.find({'parent': file_item['_id']}) @@ -292,7 +292,7 @@ def generate_link(backend, file_path, project_id=None, is_public=False): link = None elif backend == 'pillar': link = url_for('file_storage.index', file_name=file_path, _external=True, - _scheme=app.config['SCHEME']) + _scheme=app.config['SCHEME']) elif backend == 'cdnsun': link = hash_file_path(file_path, None) else: diff --git a/pillar/application/utils/activities.py b/pillar/application/utils/activities.py index 84308acd..1d0495d1 100644 --- a/pillar/application/utils/activities.py +++ b/pillar/application/utils/activities.py @@ -75,7 +75,7 @@ def notification_parse(notification): is_read=('is_read' in notification and notification['is_read']), is_subscribed=is_subscribed, subscription=subscription['_id'] - ) + ) notification.update(updates) @@ -112,7 +112,7 @@ def activity_subscribe(user_id, context_object_type, context_object_id): def activity_object_add(actor_user_id, verb, object_type, object_id, - context_object_type, context_object_id): + context_object_type, context_object_id): """Add a notification object and creates a notification for each user that - is not the original author of the post - is actively subscribed to the object @@ -138,7 +138,7 @@ def activity_object_add(actor_user_id, verb, object_type, object_id, object=object_id, context_object_type=context_object_type, context_object=context_object_id - ) + ) activity = post_internal('activities', activity) if activity[3] != 201: diff --git a/pillar/application/utils/algolia.py b/pillar/application/utils/algolia.py index 6c96d4ef..58006a8d 100644 --- a/pillar/application/utils/algolia.py +++ b/pillar/application/utils/algolia.py @@ -4,6 +4,7 @@ from application import algolia_index_users from application import algolia_index_nodes from application.modules.file_storage import generate_link + def algolia_index_user_save(user): # Define accepted roles accepted_roles = ['admin', 'subscriber', 'demo'] @@ -29,7 +30,7 @@ def algolia_index_node_save(node): if node['node_type'] in accepted_node_types and algolia_index_nodes: # If a nodes does not have status published, do not index if 'status' in node['properties'] \ - and node['properties']['status'] != 'published': + and node['properties']['status'] != 'published': return projects_collection = app.data.driver.db['projects'] @@ -44,15 +45,15 @@ def algolia_index_node_save(node): 'project': { '_id': project['_id'], 'name': project['name'] - }, + }, 'created': node['_created'], 'updated': node['_updated'], 'node_type': node['node_type'], 'user': { '_id': user['_id'], 'full_name': user['full_name'] - }, - } + }, + } if 'description' in node and node['description']: node_ob['description'] = node['description'] if 'picture' in node and node['picture']: @@ -61,11 +62,11 @@ def algolia_index_node_save(node): picture = files_collection.find_one(lookup) if picture['backend'] == 'gcs': variation_t = next((item for item in picture['variations'] \ - if item['size'] == 't'), None) + if item['size'] == 't'), None) if variation_t: node_ob['picture'] = generate_link(picture['backend'], - variation_t['file_path'], project_id=str(picture['project']), - is_public=True) + variation_t['file_path'], project_id=str(picture['project']), + is_public=True) # If the node has world permissions, compute the Free permission if 'permissions' in node and 'world' in node['permissions']: if 'GET' in node['permissions']['world']: diff --git a/pillar/application/utils/authorization.py b/pillar/application/utils/authorization.py index a9849af8..8c7baf1d 100644 --- a/pillar/application/utils/authorization.py +++ b/pillar/application/utils/authorization.py @@ -44,7 +44,7 @@ def check_permissions(resource, method, append_allowed_methods=False): project = projects_collection.find_one(resource['project']) node_type = next( (item for item in project['node_types'] if item.get('name') \ - and item['name'] == resource['node_type']), None) + and item['name'] == resource['node_type']), None) computed_permissions = node_type['permissions'] else: computed_permissions = None diff --git a/pillar/application/utils/cdn.py b/pillar/application/utils/cdn.py index 61146772..5ee18b0d 100644 --- a/pillar/application/utils/cdn.py +++ b/pillar/application/utils/cdn.py @@ -2,37 +2,38 @@ import datetime from hashlib import md5 from application import app + def hash_file_path(file_path, expiry_timestamp=None): - if not file_path.startswith('/'): + if not file_path.startswith('/'): file_path = '/' + file_path; - service_domain = app.config['CDN_SERVICE_DOMAIN'] - domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER'] - asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \ - '://' + \ - service_domain + \ - domain_subfolder + \ - file_path + service_domain = app.config['CDN_SERVICE_DOMAIN'] + domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER'] + asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \ + '://' + \ + service_domain + \ + domain_subfolder + \ + file_path - if app.config['CDN_USE_URL_SIGNING']: + if app.config['CDN_USE_URL_SIGNING']: - url_signing_key = app.config['CDN_URL_SIGNING_KEY'] - hash_string = domain_subfolder + file_path + url_signing_key; + url_signing_key = app.config['CDN_URL_SIGNING_KEY'] + hash_string = domain_subfolder + file_path + url_signing_key; - if not expiry_timestamp: - expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=24) - expiry_timestamp = expiry_timestamp.strftime('%s') + if not expiry_timestamp: + expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=24) + expiry_timestamp = expiry_timestamp.strftime('%s') - hash_string = expiry_timestamp + hash_string; + hash_string = expiry_timestamp + hash_string; - expiry_timestamp = "," + str(expiry_timestamp); + expiry_timestamp = "," + str(expiry_timestamp); - hashed_file_path = md5(hash_string).digest().encode('base64')[:-1] - hashed_file_path = hashed_file_path.replace('+', '-') - hashed_file_path = hashed_file_path.replace('/', '_') + hashed_file_path = md5(hash_string).digest().encode('base64')[:-1] + hashed_file_path = hashed_file_path.replace('+', '-') + hashed_file_path = hashed_file_path.replace('/', '_') - asset_url = asset_url + \ - '?secure=' + \ - hashed_file_path + \ - expiry_timestamp + asset_url = asset_url + \ + '?secure=' + \ + hashed_file_path + \ + expiry_timestamp - return asset_url + return asset_url diff --git a/pillar/application/utils/encoding.py b/pillar/application/utils/encoding.py index 2b5c1420..bd86d47f 100644 --- a/pillar/application/utils/encoding.py +++ b/pillar/application/utils/encoding.py @@ -3,6 +3,7 @@ from zencoder import Zencoder from application import encoding_service_client from application import app + class Encoder: """Generic Encoder wrapper. Provides a consistent API, independent from the encoding backend enabled. @@ -26,7 +27,7 @@ class Encoder: 'witdh': v['width'], 'url': os.path.join(storage_base, v['file_path'])}) r = encoding_service_client.job.create(file_input, outputs=outputs, - options=options) + options=options) if r.code == 201: return dict(process_id=r.body['id'], backend='zencoder') else: diff --git a/pillar/application/utils/gcs.py b/pillar/application/utils/gcs.py index 09487d59..a56711cd 100644 --- a/pillar/application/utils/gcs.py +++ b/pillar/application/utils/gcs.py @@ -47,7 +47,7 @@ class GoogleCloudStorageBucket(object): files = [] for f in req: filename = os.path.basename(f.name) - if filename != '': # Skip own folder name + if filename != '': # Skip own folder name files.append(dict( path=os.path.relpath(f.name, self.subdir), text=filename, @@ -67,7 +67,7 @@ class GoogleCloudStorageBucket(object): name=os.path.basename(os.path.normpath(path)), type='group_storage', children=files + directories - ) + ) return list_dict @@ -165,8 +165,8 @@ def update_file_name(item): pass except AttributeError: bugsnag.notify(Exception('Missing or conflicting ids detected'), - meta_data={'nodes_info': - {'node_id': item['_id'], 'file_id': file_id}}) + meta_data={'nodes_info': + {'node_id': item['_id'], 'file_id': file_id}}) # Currently we search for 'file' and 'files' keys in the object properties. # This could become a bit more flexible and realy on a true reference of the diff --git a/pillar/application/utils/imaging.py b/pillar/application/utils/imaging.py index aefe1b1e..e8e5e795 100644 --- a/pillar/application/utils/imaging.py +++ b/pillar/application/utils/imaging.py @@ -43,13 +43,13 @@ def generate_local_thumbnails(src, return_image_stats=False): format = im.format.lower() # Get format thumbnails[size] = dict( - file_path=dst, # Full path, to be processed before storage + file_path=dst, # Full path, to be processed before storage length=length, width=width, height=height, md5='--', content_type='image/' + format, - ) + ) if return_image_stats: return thumbnails @@ -78,38 +78,38 @@ def resize_and_crop(img_path, modified_path, size, crop_type='middle'): # Get current and desired ratio for the images img_ratio = img.size[0] / float(img.size[1]) ratio = size[0] / float(size[1]) - #The image is scaled/cropped vertically or horizontally depending on the ratio + # The image is scaled/cropped vertically or horizontally depending on the ratio if ratio > img_ratio: img = img.resize((size[0], int(round(size[0] * img.size[1] / img.size[0]))), - Image.ANTIALIAS) + Image.ANTIALIAS) # Crop in the top, middle or bottom if crop_type == 'top': box = (0, 0, img.size[0], size[1]) elif crop_type == 'middle': box = (0, int(round((img.size[1] - size[1]) / 2)), img.size[0], - int(round((img.size[1] + size[1]) / 2))) + int(round((img.size[1] + size[1]) / 2))) elif crop_type == 'bottom': box = (0, img.size[1] - size[1], img.size[0], img.size[1]) - else : + else: raise ValueError('ERROR: invalid value for crop_type') img = img.crop(box) elif ratio < img_ratio: img = img.resize((int(round(size[1] * img.size[0] / img.size[1])), size[1]), - Image.ANTIALIAS) + Image.ANTIALIAS) # Crop in the top, middle or bottom if crop_type == 'top': box = (0, 0, size[0], img.size[1]) elif crop_type == 'middle': box = (int(round((img.size[0] - size[0]) / 2)), 0, - int(round((img.size[0] + size[0]) / 2)), img.size[1]) + int(round((img.size[0] + size[0]) / 2)), img.size[1]) elif crop_type == 'bottom': box = (img.size[0] - size[0], 0, img.size[0], img.size[1]) - else : + else: raise ValueError('ERROR: invalid value for crop_type') img = img.crop(box) - else : + else: img = img.resize((size[0], size[1]), - Image.ANTIALIAS) + Image.ANTIALIAS) # If the scale is the same, we do not need to crop img.save(modified_path, "JPEG") @@ -145,7 +145,7 @@ def get_video_data(filepath): duration=duration, res_x=video_stream['width'], res_y=video_stream['height'], - ) + ) if video_stream['sample_aspect_ratio'] != '1:1': print '[warning] Pixel aspect ratio is not square!' @@ -184,8 +184,8 @@ def ffmpeg_encode(src, format, res_y=720): '-deadline', 'good', '-cpu-used', '0', '-vprofile', '0', - '-qmax', '51', '-qmin', '11', '-slices', '4','-b:v', '2M', - #'-acodec', 'libmp3lame', '-ab', '112k', '-ar', '44100', + '-qmax', '51', '-qmin', '11', '-slices', '4', '-b:v', '2M', + # '-acodec', 'libmp3lame', '-ab', '112k', '-ar', '44100', '-f', 'webm']) if not os.environ.get('VERBOSE'): @@ -205,4 +205,3 @@ def ffmpeg_encode(src, format, res_y=720): dst = None # return path of the encoded video return dst - diff --git a/pillar/application/utils/storage.py b/pillar/application/utils/storage.py index 8ba97785..854f086c 100644 --- a/pillar/application/utils/storage.py +++ b/pillar/application/utils/storage.py @@ -1,6 +1,6 @@ import os import subprocess -#import logging +# import logging from application import app from application.utils.gcs import GoogleCloudStorageBucket @@ -12,14 +12,14 @@ BIN_RSYNC = app.config['BIN_RSYNC'] def get_sizedata(filepath): outdata = dict( - size = int(os.stat(filepath).st_size) - ) + size=int(os.stat(filepath).st_size) + ) return outdata def rsync(path, remote_dir=''): DRY_RUN = False - arguments=['--verbose', '--ignore-existing', '--recursive', '--human-readable'] + arguments = ['--verbose', '--ignore-existing', '--recursive', '--human-readable'] logs_path = app.config['CDN_SYNC_LOGS'] storage_address = app.config['CDN_STORAGE_ADDRESS'] user = app.config['CDN_STORAGE_USER'] @@ -34,7 +34,7 @@ def rsync(path, remote_dir=''): '-e ' + BIN_SSH + ' -i ' + rsa_key_path + ' -o "StrictHostKeyChecking=no"') # if known_hosts_path: # folder_arguments.append("-o UserKnownHostsFile " + known_hosts_path) - folder_arguments.append("--log-file=" + logs_path + "/rsync.log") + folder_arguments.append("--log-file=" + logs_path + "/rsync.log") folder_arguments.append(path) folder_arguments.append(user + "@" + storage_address + ":/public/" + remote_dir) # print (folder_arguments) @@ -46,7 +46,7 @@ def rsync(path, remote_dir=''): subprocess.Popen(['nohup', BIN_RSYNC] + folder_arguments, stdout=devnull, stderr=devnull) -def remote_storage_sync(path): #can be both folder and file +def remote_storage_sync(path): # can be both folder and file if os.path.isfile(path): filename = os.path.split(path)[1] rsync(path, filename[:2] + '/') @@ -62,6 +62,7 @@ def push_to_storage(project_id, full_path, backend='cgs'): By default we store items in a Google Cloud Storage bucket named after the project id. """ + def push_single_file(project_id, full_path, backend): if backend == 'cgs': storage = GoogleCloudStorageBucket(project_id, subdir='_') @@ -82,4 +83,3 @@ def push_to_storage(project_id, full_path, backend='cgs'): push_single_file(project_id, os.path.join(root, name), backend) else: raise IOError('ERROR: path not found') -