archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Migration script and utils for CDN link protection

Browse Source
This commit is contained in:
Francesco Siddi
2015-09-10 12:47:29 +02:00
parent 7cecfbe4e0
commit aa3f8e6837
5 changed files with 264 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pillar/application/__init__.py
View File

@@ -224,6 +224,8 @@ def post_item(entry, data):
app = Eve(validator=ValidateCustomFields, auth=CustomTokenAuth)
import config
app.config.from_object(config.Deployment)
def global_validation():
setattr(g, 'token_data', validate_token())
@@ -290,13 +292,14 @@ def post_GET_user(request, payload):
app.on_post_GET_users += post_GET_user
from utils import hash_file_path
# Hook to check the backend of a file resource, to build an appropriate link
# that can be used by the client to retrieve the actual file.
def generate_link(backend, path):
if backend == 'pillar':
link = url_for('file_server.index', file_name=path, _external=True)
elif backend == 'cdnsun':
pass
link = hash_file_path(path, None)
else:
link = None
return link

39
pillar/application/utils.py Normal file
View File

@@ -0,0 +1,39 @@
import datetime
from hashlib import md5
from application import app
def hash_file_path(file_path, expiry_timestamp=None):
service_domain = app.config['CDN_SERVICE_DOMAIN']
domain_subfolder = app.config['CDN_CONTENT_SUBFOLDER']
asset_url = app.config['CDN_SERVICE_DOMAIN_PROTOCOL'] + \
'://' + \
service_domain + \
domain_subfolder + \
file_path
if app.config['CDN_USE_URL_SIGNING']:
url_signing_key = app.config['CDN_URL_SIGNING_KEY']
if not file_path.startswith('/'):
file_path = '/' + file_path;
hash_string = domain_subfolder + file_path + url_signing_key;
if not expiry_timestamp:
expiry_timestamp = datetime.datetime.now() + datetime.timedelta(hours=12)
expiry_timestamp = expiry_timestamp.strftime('%s')
hash_string = expiry_timestamp + hash_string;
expiry_timestamp = "," + str(expiry_timestamp);
hashed_file_path = md5(hash_string).digest().encode('base64')[:-1]
hashed_file_path = hashed_file_path.replace('+', '-')
hashed_file_path = hashed_file_path.replace('/', '_')
asset_url = asset_url + \
'?secure=' + \
hashed_file_path + \
expiry_timestamp
return asset_url

22
pillar/config.py.example
View File

@@ -1,5 +1,4 @@
import os
class Config(object):
# Configured for GMAIL
MAIL_SERVER = ''
@@ -9,20 +8,13 @@ class Config(object):
MAIL_PASSWORD = ''
DEFAULT_MAIL_SENDER = ''
# Flask-Security setup
SECURITY_LOGIN_WITHOUT_CONFIRMATION = True
SECURITY_REGISTERABLE = True
SECURITY_RECOVERABLE = True
SECURITY_CHANGEABLE = True
SECUIRTY_POST_LOGIN = '/'
SECURITY_PASSWORD_HASH = ''
SECURITY_PASSWORD_SALT = ''
SECURITY_EMAIL_SENDER = ''
class Development(Config):
SECRET_KEY = ''
HOST = '0.0.0.0'
PORT = 5000
DEBUG = True
FILE_STORAGE = '{0}/application/static/storage'.format(
os.path.join(os.path.dirname(__file__)))
CDN_USE_URL_SIGNING = False
CDN_SERVICE_DOMAIN_PROTOCOL = 'https'
CDN_SERVICE_DOMAIN = ''
CDN_CONTENT_SUBFOLDER = ''
CDN_URL_SIGNING_KEY = ''
CDN_STORAGE_USER = ''
CDN_STORAGE_ADDRESS = ''

244
pillar/manage.py
View File

@@ -241,7 +241,7 @@ def add_groups():
@manager.command
def populate_db_test():
def populate_db():
"""Populate the db with sample data
"""
populate_node_types()
@@ -593,7 +593,9 @@ def populate_node_types(old_ids={}):
'description': 'Assets for Elephants Dream',
# This data type does not have parent limitations (can be child
# of any node). An empty parent declaration is required.
'parent': {},
'parent': {
"node_types": ["group",]
},
'dyn_schema': {
'status': {
'type': 'string',
@@ -605,7 +607,7 @@ def populate_node_types(old_ids={}):
},
# We expose the type of asset we point to. Usually image, video,
# zipfile, ect.
'contentType':{
'content_type':{
'type': 'string'
},
# We point to the original file (and use it to extract any relevant
@@ -621,7 +623,7 @@ def populate_node_types(old_ids={}):
},
'form_schema': {
'status': {},
'contentType': {},
'content_type': {},
'file': {},
}
}
@@ -662,33 +664,9 @@ def populate_node_types(old_ids={}):
# upgrade(project_node_type, old_ids)
upgrade(asset_node_type, old_ids)
@manager.command
def add_group():
owners_group = {
'name': 'owners',
'permissions': [
{'node_type': get_id('node_types', 'shot'),
'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
},
{'node_type': get_id('node_types', 'task'),
'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
},
{'node_type': get_id('node_types', 'scene'),
'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
},
{'node_type': get_id('node_types', 'act'),
'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
},
{'node_type': get_id('node_types', 'comment'),
'permissions': ['GET', 'POST', 'UPDATE', 'DELETE']
},
]
}
post_item('groups', world_group)
@manager.command
def add_file():
def add_file_video():
from datetime import datetime
RFC1123_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
video = {
@@ -704,15 +682,217 @@ def add_file():
'height': 720,
'user': '552b066b41acdf5dec4436f2',
'length': 15000,
'uploadDate': datetime.strftime(
datetime.now(), RFC1123_DATE_FORMAT),
'md5': 'md5',
'filename': 'The file name',
'backend': 'pillar',
'path': '0000.mp4',
}
r = post_item('files', video)
print r
return r
@manager.command
def add_node_asset(file_id):
"""Creates a node of type asset, starting from an existing file id.
:param file_id: the file id to use
:param picture_id: the picture id to use
:param group_id: the parent folder
"""
picture_id = None
parent_id = None
from bson.objectid import ObjectId
from pymongo import MongoClient
client = MongoClient(MONGO_HOST, 27017)
db = client.eve
file_object = db.files.find_one({"_id": ObjectId(file_id)})
node_type = db.node_types.find_one({"name": "asset"})
print file_object['contentType'].split('/')[0]
node = {
'name': file_object['name'],
'description': file_object['description'],
#'picture': picture_id,
#'parent': parent_id,
'user': file_object['user'],
'node_type': node_type['_id'],
'properties': {
'status': 'published',
'contentType': file_object['contentType'].split('/')[0],
'file': file_id
}
}
r = post_item('nodes', node)
return r
@manager.command
def load_migration(path):
import json
import os
from bson import json_util
if not os.path.isfile(path):
return "File does not exist"
with open(path, 'r') as infile:
d = json.load(infile)
from pymongo import MongoClient
client = MongoClient(MONGO_HOST, 27017)
db = client.eve
def commit_object(collection, f, parent=None):
variation_id = f.get('variation_id')
if variation_id:
del f['variation_id']
asset_id = f.get('asset_id')
if asset_id:
del f['asset_id']
node_id = f.get('node_id')
if node_id:
del f['node_id']
if parent:
f['parent'] = parent
else:
if f.get('parent'):
del f['parent']
#r = [{'_status': 'OK', '_id': 'DRY-ID'}]
print 'just before commit'
print f
r = post_item(collection, f)
if r[0]['_status'] == 'ERR':
print r[0]['_issues']
# Assign the Mongo ObjectID
f['_id'] = str(r[0]['_id'])
# Restore variation_id
if variation_id:
f['variation_id'] = variation_id
if asset_id:
f['asset_id'] = asset_id
if node_id:
f['node_id'] = node_id
print "{0} {1}".format(f['_id'], f['name'])
return f
# Build list of parent files
parent_files = [f for f in d['files'] if 'parent_asset_id' in f]
children_files = [f for f in d['files'] if 'parent_asset_id' not in f]
for p in parent_files:
# Store temp property
parent_asset_id = p['parent_asset_id']
# Remove from dict to prevent invalid submission
del p['parent_asset_id']
# Commit to database
p = commit_object('files', p)
# Restore temp property
p['parent_asset_id'] = parent_asset_id
# Find children of the current file
children = [c for c in children_files if c['parent'] == p['variation_id']]
for c in children:
# Commit to database with parent id
c = commit_object('files', c, p['_id'])
# Merge the dicts and replace the original one
d['files'] = parent_files + children_files
# Files for picture previews of folders (groups)
for f in d['files_group']:
item_id = f['item_id']
del f['item_id']
f = commit_object('files', f)
f['item_id'] = item_id
# Files for picture previews of assets
for f in d['files_asset']:
item_id = f['item_id']
del f['item_id']
f = commit_object('files',f)
f['item_id'] = item_id
nodes_asset = [n for n in d['nodes'] if 'asset_id' in n]
nodes_group = [n for n in d['nodes'] if 'node_id' in n]
def get_parent(node_id):
#print "Searching for {0}".format(node_id)
try:
parent = [p for p in nodes_group if p['node_id'] == node_id][0]
except IndexError:
return None
return parent
def traverse_nodes(parent_id):
parents_list = []
while True:
parent = get_parent(parent_id)
#print parent
if not parent:
break
else:
parents_list.append(parent['node_id'])
if parent.get('parent'):
parent_id = parent['parent']
else:
break
parents_list.reverse()
return parents_list
for n in nodes_asset:
node_type_asset = db.node_types.find_one({"name": "asset"})
pictures = [p for p in d['files_asset'] if p['name'] == n['picture'][:-4]]
if pictures:
n['picture'] = pictures[0]['_id']
print "Adding picture link {0}".format(n['picture'])
n['node_type'] = node_type_asset['_id']
# An asset node must have a parent
# parent = [p for p in nodes_group if p['node_id'] == n['parent']][0]
parents_list = traverse_nodes(n['parent'])
tree_index = 0
for node_id in parents_list:
node = [p for p in nodes_group if p['node_id'] == node_id][0]
if node.get('_id') is None:
node_type_group = db.node_types.find_one({"name": "group"})
node['node_type'] = node_type_group['_id']
# Assign picture to the node group
if node.get('picture'):
picture = [p for p in d['files_group'] if p['name'] == node['picture'][:-4]][0]
node['picture'] = picture['_id']
print "Adding picture link to node {0}".format(node['picture'])
if tree_index == 0:
# We are at the root of the tree (so we link to the project)
parent = None
else:
# Get the parent node id
parents_list_node_id = parents_list[tree_index - 1]
parent_node = [p for p in nodes_group if p['node_id'] == parents_list_node_id][0]
parent = parent_node['_id']
print "About to commit Node"
commit_object('nodes', node, parent)
tree_index += 1
# Commit the asset
print "About to commit Asset"
parent_node = [p for p in nodes_group if p['node_id'] == parents_list[-1]][0]
asset_file = [a for a in d['files'] if a['md5'] == n['properties']['file']][0]
n['properties']['file'] = str(asset_file['_id'])
print n
commit_object('nodes', n, parent_node['_id'])
return
# New path with _
path = '_' + path
with open(path, 'w') as outfile:
json.dump(d, outfile, default=json_util.default)
return
if __name__ == '__main__':

9
pillar/settings.py
View File

@@ -155,7 +155,7 @@ nodes_schema = {
'description': {
'type': 'string',
'minlength': 0,
'maxlength': 128,
'maxlength': 512,
},
'picture': {
'type': 'objectid',
@@ -253,7 +253,6 @@ files_schema = {
},
'description': {
'type': 'string',
'required': True,
},
# If the object has a parent, it is a variation of its parent. When querying
# for a file we are going to check if the object does NOT have a parent. In
@@ -297,10 +296,6 @@ files_schema = {
'type': 'integer',
'required': True,
},
'uploadDate': {
'type': 'datetime',
'required': True,
},
'md5': {
'type': 'string',
'required': True,
@@ -312,7 +307,7 @@ files_schema = {
'backend': {
'type': 'string',
'required': True,
'allowed': ["attract-web", "pillar"]
'allowed': ["attract-web", "pillar", "cdnsun"]
},
'path': {
'type': 'string',