965 lines
33 KiB
Python
Executable File
965 lines
33 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
|
|
import copy
|
|
import os
|
|
import logging
|
|
|
|
from bson.objectid import ObjectId
|
|
from eve.methods.put import put_internal
|
|
from eve.methods.post import post_internal
|
|
from flask.ext.script import Manager
|
|
|
|
# Use a sensible default when running manage.py commands.
|
|
if not os.environ.get('EVE_SETTINGS'):
|
|
settings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
'settings.py')
|
|
os.environ['EVE_SETTINGS'] = settings_path
|
|
|
|
from application import app
|
|
from application.utils.gcs import GoogleCloudStorageBucket
|
|
from manage_extra.node_types.asset import node_type_asset
|
|
from manage_extra.node_types.blog import node_type_blog
|
|
from manage_extra.node_types.comment import node_type_comment
|
|
from manage_extra.node_types.group import node_type_group
|
|
from manage_extra.node_types.post import node_type_post
|
|
from manage_extra.node_types.project import node_type_project
|
|
from manage_extra.node_types.storage import node_type_storage
|
|
from manage_extra.node_types.texture import node_type_texture
|
|
from manage_extra.node_types.group_texture import node_type_group_texture
|
|
|
|
manager = Manager(app)
|
|
|
|
log = logging.getLogger('manage')
|
|
log.setLevel(logging.INFO)
|
|
|
|
MONGO_HOST = os.environ.get('MONGO_HOST', 'localhost')
|
|
|
|
|
|
@manager.command
|
|
def runserver():
|
|
# Automatic creation of STORAGE_DIR path if it's missing
|
|
if not os.path.exists(app.config['STORAGE_DIR']):
|
|
os.makedirs(app.config['STORAGE_DIR'])
|
|
|
|
app.run(host=app.config['HOST'],
|
|
port=app.config['PORT'],
|
|
debug=app.config['DEBUG'])
|
|
|
|
|
|
@manager.command
|
|
def runserver_memlimit(limit_kb=1000000):
|
|
import resource
|
|
|
|
limit_b = int(limit_kb) * 1024
|
|
for rsrc in (resource.RLIMIT_AS, resource.RLIMIT_DATA, resource.RLIMIT_RSS):
|
|
resource.setrlimit(rsrc, (limit_b, limit_b))
|
|
|
|
runserver()
|
|
|
|
|
|
def post_item(entry, data):
|
|
return post_internal(entry, data)
|
|
|
|
|
|
def put_item(collection, item):
|
|
item_id = item['_id']
|
|
internal_fields = ['_id', '_etag', '_updated', '_created']
|
|
for field in internal_fields:
|
|
item.pop(field, None)
|
|
# print item
|
|
# print type(item_id)
|
|
p = put_internal(collection, item, **{'_id': item_id})
|
|
if p[0]['_status'] == 'ERR':
|
|
print(p)
|
|
print(item)
|
|
|
|
|
|
@manager.command
|
|
def setup_db(admin_email):
|
|
"""Setup the database
|
|
- Create admin, subscriber and demo Group collection
|
|
- Create admin user (must use valid blender-id credentials)
|
|
- Create one project
|
|
"""
|
|
|
|
# Create default groups
|
|
groups_list = []
|
|
for group in ['admin', 'subscriber', 'demo']:
|
|
g = {'name': group}
|
|
g = post_internal('groups', g)
|
|
groups_list.append(g[0]['_id'])
|
|
print("Creating group {0}".format(group))
|
|
|
|
# Create admin user
|
|
user = {'username': admin_email,
|
|
'groups': groups_list,
|
|
'roles': ['admin', 'subscriber', 'demo'],
|
|
'settings': {'email_communications': 1},
|
|
'auth': [],
|
|
'full_name': admin_email,
|
|
'email': admin_email}
|
|
result, _, _, status = post_internal('users', user)
|
|
if status != 201:
|
|
raise SystemExit('Error creating user {}: {}'.format(admin_email, result))
|
|
user.update(result)
|
|
print("Created user {0}".format(user['_id']))
|
|
|
|
# Create a default project by faking a POST request.
|
|
with app.test_request_context(data={'project_name': u'Default Project'}):
|
|
from flask import g
|
|
from application.modules import projects
|
|
|
|
g.current_user = {'user_id': user['_id'],
|
|
'groups': user['groups'],
|
|
'roles': set(user['roles'])}
|
|
|
|
projects.create_project(overrides={'url': 'default-project',
|
|
'is_private': False})
|
|
|
|
|
|
def _default_permissions():
|
|
"""Returns a dict of default permissions.
|
|
|
|
Usable for projects, node types, and others.
|
|
|
|
:rtype: dict
|
|
"""
|
|
|
|
groups_collection = app.data.driver.db['groups']
|
|
admin_group = groups_collection.find_one({'name': 'admin'})
|
|
|
|
default_permissions = {
|
|
'world': ['GET'],
|
|
'users': [],
|
|
'groups': [
|
|
{'group': admin_group['_id'],
|
|
'methods': ['GET', 'PUT', 'POST']},
|
|
]
|
|
}
|
|
|
|
return default_permissions
|
|
|
|
|
|
@manager.command
|
|
def setup_for_attract(project_uuid, replace=False):
|
|
"""Adds Attract node types to the project.
|
|
|
|
:param project_uuid: the UUID of the project to update
|
|
:type project_uuid: str
|
|
:param replace: whether to replace existing Attract node types (True),
|
|
or to keep existing node types (False, the default).
|
|
:type replace: bool
|
|
"""
|
|
|
|
from manage_extra.node_types.act import node_type_act
|
|
from manage_extra.node_types.scene import node_type_scene
|
|
from manage_extra.node_types.shot import node_type_shot
|
|
|
|
# Copy permissions from the project, then give everyone with PUT
|
|
# access also DELETE access.
|
|
project = _get_project(project_uuid)
|
|
permissions = copy.deepcopy(project['permissions'])
|
|
|
|
for perms in permissions.values():
|
|
for perm in perms:
|
|
methods = set(perm['methods'])
|
|
if 'PUT' not in perm['methods']:
|
|
continue
|
|
methods.add('DELETE')
|
|
perm['methods'] = list(methods)
|
|
|
|
node_type_act['permissions'] = permissions
|
|
node_type_scene['permissions'] = permissions
|
|
node_type_shot['permissions'] = permissions
|
|
|
|
# Add the missing node types.
|
|
for node_type in (node_type_act, node_type_scene, node_type_shot):
|
|
found = [nt for nt in project['node_types']
|
|
if nt['name'] == node_type['name']]
|
|
if found:
|
|
assert len(found) == 1, 'node type name should be unique (found %ix)' % len(found)
|
|
|
|
# TODO: validate that the node type contains all the properties Attract needs.
|
|
if replace:
|
|
log.info('Replacing existing node type %s', node_type['name'])
|
|
project['node_types'].remove(found[0])
|
|
else:
|
|
continue
|
|
|
|
project['node_types'].append(node_type)
|
|
|
|
_update_project(project_uuid, project)
|
|
|
|
log.info('Project %s was updated for Attract.', project_uuid)
|
|
|
|
|
|
def _get_project(project_uuid):
|
|
"""Find a project in the database, or SystemExit()s.
|
|
|
|
:param project_uuid: UUID of the project
|
|
:type: str
|
|
:return: the project
|
|
:rtype: dict
|
|
"""
|
|
|
|
projects_collection = app.data.driver.db['projects']
|
|
project_id = ObjectId(project_uuid)
|
|
|
|
# Find the project in the database.
|
|
project = projects_collection.find_one(project_id)
|
|
if not project:
|
|
log.error('Project %s does not exist.', project_uuid)
|
|
raise SystemExit()
|
|
|
|
return project
|
|
|
|
|
|
def _update_project(project_uuid, project):
|
|
"""Updates a project in the database, or SystemExit()s.
|
|
|
|
:param project_uuid: UUID of the project
|
|
:type: str
|
|
:param project: the project data, should be the entire project document
|
|
:type: dict
|
|
:return: the project
|
|
:rtype: dict
|
|
"""
|
|
|
|
from application.utils import remove_private_keys
|
|
|
|
project_id = ObjectId(project_uuid)
|
|
project = remove_private_keys(project)
|
|
result, _, _, _ = put_internal('projects', project, _id=project_id)
|
|
|
|
if result['_status'] != 'OK':
|
|
log.error("Can't update project %s, issues: %s", project_uuid, result['_issues'])
|
|
raise SystemExit()
|
|
|
|
|
|
@manager.command
|
|
def clear_db():
|
|
"""Wipes the database
|
|
"""
|
|
from pymongo import MongoClient
|
|
|
|
client = MongoClient(MONGO_HOST, 27017)
|
|
db = client.eve
|
|
db.drop_collection('nodes')
|
|
db.drop_collection('node_types')
|
|
db.drop_collection('tokens')
|
|
db.drop_collection('users')
|
|
|
|
|
|
@manager.command
|
|
def upgrade_node_types():
|
|
"""Wipes node_types collection and populates it again"""
|
|
node_types_collection = app.data.driver.db['node_types']
|
|
node_types = node_types_collection.find({})
|
|
old_ids = {}
|
|
for node_type in node_types:
|
|
old_ids[node_type['name']] = node_type['_id']
|
|
populate_node_types(old_ids)
|
|
|
|
|
|
@manager.command
|
|
def manage_groups():
|
|
"""Take user email and group name,
|
|
and add or remove the user from that group.
|
|
"""
|
|
from pymongo import MongoClient
|
|
client = MongoClient(MONGO_HOST, 27017)
|
|
db = client.eve
|
|
|
|
print("")
|
|
print("Add or Remove user from group")
|
|
print("leave empty to cancel")
|
|
print("")
|
|
|
|
# Select Action
|
|
print("Do you want to Add or Remove the user from the group?")
|
|
retry = True
|
|
while retry:
|
|
action = raw_input('add/remove: ')
|
|
if action == '':
|
|
return
|
|
elif action.lower() in ['add', 'a', 'insert']:
|
|
action = 'add'
|
|
retry = False
|
|
elif action.lower() in ['remove', 'r', 'rmv', 'rem', 'delete', 'del']:
|
|
action = 'remove'
|
|
retry = False
|
|
else:
|
|
print("Incorrect action, press type 'add' or 'remove'")
|
|
|
|
# Select User
|
|
retry = True
|
|
while retry:
|
|
user_email = raw_input('User email: ')
|
|
if user_email == '':
|
|
return
|
|
user = db.users.find_one({'email': user_email})
|
|
if user:
|
|
retry = False
|
|
else:
|
|
print("Incorrect user email, try again, or leave empty to cancel")
|
|
|
|
# Select group
|
|
retry = True
|
|
while retry:
|
|
group_name = raw_input('Group name: ')
|
|
if group_name == '':
|
|
return
|
|
group = db.groups.find_one({'name': group_name})
|
|
if group:
|
|
retry = False
|
|
else:
|
|
print("Incorrect group name, try again, or leave empty to cancel")
|
|
|
|
# Do
|
|
current_groups = user.get('groups', [])
|
|
if action == 'add':
|
|
if group['_id'] in current_groups:
|
|
print("User {0} is already in group {1}".format(
|
|
user_email, group_name))
|
|
else:
|
|
current_groups.append(group['_id'])
|
|
db.users.update({'_id': user['_id']},
|
|
{"$set": {'groups': current_groups}})
|
|
print("User {0} added to group {1}".format(user_email, group_name))
|
|
elif action == 'remove':
|
|
if group['_id'] not in current_groups:
|
|
print("User {0} is not in group {1}".format(user_email, group_name))
|
|
else:
|
|
current_groups.remove(group['_id'])
|
|
db.users.update({'_id': user['_id']},
|
|
{"$set": {'groups': current_groups}})
|
|
print("User {0} removed from group {1}".format(
|
|
user_email, group_name))
|
|
|
|
|
|
def populate_node_types(old_ids={}):
|
|
node_types_collection = app.data.driver.db['node_types']
|
|
|
|
def mix_node_type(old_id, node_type_dict):
|
|
# Take eve parameters
|
|
node_type = node_types_collection.find_one({'_id': old_id})
|
|
for attr in node_type:
|
|
if attr[0] == '_':
|
|
# Mix with node eve attributes. This is really not needed since
|
|
# the attributes are stripped before doing a put_internal.
|
|
node_type_dict[attr] = node_type[attr]
|
|
elif attr == 'permissions':
|
|
node_type_dict['permissions'] = node_type['permissions']
|
|
return node_type_dict
|
|
|
|
def upgrade(node_type, old_ids):
|
|
print("Node {0}".format(node_type['name']))
|
|
node_name = node_type['name']
|
|
if node_name in old_ids:
|
|
node_id = old_ids[node_name]
|
|
node_type = mix_node_type(node_id, node_type)
|
|
|
|
# Removed internal fields that would cause validation error
|
|
internal_fields = ['_id', '_etag', '_updated', '_created']
|
|
for field in internal_fields:
|
|
node_type.pop(field, None)
|
|
p = put_internal('node_types', node_type, **{'_id': node_id})
|
|
else:
|
|
print("Making the node")
|
|
print(node_type)
|
|
post_item('node_types', node_type)
|
|
|
|
# upgrade(shot_node_type, old_ids)
|
|
# upgrade(task_node_type, old_ids)
|
|
# upgrade(scene_node_type, old_ids)
|
|
# upgrade(act_node_type, old_ids)
|
|
upgrade(node_type_project, old_ids)
|
|
upgrade(node_type_group, old_ids)
|
|
upgrade(node_type_asset, old_ids)
|
|
upgrade(node_type_storage, old_ids)
|
|
upgrade(node_type_comment, old_ids)
|
|
upgrade(node_type_blog, old_ids)
|
|
upgrade(node_type_post, old_ids)
|
|
upgrade(node_type_texture, old_ids)
|
|
upgrade(node_type_group_texture, old_ids)
|
|
|
|
|
|
@manager.command
|
|
def add_parent_to_nodes():
|
|
"""Find the parent of any node in the nodes collection"""
|
|
import codecs
|
|
import sys
|
|
|
|
UTF8Writer = codecs.getwriter('utf8')
|
|
sys.stdout = UTF8Writer(sys.stdout)
|
|
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
|
|
def find_parent_project(node):
|
|
if node and 'parent' in node:
|
|
parent = nodes_collection.find_one({'_id': node['parent']})
|
|
return find_parent_project(parent)
|
|
if node:
|
|
return node
|
|
else:
|
|
return None
|
|
|
|
nodes = nodes_collection.find()
|
|
nodes_index = 0
|
|
nodes_orphan = 0
|
|
for node in nodes:
|
|
nodes_index += 1
|
|
if node['node_type'] == ObjectId("55a615cfea893bd7d0489f2d"):
|
|
print(u"Skipping project node - {0}".format(node['name']))
|
|
else:
|
|
project = find_parent_project(node)
|
|
if project:
|
|
nodes_collection.update({'_id': node['_id']},
|
|
{"$set": {'project': project['_id']}})
|
|
print(u"{0} {1}".format(node['_id'], node['name']))
|
|
else:
|
|
nodes_orphan += 1
|
|
nodes_collection.remove({'_id': node['_id']})
|
|
print("Removed {0} {1}".format(node['_id'], node['name']))
|
|
|
|
print("Edited {0} nodes".format(nodes_index))
|
|
print("Orphan {0} nodes".format(nodes_orphan))
|
|
|
|
|
|
@manager.command
|
|
def remove_children_files():
|
|
"""Remove any file object with a parent field"""
|
|
files_collection = app.data.driver.db['files']
|
|
for f in files_collection.find():
|
|
if 'parent' in f:
|
|
file_id = f['_id']
|
|
# Delete child object
|
|
files_collection.remove({'_id': file_id})
|
|
print("deleted {0}".format(file_id))
|
|
|
|
|
|
@manager.command
|
|
def make_project_public(project_id):
|
|
"""Convert every node of a project from pending to public"""
|
|
|
|
DRY_RUN = False
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
for n in nodes_collection.find({'project': ObjectId(project_id)}):
|
|
n['properties']['status'] = 'published'
|
|
print(u"Publishing {0} {1}".format(n['_id'], n['name'].encode('ascii', 'ignore')))
|
|
if not DRY_RUN:
|
|
put_item('nodes', n)
|
|
|
|
|
|
@manager.command
|
|
def convert_assets_to_textures(project_id):
|
|
"""Get any node of type asset in a certain project and convert it to a
|
|
node_type texture.
|
|
"""
|
|
|
|
DRY_RUN = False
|
|
|
|
node_types_collection = app.data.driver.db['node_types']
|
|
files_collection = app.data.driver.db['files']
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
|
|
def parse_name(name):
|
|
"""Parse a texture name to infer properties"""
|
|
variation = 'col'
|
|
is_tileable = False
|
|
variations = ['_bump', '_spec', '_nor', '_col', '_translucency']
|
|
for v in variations:
|
|
if v in name:
|
|
variation = v[1:]
|
|
break
|
|
if '_tileable' in name:
|
|
is_tileable = True
|
|
return dict(variation=variation, is_tileable=is_tileable)
|
|
|
|
def make_texture_node(base_node, files, parent_id=None):
|
|
texture_node_type = node_types_collection.find_one({'name': 'texture'})
|
|
files_list = []
|
|
is_tileable = False
|
|
|
|
if parent_id is None:
|
|
parent_id = base_node['parent']
|
|
else:
|
|
print("Using provided parent {0}".format(parent_id))
|
|
|
|
# Create a list with all the file fariations for the texture
|
|
for f in files:
|
|
print("Processing {1} {0}".format(f['name'], f['_id']))
|
|
attributes = parse_name(f['name'])
|
|
if attributes['is_tileable']:
|
|
is_tileable = True
|
|
file_entry = dict(
|
|
file=f['properties']['file'],
|
|
is_tileable=attributes['is_tileable'],
|
|
map_type=attributes['variation'])
|
|
files_list.append(file_entry)
|
|
# Get the first file from the files list and use it as base for some
|
|
# node properties
|
|
first_file = files_collection.find_one({'_id': files[0]['properties']['file']})
|
|
if 'picture' in base_node and base_node['picture'] != None:
|
|
picture = base_node['picture']
|
|
else:
|
|
picture = first_file['_id']
|
|
if 'height' in first_file:
|
|
node = dict(
|
|
name=base_node['name'],
|
|
picture=picture,
|
|
parent=parent_id,
|
|
project=base_node['project'],
|
|
user=base_node['user'],
|
|
node_type=texture_node_type['_id'],
|
|
properties=dict(
|
|
status=base_node['properties']['status'],
|
|
files=files_list,
|
|
resolution="{0}x{1}".format(first_file['height'], first_file['width']),
|
|
is_tileable=is_tileable,
|
|
is_landscape=(first_file['height'] < first_file['width']),
|
|
aspect_ratio=round(
|
|
(first_file['width'] / first_file['height']), 2)
|
|
)
|
|
)
|
|
print("Making {0}".format(node['name']))
|
|
if not DRY_RUN:
|
|
p = post_internal('nodes', node)
|
|
if p[0]['_status'] == 'ERR':
|
|
import pprint
|
|
pprint.pprint(node)
|
|
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
|
|
for n in nodes_collection.find({'project': ObjectId(project_id)}):
|
|
n_type = node_types_collection.find_one({'_id': n['node_type']})
|
|
processed_nodes = []
|
|
if n_type['name'] == 'group' and n['name'].startswith('_'):
|
|
print("Processing {0}".format(n['name']))
|
|
# Get the content of the group
|
|
children = [c for c in nodes_collection.find({'parent': n['_id']})]
|
|
make_texture_node(children[0], children, parent_id=n['parent'])
|
|
processed_nodes += children
|
|
processed_nodes.append(n)
|
|
elif n_type['name'] == 'group':
|
|
# Change group type to texture group
|
|
node_type_texture = node_types_collection.find_one(
|
|
{'name': 'group_texture'})
|
|
n['node_type'] = node_type_texture['_id']
|
|
n['properties'].pop('notes', None)
|
|
print("Updating {0}".format(n['name']))
|
|
if not DRY_RUN:
|
|
put_item('nodes', n)
|
|
# Delete processed nodes
|
|
for node in processed_nodes:
|
|
print("Removing {0} {1}".format(node['_id'], node['name']))
|
|
if not DRY_RUN:
|
|
nodes_collection.remove({'_id': node['_id']})
|
|
# Make texture out of single image
|
|
for n in nodes_collection.find({'project': ObjectId(project_id)}):
|
|
n_type = node_types_collection.find_one({'_id': n['node_type']})
|
|
if n_type['name'] == 'asset':
|
|
make_texture_node(n, [n])
|
|
# Delete processed nodes
|
|
print("Removing {0} {1}".format(n['_id'], n['name']))
|
|
if not DRY_RUN:
|
|
nodes_collection.remove({'_id': n['_id']})
|
|
|
|
|
|
@manager.command
|
|
def set_attachment_names():
|
|
"""Loop through all existing nodes and assign proper ContentDisposition
|
|
metadata to referenced files that are using GCS.
|
|
"""
|
|
from application import update_file_name
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
for n in nodes_collection.find():
|
|
print("Updating node {0}".format(n['_id']))
|
|
update_file_name(n)
|
|
|
|
|
|
@manager.command
|
|
def files_verify_project():
|
|
"""Verify for missing or conflicting node/file ids"""
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
files_collection = app.data.driver.db['files']
|
|
issues = dict(missing=[], conflicting=[], processing=[])
|
|
|
|
def _parse_file(item, file_id):
|
|
f = files_collection.find_one({'_id': file_id})
|
|
if f:
|
|
if 'project' in item and 'project' in f:
|
|
if item['project'] != f['project']:
|
|
issues['conflicting'].append(item['_id'])
|
|
if 'status' in item['properties'] \
|
|
and item['properties']['status'] == 'processing':
|
|
issues['processing'].append(item['_id'])
|
|
else:
|
|
issues['missing'].append(
|
|
"{0} missing {1}".format(item['_id'], file_id))
|
|
|
|
for item in nodes_collection.find():
|
|
print("Verifying node {0}".format(item['_id']))
|
|
if 'file' in item['properties']:
|
|
_parse_file(item, item['properties']['file'])
|
|
elif 'files' in item['properties']:
|
|
for f in item['properties']['files']:
|
|
_parse_file(item, f['file'])
|
|
|
|
print("===")
|
|
print("Issues detected:")
|
|
for k, v in issues.iteritems():
|
|
print("{0}:".format(k))
|
|
for i in v:
|
|
print(i)
|
|
print("===")
|
|
|
|
|
|
def replace_node_type(project, node_type_name, new_node_type):
|
|
"""Update or create the specified node type. We rely on the fact that
|
|
node_types have a unique name in a project.
|
|
"""
|
|
|
|
old_node_type = next(
|
|
(item for item in project['node_types'] if item.get('name') \
|
|
and item['name'] == node_type_name), None)
|
|
if old_node_type:
|
|
for i, v in enumerate(project['node_types']):
|
|
if v['name'] == node_type_name:
|
|
project['node_types'][i] = new_node_type
|
|
else:
|
|
project['node_types'].append(new_node_type)
|
|
|
|
|
|
@manager.command
|
|
def project_upgrade_node_types(project_id):
|
|
projects_collection = app.data.driver.db['projects']
|
|
project = projects_collection.find_one({'_id': ObjectId(project_id)})
|
|
replace_node_type(project, 'group', node_type_group)
|
|
replace_node_type(project, 'asset', node_type_asset)
|
|
replace_node_type(project, 'storage', node_type_storage)
|
|
replace_node_type(project, 'comment', node_type_comment)
|
|
replace_node_type(project, 'blog', node_type_blog)
|
|
replace_node_type(project, 'post', node_type_post)
|
|
replace_node_type(project, 'texture', node_type_texture)
|
|
put_item('projects', project)
|
|
|
|
|
|
@manager.command
|
|
def test_put_item(node_id):
|
|
import pprint
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
node = nodes_collection.find_one(ObjectId(node_id))
|
|
pprint.pprint(node)
|
|
put_item('nodes', node)
|
|
|
|
|
|
@manager.command
|
|
def test_post_internal(node_id):
|
|
import pprint
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
node = nodes_collection.find_one(ObjectId(node_id))
|
|
internal_fields = ['_id', '_etag', '_updated', '_created']
|
|
for field in internal_fields:
|
|
node.pop(field, None)
|
|
pprint.pprint(node)
|
|
print(post_internal('nodes', node))
|
|
|
|
|
|
@manager.command
|
|
def algolia_push_users():
|
|
"""Loop through all users and push them to Algolia"""
|
|
from application.utils.algolia import algolia_index_user_save
|
|
users_collection = app.data.driver.db['users']
|
|
for user in users_collection.find():
|
|
print("Pushing {0}".format(user['username']))
|
|
algolia_index_user_save(user)
|
|
|
|
|
|
@manager.command
|
|
def algolia_push_nodes():
|
|
"""Loop through all nodes and push them to Algolia"""
|
|
from application.utils.algolia import algolia_index_node_save
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
for node in nodes_collection.find():
|
|
print(u"Pushing {0}: {1}".format(node['_id'], node['name'].encode(
|
|
'ascii', 'ignore')))
|
|
algolia_index_node_save(node)
|
|
|
|
|
|
@manager.command
|
|
def files_make_public_t():
|
|
"""Loop through all files and if they are images on GCS, make the size t
|
|
public
|
|
"""
|
|
from gcloud.exceptions import InternalServerError
|
|
from application.utils.gcs import GoogleCloudStorageBucket
|
|
files_collection = app.data.driver.db['files']
|
|
|
|
for f in files_collection.find({'backend': 'gcs'}):
|
|
if 'variations' not in f:
|
|
continue
|
|
|
|
variation_t = next((item for item in f['variations']
|
|
if item['size'] == 't'), None)
|
|
if not variation_t:
|
|
continue
|
|
|
|
try:
|
|
storage = GoogleCloudStorageBucket(str(f['project']))
|
|
blob = storage.Get(variation_t['file_path'], to_dict=False)
|
|
if not blob:
|
|
print('Unable to find blob for project %s file %s' %(f['project'], f['_id']))
|
|
continue
|
|
|
|
print('Making blob public: {0}'.format(blob.path))
|
|
blob.make_public()
|
|
except InternalServerError as ex:
|
|
print('Internal Server Error: ', ex)
|
|
|
|
|
|
@manager.command
|
|
def subscribe_node_owners():
|
|
"""Automatically subscribe node owners to notifications for items created
|
|
in the past.
|
|
"""
|
|
from application import after_inserting_nodes
|
|
nodes_collection = app.data.driver.db['nodes']
|
|
for n in nodes_collection.find():
|
|
if 'parent' in n:
|
|
after_inserting_nodes([n])
|
|
|
|
|
|
@manager.command
|
|
def refresh_project_links(project, chunk_size=50, quiet=False):
|
|
"""Regenerates almost-expired file links for a certain project."""
|
|
|
|
if quiet:
|
|
import logging
|
|
from application import log
|
|
|
|
logging.getLogger().setLevel(logging.WARNING)
|
|
log.setLevel(logging.WARNING)
|
|
|
|
chunk_size = int(chunk_size) # CLI parameters are passed as strings
|
|
from application.modules import file_storage
|
|
file_storage.refresh_links_for_project(project, chunk_size, 2 * 3600)
|
|
|
|
|
|
@manager.command
|
|
def refresh_backend_links(backend_name, chunk_size=50, quiet=False):
|
|
"""Refreshes all file links that are using a certain storage backend."""
|
|
|
|
if quiet:
|
|
import logging
|
|
from application import log
|
|
|
|
logging.getLogger().setLevel(logging.WARNING)
|
|
log.setLevel(logging.WARNING)
|
|
|
|
chunk_size = int(chunk_size) # CLI parameters are passed as strings
|
|
from application.modules import file_storage
|
|
file_storage.refresh_links_for_backend(backend_name, chunk_size, 2 * 3600)
|
|
|
|
|
|
@manager.command
|
|
def expire_all_project_links(project_uuid):
|
|
"""Expires all file links for a certain project without refreshing.
|
|
|
|
This is just for testing.
|
|
"""
|
|
|
|
import datetime
|
|
import bson.tz_util
|
|
|
|
files_collection = app.data.driver.db['files']
|
|
|
|
now = datetime.datetime.now(tz=bson.tz_util.utc)
|
|
expires = now - datetime.timedelta(days=1)
|
|
|
|
result = files_collection.update_many(
|
|
{'project': ObjectId(project_uuid)},
|
|
{'$set': {'link_expires': expires}}
|
|
)
|
|
|
|
print('Expired %i links' % result.matched_count)
|
|
|
|
|
|
@manager.command
|
|
def register_local_user(email, password):
|
|
from application.modules.local_auth import create_local_user
|
|
create_local_user(email, password)
|
|
|
|
|
|
@manager.command
|
|
def add_group_to_projects(group_name):
|
|
"""Prototype to add a specific group, in read-only mode, to all node_types
|
|
for all projects.
|
|
"""
|
|
methods = ['GET']
|
|
groups_collection = app.data.driver.db['groups']
|
|
projects_collections = app.data.driver.db['projects']
|
|
group = groups_collection.find_one({'name': group_name})
|
|
for project in projects_collections.find():
|
|
print("Processing: {}".format(project['name']))
|
|
for node_type in project['node_types']:
|
|
node_type_name = node_type['name']
|
|
base_node_types = ['group', 'asset', 'blog', 'post', 'page',
|
|
'comment', 'group_texture', 'storage', 'texture']
|
|
if node_type_name in base_node_types:
|
|
print("Processing: {0}".format(node_type_name))
|
|
# Check if group already exists in the permissions
|
|
g = next((g for g in node_type['permissions']['groups']
|
|
if g['group'] == group['_id']), None)
|
|
# If not, we add it
|
|
if g is None:
|
|
print("Adding permissions")
|
|
permissions = {
|
|
'group': group['_id'],
|
|
'methods': methods}
|
|
node_type['permissions']['groups'].append(permissions)
|
|
projects_collections.update(
|
|
{'_id': project['_id']}, project)
|
|
|
|
|
|
@manager.command
|
|
def add_license_props():
|
|
"""Add license fields to all node types asset for every project."""
|
|
projects_collections = app.data.driver.db['projects']
|
|
for project in projects_collections.find():
|
|
print("Processing {}".format(project['_id']))
|
|
for node_type in project['node_types']:
|
|
if node_type['name'] == 'asset':
|
|
node_type['dyn_schema']['license_notes'] = {'type': 'string'}
|
|
node_type['dyn_schema']['license_type'] = {
|
|
'type': 'string',
|
|
'allowed': [
|
|
'cc-by',
|
|
'cc-0',
|
|
'cc-by-sa',
|
|
'cc-by-nd',
|
|
'cc-by-nc',
|
|
'copyright'
|
|
],
|
|
'default': 'cc-by'
|
|
}
|
|
node_type['form_schema']['license_notes'] = ''
|
|
node_type['form_schema']['license_type'] = ''
|
|
projects_collections.update(
|
|
{'_id': project['_id']}, project)
|
|
|
|
|
|
@manager.command
|
|
def refresh_file_sizes():
|
|
"""Computes & stores the 'length_aggregate_in_bytes' fields of all files."""
|
|
|
|
from application.modules import file_storage
|
|
|
|
matched = 0
|
|
unmatched = 0
|
|
total_size = 0
|
|
|
|
files_collection = app.data.driver.db['files']
|
|
for file_doc in files_collection.find():
|
|
file_storage.compute_aggregate_length(file_doc)
|
|
length = file_doc['length_aggregate_in_bytes']
|
|
total_size += length
|
|
|
|
result = files_collection.update_one({'_id': file_doc['_id']},
|
|
{'$set': {'length_aggregate_in_bytes': length}})
|
|
if result.matched_count != 1:
|
|
log.warning('Unable to update document %s', file_doc['_id'])
|
|
unmatched += 1
|
|
else:
|
|
matched += 1
|
|
|
|
log.info('Updated %i file documents.', matched)
|
|
if unmatched:
|
|
log.warning('Unable to update %i documents.', unmatched)
|
|
log.info('%i bytes (%.3f GiB) storage used in total.',
|
|
total_size, total_size / 1024 ** 3)
|
|
|
|
|
|
@manager.command
|
|
def project_stats():
|
|
import csv
|
|
import sys
|
|
from collections import defaultdict
|
|
from functools import partial
|
|
|
|
from application.modules import projects
|
|
|
|
proj_coll = app.data.driver.db['projects']
|
|
nodes = app.data.driver.db['nodes']
|
|
|
|
aggr = defaultdict(partial(defaultdict, int))
|
|
|
|
csvout = csv.writer(sys.stdout)
|
|
csvout.writerow(['project ID', 'owner', 'private', 'file size',
|
|
'nr of nodes', 'nr of top-level nodes', ])
|
|
|
|
for proj in proj_coll.find(projection={'user': 1,
|
|
'name': 1,
|
|
'is_private': 1,
|
|
'_id': 1}):
|
|
project_id = proj['_id']
|
|
is_private = proj.get('is_private', False)
|
|
row = [str(project_id),
|
|
unicode(proj['user']).encode('utf-8'),
|
|
is_private]
|
|
|
|
file_size = projects.project_total_file_size(project_id)
|
|
row.append(file_size)
|
|
|
|
node_count_result = nodes.aggregate([
|
|
{'$match': {'project': project_id}},
|
|
{'$project': {'parent': 1,
|
|
'is_top': {'$cond': [{'$gt': ['$parent', None]}, 0, 1]},
|
|
}},
|
|
{'$group': {
|
|
'_id': None,
|
|
'all': {'$sum': 1},
|
|
'top': {'$sum': '$is_top'},
|
|
}}
|
|
])
|
|
|
|
try:
|
|
node_counts = next(node_count_result)
|
|
nodes_all = node_counts['all']
|
|
nodes_top = node_counts['top']
|
|
except StopIteration:
|
|
# No result from the nodes means nodeless project.
|
|
nodes_all = 0
|
|
nodes_top = 0
|
|
row.append(nodes_all)
|
|
row.append(nodes_top)
|
|
|
|
for collection in aggr[None], aggr[is_private]:
|
|
collection['project_count'] += 1
|
|
collection['project_count'] += 1
|
|
collection['file_size'] += file_size
|
|
collection['node_count'] += nodes_all
|
|
collection['top_nodes'] += nodes_top
|
|
|
|
csvout.writerow(row)
|
|
|
|
csvout.writerow([
|
|
'public', '', '%i projects' % aggr[False]['project_count'],
|
|
aggr[False]['file_size'], aggr[False]['node_count'], aggr[False]['top_nodes'],
|
|
])
|
|
csvout.writerow([
|
|
'private', '', '%i projects' % aggr[True]['project_count'],
|
|
aggr[True]['file_size'], aggr[True]['node_count'], aggr[True]['top_nodes'],
|
|
])
|
|
csvout.writerow([
|
|
'total', '', '%i projects' % aggr[None]['project_count'],
|
|
aggr[None]['file_size'], aggr[None]['node_count'], aggr[None]['top_nodes'],
|
|
])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
manager.run()
|