From 9e6df4aefada22ab74b9cc192bc7a97a361bdfbc Mon Sep 17 00:00:00 2001 From: Eibriel Date: Mon, 16 Mar 2015 21:44:34 +0100 Subject: [PATCH] Starting cleanup of old settings Moving Server Storage to Public Storage location set automatically --- .gitignore | 5 ++- brender/manager/application/__init__.py | 17 ++++---- brender/manager/application/config.py.example | 3 +- .../application/modules/tasks/__init__.py | 5 ++- .../application/static/storage/placeholder | 0 .../task_compilers/simple_blender_render.py | 11 +++-- brender/manager/manage.py | 31 +++++++------- brender/server/application/__init__.py | 5 ++- brender/server/application/config.py.example | 3 +- .../job_compilers/simple_blender_render.py | 16 ++++--- .../application/modules/jobs/__init__.py | 2 +- .../application/modules/projects/__init__.py | 42 +++++++++++-------- .../application/modules/settings/__init__.py | 5 ++- .../application/modules/tasks/__init__.py | 10 +++-- .../application/static/storage/placeholder | 0 brender/server/manage.py | 22 ++++++---- brender/worker/application/controllers.py | 2 + plugins/blender/render_brender.py | 6 +-- 18 files changed, 108 insertions(+), 77 deletions(-) create mode 100644 brender/manager/application/static/storage/placeholder create mode 100644 brender/server/application/static/storage/placeholder diff --git a/.gitignore b/.gitignore index 8bfe6b47..b964497d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,9 @@ /temp/* venv/ *.log -.ropeproject/* +*.ropeproject* config.py /brender/config.py /brender/server/render -/brender/manager/application/static/storage +/brender/manager/application/static/storage/* +/brender/server/application/static/storage/* diff --git a/brender/manager/application/__init__.py b/brender/manager/application/__init__.py index 8cfa758a..ccbbc0fd 100644 --- a/brender/manager/application/__init__.py +++ b/brender/manager/application/__init__.py @@ -32,7 +32,7 @@ try: ) if not config.Config.IS_PRIVATE_MANAGER: - try: + """try: server_settings = http_request(app.config['BRENDER_SERVER'], '/settings', 'get') app.config.update( BLENDER_PATH_LINUX=server_settings['blender_path_linux'], @@ -47,7 +47,7 @@ try: exit(3) except KeyError: logging.error("Please, configure Brender Paths browsing Dashboard->Server->Settings") - exit(3) + exit(3)""" else: app.config.update( BLENDER_PATH_LINUX=config.Config.BLENDER_PATH_LINUX, @@ -68,9 +68,10 @@ except ImportError: app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(os.path.dirname(__file__), '../task_queue.sqlite') app.config['TMP_FOLDER'] = tempfile.gettempdir() app.config['THUMBNAIL_EXTENSIONS'] = set(['png']) - app.config['MANAGER_STORAGE'] = tempfile.gettempdir() + app.config['MANAGER_STORAGE'] = '{0}/static/storage'.format( + os.path.join(os.path.dirname(__file__))) - try: + """try: server_settings = http_request(app.config['BRENDER_SERVER'], '/settings', 'get') app.config.update( BLENDER_PATH_LINUX=server_settings['blender_path_linux'], @@ -85,7 +86,7 @@ except ImportError: exit(3) except KeyError: logging.error("Please, configure Brender Paths browsing Dashboard->Server->Settings") - exit(3) + exit(3)""" api = Api(app) @@ -124,14 +125,16 @@ def register_manager(port, name, has_virtual_workers): registering the render node. This is called by the runserver script. """ import httplib + import socket + import time while True: try: connection = httplib.HTTPConnection(app.config['BRENDER_SERVER']) connection.request("GET", "/managers") break except socket.error: - pass - time.sleep(0.1) + print ("Cant connect with Server, retrying...") + time.sleep(1) params = { 'port' : port, diff --git a/brender/manager/application/config.py.example b/brender/manager/application/config.py.example index 24da6a4f..cfc10436 100644 --- a/brender/manager/application/config.py.example +++ b/brender/manager/application/config.py.example @@ -22,4 +22,5 @@ class Config(object): SETTINGS_PATH_WIN = "" TMP_FOLDER = '/tmp/' THUMBNAIL_EXTENSIONS = set(['png']) - MANAGER_STORAGE = '/path/to/brender/manager/application/static/storage' + MANAGER_STORAGE = '{0}/static/storage'.format( + os.path.join(os.path.dirname(__file__))) diff --git a/brender/manager/application/modules/tasks/__init__.py b/brender/manager/application/modules/tasks/__init__.py index b8154d37..1257fcc6 100644 --- a/brender/manager/application/modules/tasks/__init__.py +++ b/brender/manager/application/modules/tasks/__init__.py @@ -161,8 +161,9 @@ class TaskCompiledApi(Resource): f.write("locked") r = requests.get( - 'http://{0}/jobs/file/{1}'.format( - app.config['BRENDER_SERVER'], task['job_id']) + #'http://{0}/jobs/file/{1}'.format( + 'http://{0}/static/storage/{1}/{2}/jobfile_{2}.zip'.format( + app.config['BRENDER_SERVER'], task['project_id'], task['job_id']) ) with open(tmpfile, 'wb') as f: diff --git a/brender/manager/application/static/storage/placeholder b/brender/manager/application/static/storage/placeholder new file mode 100644 index 00000000..e69de29b diff --git a/brender/manager/application/task_compilers/simple_blender_render.py b/brender/manager/application/task_compilers/simple_blender_render.py index 7fbffd5b..977188c6 100644 --- a/brender/manager/application/task_compilers/simple_blender_render.py +++ b/brender/manager/application/task_compilers/simple_blender_render.py @@ -1,8 +1,8 @@ import os import json -import logging +#import logging -from application import app +#from application import app class task_compiler(): @staticmethod @@ -10,7 +10,7 @@ class task_compiler(): settings=json.loads(task['settings']) - if 'Darwin' in worker.system: + """if 'Darwin' in worker.system: setting_blender_path = app.config['BLENDER_PATH_OSX'] setting_render_settings = app.config['SETTINGS_PATH_OSX'] file_path = settings['file_path_osx'] @@ -30,8 +30,6 @@ class task_compiler(): logging.info('[Debug] blender path is not set') return None - blender_path = setting_blender_path - if setting_render_settings is None: logging.warning("Render settings path not set!") return None @@ -39,13 +37,14 @@ class task_compiler(): setting_render_settings = app.config['SETTINGS_PATH_LINUX'] render_settings = os.path.join( setting_render_settings, - settings['render_settings']) + settings['render_settings'])""" # TODO file_path = os.path.split(settings['file_path_linux'])[1] file_path = os.path.join('==jobpath==', file_path) output_path = "==outputpath==" + blender_path = "==blenderpath==" dir = os.path.dirname(__file__) template_path = os.path.join(dir, 'simple_blender_render.template') diff --git a/brender/manager/manage.py b/brender/manager/manage.py index cf8fa5b7..e02f1231 100755 --- a/brender/manager/manage.py +++ b/brender/manager/manage.py @@ -7,6 +7,10 @@ from threading import Thread from flask.ext.script import Manager from flask.ext.migrate import MigrateCommand +from flask.ext.migrate import upgrade + +from sqlalchemy import create_engine +from alembic.migration import MigrationContext from application import app from application import register_manager @@ -19,14 +23,16 @@ manager.add_command('db', MigrateCommand) def runserver(): """This command is meant for development. If no configuration is found, we start the app listening from all hosts, from port 7777.""" - #Testing Database - from application.modules.settings.model import Setting - from sqlalchemy.exc import OperationalError - try: - Setting.query.first() - except OperationalError: - logging.error("Please run \"python manager.py db upgrade\" to initialize the database") - exit(3) + # Testig Alembic + engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI']) + conn = engine.connect() + context = MigrationContext.configure(conn) + current_ver = context.get_current_revision() + if not current_ver: + print("Automatic DB Upgrade") + print("Press Ctrl+C when finished") + upgrade() + print("Upgrade completed") try: from application import config @@ -43,18 +49,13 @@ def runserver(): HOSTNAME = socket.gethostname() - # Use multiprocessing to register the manager to the server - # while the manager app starts up + # Register the manager to the server if os.environ.get('WERKZEUG_RUN_MAIN') != 'true': if VIRTUAL_WORKERS: has_virtual_worker = 1 else: has_virtual_worker = 0 - register_thread = Thread( - target=register_manager, - args=(PORT, HOSTNAME, has_virtual_worker)) - register_thread.setDaemon(False) - register_thread.start() + register_manager(PORT, HOSTNAME, has_virtual_worker) app.run( port=PORT, diff --git a/brender/server/application/__init__.py b/brender/server/application/__init__.py index fa8fac0d..26cacc42 100644 --- a/brender/server/application/__init__.py +++ b/brender/server/application/__init__.py @@ -22,12 +22,13 @@ try: app.config['THUMBNAIL_EXTENSIONS']= config.Config.THUMBNAIL_EXTENSIONS app.config['SERVER_STORAGE'] = config.Config.SERVER_STORAGE except ImportError: - from modules.managers.model import Manager + #from modules.managers.model import Manager app.config.update( SQLALCHEMY_DATABASE_URI='sqlite:///' + os.path.join(os.path.dirname(__file__), '../brender.sqlite'), TMP_FOLDER=tempfile.gettempdir(), THUMBNAIL_EXTENSIONS=set(['png']), - SERVER_STORAGE = tempfile.gettempdir() + SERVER_STORAGE = '{0}/static/storage'.format( + os.path.join(os.path.dirname(__file__))) ) api = Api(app) diff --git a/brender/server/application/config.py.example b/brender/server/application/config.py.example index 63e440b2..352e023e 100644 --- a/brender/server/application/config.py.example +++ b/brender/server/application/config.py.example @@ -7,4 +7,5 @@ class Config(object): HOST='0.0.0.0' # or 'localhost' TMP_FOLDER = '/tmp/' THUMBNAIL_EXTENSIONS = set(['png']) - SERVER_STORAGE = '/path/to/server_storage' + SERVER_STORAGE = '{0}/static/storage'.format( + os.path.join(os.path.dirname(__file__))) diff --git a/brender/server/application/job_compilers/simple_blender_render.py b/brender/server/application/job_compilers/simple_blender_render.py index d2e09b81..048513f4 100644 --- a/brender/server/application/job_compilers/simple_blender_render.py +++ b/brender/server/application/job_compilers/simple_blender_render.py @@ -15,15 +15,21 @@ class job_compiler(): task_settings['format'] = job_settings['format'] #project = Project.query.filter_by(id = job.project_id).first() - filepath = task_settings['filepath'] + #filepath = task_settings['filepath'] - task_settings['file_path_linux'] = os.path.join(project.path_linux, filepath) + """task_settings['file_path_linux'] = os.path.join(project.path_linux, filepath) task_settings['file_path_win'] = os.path.join(project.path_win, filepath) - task_settings['file_path_osx'] = os.path.join(project.path_osx, filepath) + task_settings['file_path_osx'] = os.path.join(project.path_osx, filepath)""" + task_settings['file_path_linux'] = "" + task_settings['file_path_win'] = "" + task_settings['file_path_osx'] = "" #task_settings['settings'] = task.settings - task_settings['output_path_linux'] = os.path.join(project.render_path_linux, str(job.id), '#####') + """task_settings['output_path_linux'] = os.path.join(project.render_path_linux, str(job.id), '#####') task_settings['output_path_win'] = os.path.join(project.render_path_win, str(job.id), '#####') - task_settings['output_path_osx'] = os.path.join(project.render_path_osx, str(job.id), '#####') + task_settings['output_path_osx'] = os.path.join(project.render_path_osx, str(job.id), '#####')""" + task_settings['output_path_linux'] = '#####' + task_settings['output_path_win'] = '#####' + task_settings['output_path_osx'] = '#####' task_settings['priority'] = job.priority #Chunk Generation diff --git a/brender/server/application/modules/jobs/__init__.py b/brender/server/application/modules/jobs/__init__.py index 283038dc..c0583fdb 100644 --- a/brender/server/application/modules/jobs/__init__.py +++ b/brender/server/application/modules/jobs/__init__.py @@ -426,7 +426,7 @@ class JobDeleteApi(Resource): TaskApi.delete_tasks(j) job = Job.query.get(j) if job: - path = os.path.join(job.project.render_path_server, str(j)) + #path = os.path.join(job.project.render_path_server, str(j)) #Security check #insecure_names=[None, "", "/", "\\", ".", ".."] #if job.project.render_path_server not in insecure_names and str(j) not in insecure_names: diff --git a/brender/server/application/modules/projects/__init__.py b/brender/server/application/modules/projects/__init__.py index 955675a3..4316f8c8 100644 --- a/brender/server/application/modules/projects/__init__.py +++ b/brender/server/application/modules/projects/__init__.py @@ -41,32 +41,38 @@ project_fields = { class ProjectListApi(Resource): def get(self): projects = {} + count = Project.query.count() + if count == 0: + project = Project(name="Default Project") + db.session.add(project) + db.session.commit() + for project in Project.query.all(): projects[project.id] = dict( - name=project.name, - path_server=project.path_server, - path_linux=project.path_linux, - path_win=project.path_win, - path_osx=project.path_osx, - render_path_server=project.render_path_server, - render_path_linux=project.render_path_linux, - render_path_win=project.render_path_win, - render_path_osx=project.render_path_osx) + name=project.name) + """path_server=project.path_server, + path_linux=project.path_linux, + path_win=project.path_win, + path_osx=project.path_osx, + render_path_server=project.render_path_server, + render_path_linux=project.render_path_linux, + render_path_win=project.render_path_win, + render_path_osx=project.render_path_osx)""" return jsonify(projects) @marshal_with(project_fields) def post(self): args = parser.parse_args() project = Project( - name=args['name'], - path_server=args['path_server'], - path_linux=args['path_linux'], - path_win=args['path_win'], - path_osx=args['path_osx'], - render_path_server=args['render_path_server'], - render_path_linux=args['render_path_linux'], - render_path_win=args['render_path_win'], - render_path_osx=args['render_path_osx']) + name=args['name']) + """path_server=args['path_server'], + path_linux=args['path_linux'], + path_win=args['path_win'], + path_osx=args['path_osx'], + render_path_server=args['render_path_server'], + render_path_linux=args['render_path_linux'], + render_path_win=args['render_path_win'], + render_path_osx=args['render_path_osx'])""" db.session.add(project) db.session.commit() diff --git a/brender/server/application/modules/settings/__init__.py b/brender/server/application/modules/settings/__init__.py index c4fd5e8c..a5594181 100644 --- a/brender/server/application/modules/settings/__init__.py +++ b/brender/server/application/modules/settings/__init__.py @@ -46,7 +46,7 @@ class SettingsListApi(Resource): class RenderSettingsApi(Resource): def get(self): - name = '' + """name = '' if system() == 'Linux': name = 'render_settings_path_linux' elif system() == 'Windows': @@ -56,4 +56,5 @@ class RenderSettingsApi(Resource): path = Setting.query.filter_by(name=name).first() onlyfiles = [f for f in listdir(path.value) if isfile(join(path.value, f))] settings_files = dict(settings_files=onlyfiles) - return jsonify(settings_files) + return jsonify(settings_files)""" + return {'':''} diff --git a/brender/server/application/modules/tasks/__init__.py b/brender/server/application/modules/tasks/__init__.py index cc3175f1..58e01745 100644 --- a/brender/server/application/modules/tasks/__init__.py +++ b/brender/server/application/modules/tasks/__init__.py @@ -247,7 +247,6 @@ class TaskApi(Resource): for man in managers: params = {'tasks': managers[man]} - #print (params) try: delete_task = http_rest_request( manager.host, @@ -256,7 +255,6 @@ class TaskApi(Resource): params=params) except: logging.info("Error deleting task from Manager") - #raise return pass task.status = 'ready' @@ -278,9 +276,7 @@ class TaskApi(Resource): for t in tasks: print t tasklist.append(t.id) - #map(lambda t : TaskApi.stop_task(t.id), tasks) TaskApi.stop_task(tasklist) - #TaskApi.delete_tasks(job_id) def get(self): @@ -292,6 +288,9 @@ class TaskApi(Resource): ip_address = request.remote_addr manager = Manager.query.filter_by(ip_address=ip_address).first() + if not manager: + return '', 404 + tasks = Task.query.filter( or_(Task.status == 'ready', Task.status=='failed'), @@ -315,6 +314,8 @@ class TaskApi(Resource): db.session.add(task) db.session.commit() + job = Job.query.get(task.job_id) + tasks = {} frame_count = 1 current_frame = 0 @@ -332,6 +333,7 @@ class TaskApi(Resource): "child_id": task.child_id, "parser": task.parser, "time_cost": task.time_cost, + "project_id": job.project_id, "chunk_start": 0, "chunk_end": 0, diff --git a/brender/server/application/static/storage/placeholder b/brender/server/application/static/storage/placeholder new file mode 100644 index 00000000..e69de29b diff --git a/brender/server/manage.py b/brender/server/manage.py index bb35a60f..43e62622 100755 --- a/brender/server/manage.py +++ b/brender/server/manage.py @@ -4,6 +4,10 @@ import logging from flask.ext.script import Manager from flask.ext.migrate import MigrateCommand +from flask.ext.migrate import current +from flask.ext.migrate import upgrade +from sqlalchemy import create_engine +from alembic.migration import MigrationContext from application import app from application import db @@ -17,14 +21,16 @@ def runserver(): """This command is meant for development. If no configuration is found, we start the app listening from all hosts, from port 9999.""" - #Testing Database - from application.modules.settings.model import Setting - from sqlalchemy.exc import OperationalError - try: - Setting.query.first() - except OperationalError: - logging.error("Please run \"python manager.py db upgrade\" to initialize the database") - exit(3) + # Testig Alembic + engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI']) + conn = engine.connect() + context = MigrationContext.configure(conn) + current_ver = context.get_current_revision() + if not current_ver: + print("Automatic DB Upgrade") + print("Press Ctrl+C when finished") + upgrade() + print("Upgrade completed") try: from application import config diff --git a/brender/worker/application/controllers.py b/brender/worker/application/controllers.py index ab9c7d9b..69ac9cee 100644 --- a/brender/worker/application/controllers.py +++ b/brender/worker/application/controllers.py @@ -458,6 +458,8 @@ def run_blender_in_thread(options): "==jobpath==",jobpath) render_command[cmd] = render_command[cmd].replace( "==outputpath==",outputpath) + render_command[cmd] = render_command[cmd].replace( + "==blenderpath==", "/shared/software/blender/blender_farm_latest/blender") os.environ['WORKER_DEPENDPATH'] = dependpath os.environ['WORKER_OUTPUTPATH'] = outputpath diff --git a/plugins/blender/render_brender.py b/plugins/blender/render_brender.py index f71c76c4..8f9e7865 100644 --- a/plugins/blender/render_brender.py +++ b/plugins/blender/render_brender.py @@ -74,7 +74,7 @@ class flamencoUpdate (bpy.types.Operator): try: projects = requests.get('{0}/projects'.format(serverurl), timeout=1) - settings = requests.get('{0}/settings/render'.format(serverurl), timeout=1) + #settings = requests.get('{0}/settings/render'.format(serverurl), timeout=1) managers = requests.get('{0}/managers'.format(serverurl), timeout=1) except ConnectionError: self.report( {'ERROR'}, "Can't connect to server on {0}".format(serverurl) ) @@ -152,7 +152,7 @@ class bamToRenderfarm (bpy.types.Operator): } print (job_properties) - + amaranth_addon = False try: scn.use_unsimplify_render @@ -166,7 +166,7 @@ class bamToRenderfarm (bpy.types.Operator): bpy.ops.wm.save_mainfile() scn.render.use_simplify = tmp_simplify - + tmppath = C.user_preferences.filepaths.temporary_directory zipname = "job" zippath = os.path.join(tmppath, "%s.zip" % zipname)