archive/flamenco-manager
Archived
0
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Starting cleanup of old settings

Browse Source 
Moving Server Storage to Public
Storage location set automatically
This commit is contained in:
Eibriel
2015-03-16 21:44:34 +01:00
parent 85d372985c
commit 9e6df4aefa
18 changed files with 108 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3 -2
View File
@@ -8,8 +8,9 @@
/temp/*
venv/
*.log
.ropeproject/*
*.ropeproject*
config.py
/brender/config.py
/brender/server/render
/brender/manager/application/static/storage
/brender/manager/application/static/storage/*
/brender/server/application/static/storage/*
brender/manager/application/__init__.py
+10 -7
View File
@@ -32,7 +32,7 @@ try:
)
if not config.Config.IS_PRIVATE_MANAGER:
try:
"""try:
server_settings = http_request(app.config['BRENDER_SERVER'], '/settings', 'get')
app.config.update(
BLENDER_PATH_LINUX=server_settings['blender_path_linux'],
@@ -47,7 +47,7 @@ try:
exit(3)
except KeyError:
logging.error("Please, configure Brender Paths browsing Dashboard->Server->Settings")
exit(3)
exit(3)"""
else:
app.config.update(
BLENDER_PATH_LINUX=config.Config.BLENDER_PATH_LINUX,
@@ -68,9 +68,10 @@ except ImportError:
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(os.path.dirname(__file__), '../task_queue.sqlite')
app.config['TMP_FOLDER'] = tempfile.gettempdir()
app.config['THUMBNAIL_EXTENSIONS'] = set(['png'])
app.config['MANAGER_STORAGE'] = tempfile.gettempdir()
app.config['MANAGER_STORAGE'] = '{0}/static/storage'.format(
os.path.join(os.path.dirname(__file__)))
try:
"""try:
server_settings = http_request(app.config['BRENDER_SERVER'], '/settings', 'get')
app.config.update(
BLENDER_PATH_LINUX=server_settings['blender_path_linux'],
@@ -85,7 +86,7 @@ except ImportError:
exit(3)
except KeyError:
logging.error("Please, configure Brender Paths browsing Dashboard->Server->Settings")
exit(3)
exit(3)"""
api = Api(app)
@@ -124,14 +125,16 @@ def register_manager(port, name, has_virtual_workers):
registering the render node. This is called by the runserver script.
"""
import httplib
import socket
import time
while True:
try:
connection = httplib.HTTPConnection(app.config['BRENDER_SERVER'])
connection.request("GET", "/managers")
break
except socket.error:
pass
time.sleep(0.1)
print ("Cant connect with Server, retrying...")
time.sleep(1)
params = {
'port' : port,
brender/manager/application/config.py.example
+2 -1
View File
@@ -22,4 +22,5 @@ class Config(object):
SETTINGS_PATH_WIN = ""
TMP_FOLDER = '/tmp/'
THUMBNAIL_EXTENSIONS = set(['png'])
MANAGER_STORAGE = '/path/to/brender/manager/application/static/storage'
MANAGER_STORAGE = '{0}/static/storage'.format(
os.path.join(os.path.dirname(__file__)))
brender/manager/application/modules/tasks/__init__.py
+3 -2
View File
@@ -161,8 +161,9 @@ class TaskCompiledApi(Resource):
f.write("locked")
r = requests.get(
'http://{0}/jobs/file/{1}'.format(
app.config['BRENDER_SERVER'], task['job_id'])
#'http://{0}/jobs/file/{1}'.format(
'http://{0}/static/storage/{1}/{2}/jobfile_{2}.zip'.format(
app.config['BRENDER_SERVER'], task['project_id'], task['job_id'])
)
with open(tmpfile, 'wb') as f:
brender/manager/application/static/storage/placeholder
View File
brender/manager/application/task_compilers/simple_blender_render.py
+5 -6
View File
@@ -1,8 +1,8 @@
import os
import json
import logging
#import logging
from application import app
#from application import app
class task_compiler():
@staticmethod
@@ -10,7 +10,7 @@ class task_compiler():
settings=json.loads(task['settings'])
if 'Darwin' in worker.system:
"""if 'Darwin' in worker.system:
setting_blender_path = app.config['BLENDER_PATH_OSX']
setting_render_settings = app.config['SETTINGS_PATH_OSX']
file_path = settings['file_path_osx']
@@ -30,8 +30,6 @@ class task_compiler():
logging.info('[Debug] blender path is not set')
return None
blender_path = setting_blender_path
if setting_render_settings is None:
logging.warning("Render settings path not set!")
return None
@@ -39,13 +37,14 @@ class task_compiler():
setting_render_settings = app.config['SETTINGS_PATH_LINUX']
render_settings = os.path.join(
setting_render_settings,
settings['render_settings'])
settings['render_settings'])"""
# TODO
file_path = os.path.split(settings['file_path_linux'])[1]
file_path = os.path.join('==jobpath==', file_path)
output_path = "==outputpath=="
blender_path = "==blenderpath=="
dir = os.path.dirname(__file__)
template_path = os.path.join(dir, 'simple_blender_render.template')
brender/manager/manage.py
+16 -15
View File
@@ -7,6 +7,10 @@ from threading import Thread
from flask.ext.script import Manager
from flask.ext.migrate import MigrateCommand
from flask.ext.migrate import upgrade
from sqlalchemy import create_engine
from alembic.migration import MigrationContext
from application import app
from application import register_manager
@@ -19,14 +23,16 @@ manager.add_command('db', MigrateCommand)
def runserver():
"""This command is meant for development. If no configuration is found,
we start the app listening from all hosts, from port 7777."""
#Testing Database
from application.modules.settings.model import Setting
from sqlalchemy.exc import OperationalError
try:
Setting.query.first()
except OperationalError:
logging.error("Please run \"python manager.py db upgrade\" to initialize the database")
exit(3)
# Testig Alembic
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
conn = engine.connect()
context = MigrationContext.configure(conn)
current_ver = context.get_current_revision()
if not current_ver:
print("Automatic DB Upgrade")
print("Press Ctrl+C when finished")
upgrade()
print("Upgrade completed")
try:
from application import config
@@ -43,18 +49,13 @@ def runserver():
HOSTNAME = socket.gethostname()
# Use multiprocessing to register the manager to the server
# while the manager app starts up
# Register the manager to the server
if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
if VIRTUAL_WORKERS:
has_virtual_worker = 1
else:
has_virtual_worker = 0
register_thread = Thread(
target=register_manager,
args=(PORT, HOSTNAME, has_virtual_worker))
register_thread.setDaemon(False)
register_thread.start()
register_manager(PORT, HOSTNAME, has_virtual_worker)
app.run(
port=PORT,
brender/server/application/__init__.py
+3 -2
View File
@@ -22,12 +22,13 @@ try:
app.config['THUMBNAIL_EXTENSIONS']= config.Config.THUMBNAIL_EXTENSIONS
app.config['SERVER_STORAGE'] = config.Config.SERVER_STORAGE
except ImportError:
from modules.managers.model import Manager
#from modules.managers.model import Manager
app.config.update(
SQLALCHEMY_DATABASE_URI='sqlite:///' + os.path.join(os.path.dirname(__file__), '../brender.sqlite'),
TMP_FOLDER=tempfile.gettempdir(),
THUMBNAIL_EXTENSIONS=set(['png']),
SERVER_STORAGE = tempfile.gettempdir()
SERVER_STORAGE = '{0}/static/storage'.format(
os.path.join(os.path.dirname(__file__)))
)
api = Api(app)
brender/server/application/config.py.example
+2 -1
View File
@@ -7,4 +7,5 @@ class Config(object):
HOST='0.0.0.0' # or 'localhost'
TMP_FOLDER = '/tmp/'
THUMBNAIL_EXTENSIONS = set(['png'])
SERVER_STORAGE = '/path/to/server_storage'
SERVER_STORAGE = '{0}/static/storage'.format(
os.path.join(os.path.dirname(__file__)))
brender/server/application/job_compilers/simple_blender_render.py
+11 -5
View File
@@ -15,15 +15,21 @@ class job_compiler():
task_settings['format'] = job_settings['format']
#project = Project.query.filter_by(id = job.project_id).first()
filepath = task_settings['filepath']
#filepath = task_settings['filepath']
task_settings['file_path_linux'] = os.path.join(project.path_linux, filepath)
"""task_settings['file_path_linux'] = os.path.join(project.path_linux, filepath)
task_settings['file_path_win'] = os.path.join(project.path_win, filepath)
task_settings['file_path_osx'] = os.path.join(project.path_osx, filepath)
task_settings['file_path_osx'] = os.path.join(project.path_osx, filepath)"""
task_settings['file_path_linux'] = ""
task_settings['file_path_win'] = ""
task_settings['file_path_osx'] = ""
#task_settings['settings'] = task.settings
task_settings['output_path_linux'] = os.path.join(project.render_path_linux, str(job.id), '#####')
"""task_settings['output_path_linux'] = os.path.join(project.render_path_linux, str(job.id), '#####')
task_settings['output_path_win'] = os.path.join(project.render_path_win, str(job.id), '#####')
task_settings['output_path_osx'] = os.path.join(project.render_path_osx, str(job.id), '#####')
task_settings['output_path_osx'] = os.path.join(project.render_path_osx, str(job.id), '#####')"""
task_settings['output_path_linux'] = '#####'
task_settings['output_path_win'] = '#####'
task_settings['output_path_osx'] = '#####'
task_settings['priority'] = job.priority
#Chunk Generation
brender/server/application/modules/jobs/__init__.py
+1 -1
View File
@@ -426,7 +426,7 @@ class JobDeleteApi(Resource):
TaskApi.delete_tasks(j)
job = Job.query.get(j)
if job:
path = os.path.join(job.project.render_path_server, str(j))
#path = os.path.join(job.project.render_path_server, str(j))
#Security check
#insecure_names=[None, "", "/", "\\", ".", ".."]
#if job.project.render_path_server not in insecure_names and str(j) not in insecure_names:
brender/server/application/modules/projects/__init__.py
+24 -18
View File
@@ -41,32 +41,38 @@ project_fields = {
class ProjectListApi(Resource):
def get(self):
projects = {}
count = Project.query.count()
if count == 0:
project = Project(name="Default Project")
db.session.add(project)
db.session.commit()
for project in Project.query.all():
projects[project.id] = dict(
name=project.name,
path_server=project.path_server,
path_linux=project.path_linux,
path_win=project.path_win,
path_osx=project.path_osx,
render_path_server=project.render_path_server,
render_path_linux=project.render_path_linux,
render_path_win=project.render_path_win,
render_path_osx=project.render_path_osx)
name=project.name)
"""path_server=project.path_server,
path_linux=project.path_linux,
path_win=project.path_win,
path_osx=project.path_osx,
render_path_server=project.render_path_server,
render_path_linux=project.render_path_linux,
render_path_win=project.render_path_win,
render_path_osx=project.render_path_osx)"""
return jsonify(projects)
@marshal_with(project_fields)
def post(self):
args = parser.parse_args()
project = Project(
name=args['name'],
path_server=args['path_server'],
path_linux=args['path_linux'],
path_win=args['path_win'],
path_osx=args['path_osx'],
render_path_server=args['render_path_server'],
render_path_linux=args['render_path_linux'],
render_path_win=args['render_path_win'],
render_path_osx=args['render_path_osx'])
name=args['name'])
"""path_server=args['path_server'],
path_linux=args['path_linux'],
path_win=args['path_win'],
path_osx=args['path_osx'],
render_path_server=args['render_path_server'],
render_path_linux=args['render_path_linux'],
render_path_win=args['render_path_win'],
render_path_osx=args['render_path_osx'])"""
db.session.add(project)
db.session.commit()
brender/server/application/modules/settings/__init__.py
+3 -2
View File
@@ -46,7 +46,7 @@ class SettingsListApi(Resource):
class RenderSettingsApi(Resource):
def get(self):
name = ''
"""name = ''
if system() == 'Linux':
name = 'render_settings_path_linux'
elif system() == 'Windows':
@@ -56,4 +56,5 @@ class RenderSettingsApi(Resource):
path = Setting.query.filter_by(name=name).first()
onlyfiles = [f for f in listdir(path.value) if isfile(join(path.value, f))]
settings_files = dict(settings_files=onlyfiles)
return jsonify(settings_files)
return jsonify(settings_files)"""
return {'':''}
brender/server/application/modules/tasks/__init__.py
+6 -4
View File
@@ -247,7 +247,6 @@ class TaskApi(Resource):
for man in managers:
params = {'tasks': managers[man]}
#print (params)
try:
delete_task = http_rest_request(
manager.host,
@@ -256,7 +255,6 @@ class TaskApi(Resource):
params=params)
except:
logging.info("Error deleting task from Manager")
#raise
return
pass
task.status = 'ready'
@@ -278,9 +276,7 @@ class TaskApi(Resource):
for t in tasks:
print t
tasklist.append(t.id)
#map(lambda t : TaskApi.stop_task(t.id), tasks)
TaskApi.stop_task(tasklist)
#TaskApi.delete_tasks(job_id)
def get(self):
@@ -292,6 +288,9 @@ class TaskApi(Resource):
ip_address = request.remote_addr
manager = Manager.query.filter_by(ip_address=ip_address).first()
if not manager:
return '', 404
tasks = Task.query.filter(
or_(Task.status == 'ready',
Task.status=='failed'),
@@ -315,6 +314,8 @@ class TaskApi(Resource):
db.session.add(task)
db.session.commit()
job = Job.query.get(task.job_id)
tasks = {}
frame_count = 1
current_frame = 0
@@ -332,6 +333,7 @@ class TaskApi(Resource):
"child_id": task.child_id,
"parser": task.parser,
"time_cost": task.time_cost,
"project_id": job.project_id,
"chunk_start": 0,
"chunk_end": 0,
brender/server/application/static/storage/placeholder
View File
brender/server/manage.py
+14 -8
View File
@@ -4,6 +4,10 @@ import logging
from flask.ext.script import Manager
from flask.ext.migrate import MigrateCommand
from flask.ext.migrate import current
from flask.ext.migrate import upgrade
from sqlalchemy import create_engine
from alembic.migration import MigrationContext
from application import app
from application import db
@@ -17,14 +21,16 @@ def runserver():
"""This command is meant for development. If no configuration is found,
we start the app listening from all hosts, from port 9999."""
#Testing Database
from application.modules.settings.model import Setting
from sqlalchemy.exc import OperationalError
try:
Setting.query.first()
except OperationalError:
logging.error("Please run \"python manager.py db upgrade\" to initialize the database")
exit(3)
# Testig Alembic
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'])
conn = engine.connect()
context = MigrationContext.configure(conn)
current_ver = context.get_current_revision()
if not current_ver:
print("Automatic DB Upgrade")
print("Press Ctrl+C when finished")
upgrade()
print("Upgrade completed")
try:
from application import config
brender/worker/application/controllers.py
+2
View File
@@ -458,6 +458,8 @@ def run_blender_in_thread(options):
"==jobpath==",jobpath)
render_command[cmd] = render_command[cmd].replace(
"==outputpath==",outputpath)
render_command[cmd] = render_command[cmd].replace(
"==blenderpath==", "/shared/software/blender/blender_farm_latest/blender")
os.environ['WORKER_DEPENDPATH'] = dependpath
os.environ['WORKER_OUTPUTPATH'] = outputpath
plugins/blender/render_brender.py
+3 -3
View File
@@ -74,7 +74,7 @@ class flamencoUpdate (bpy.types.Operator):
try:
projects = requests.get('{0}/projects'.format(serverurl), timeout=1)
settings = requests.get('{0}/settings/render'.format(serverurl), timeout=1)
#settings = requests.get('{0}/settings/render'.format(serverurl), timeout=1)
managers = requests.get('{0}/managers'.format(serverurl), timeout=1)
except ConnectionError:
self.report( {'ERROR'}, "Can't connect to server on {0}".format(serverurl) )
@@ -152,7 +152,7 @@ class bamToRenderfarm (bpy.types.Operator):
}
print (job_properties)
amaranth_addon = False
try:
scn.use_unsimplify_render
@@ -166,7 +166,7 @@ class bamToRenderfarm (bpy.types.Operator):
bpy.ops.wm.save_mainfile()
scn.render.use_simplify = tmp_simplify
tmppath = C.user_preferences.filepaths.temporary_directory
zipname = "job"
zippath = os.path.join(tmppath, "%s.zip" % zipname)