archive/blender-asset-manager
Archived
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support for the bundle command

Browse Source 
This is primarily meant to be used via 3rd party applications (like the
Blender Cloud). The external software requires a bundle, and if was
already build it gets served a local filesystem path, that can be
further used. Otherwise the bundle is built.
This commit is contained in:
Francesco Siddi
2015-01-12 18:15:54 +01:00
parent 1379f375ee
commit 0f80f31ac4
6 changed files with 136 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/test_webserver.py
View File

@@ -6,7 +6,7 @@ that all tests pass.
Individual tests can be run with the following syntax:
python tests.py ServerTestCase.test_job_delete
python3 -m unittest test_webserver.ServerUsageTest.test_file_bundle
"""
@@ -33,6 +33,7 @@ from application import db
from test_cli import svn_repo_create
from test_cli import svn_repo_checkout
from test_cli import file_quick_touch
from test_cli import file_quick_write
from test_cli import run_check
from test_cli import wait_for_input
@@ -68,7 +69,9 @@ class ServerTestCase(unittest.TestCase):
def setUp(self):
if not os.path.isdir(TMP_DIR):
if os.path.isdir(TMP_DIR):
shutil.rmtree(TMP_DIR)
else:
os.makedirs(TMP_DIR)
# Create remote storage (usually is on the server).
@@ -93,8 +96,9 @@ class ServerTestCase(unittest.TestCase):
if not svn_repo_checkout(path_svn_repo_url, path_svn_checkout):
self.fail("svn_repo: checkout %r" % path_svn_repo_url)
file_data = b"goodbye cruel world!\n"
file_quick_write(path_svn_checkout, "file1", file_data)
dummy_file = os.path.join(path_svn_checkout, "file1")
file_quick_touch(dummy_file)
# adds all files recursively
if not run_check(["svn", "add", dummy_file]):
@@ -107,6 +111,8 @@ class ServerTestCase(unittest.TestCase):
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+ TMP_DIR +'/test.sqlite'
app.config['TESTING'] = True
db.create_all()
# os.chmod(TMP_DIR + "/test.sqlite", 0o777)
# Create a testing project, based on the global configuration (depends on a
# correct initialization of the SVN repo and on the creation of a checkout)
@@ -177,7 +183,7 @@ class ServerUsageTest(ServerTestCase):
assert res.status_code == 200
d = json.loads(res.data.decode('utf-8'))
# print(d)
def test_file_info(self):
@@ -197,7 +203,21 @@ class ServerUsageTest(ServerTestCase):
assert res.status_code == 200
f = json.loads(res.data.decode('utf-8'))
print(f['filepath'])
assert f['status'] == 'building'
# Not ideal, but we have to wait for the server to build the bundle
# and this happens in the background.
import time
time.sleep(2)
res = self.open_with_auth('/{0}/file'.format(PROJECT_NAME),
'GET',
data=dict(filepath='file1', command='bundle'))
f = json.loads(res.data.decode('utf-8'))
assert f['status'] == 'available'
#input()
if __name__ == '__main__':

10
webservice/bam/application/__init__.py
View File

@@ -59,13 +59,11 @@ auth = HTTPBasicAuth()
try:
import config
except ImportError:
config = None
if config is None:
app.config["ALLOWED_EXTENSIONS"] = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'}
else:
app.config.from_object(config.Development)
except ImportError:
app.config["ALLOWED_EXTENSIONS"] = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'}
app.config["STORAGE_BUNDLES"] = "/tmp/bam_storage_bundles"
db = SQLAlchemy(app)
log = logging.getLogger("webservice")

50
webservice/bam/application/modules/resources/__init__.py
View File

@@ -4,6 +4,7 @@ import svn.local
import werkzeug
import xml.etree.ElementTree
import logging
from multiprocessing import Process
from flask import Flask
from flask import jsonify
@@ -22,12 +23,14 @@ from flask.ext.restful import marshal
from application import auth
from application import app
from application import log
from application import db
from application.modules.admin import backend
from application.modules.admin import settings
from application.modules.projects import admin
from application.modules.projects.model import Project
from application.modules.projects.model import ProjectSetting
from application.modules.resources.model import Bundle
class DirectoryAPI(Resource):
@@ -129,12 +132,22 @@ class FileAPI(Resource):
size = os.path.getsize(os.path.join(r.path, filepath))
# Check bundle_status: (ready, in_progress)
full_filepath = os.path.join(project.repository_path, filepath)
b = Bundle.query.filter_by(source_file_path=full_filepath).first()
if b:
bundle_status = b.status
else:
bundle_status = None
return jsonify(
filepath=filepath,
log=svn_log,
size=size)
size=size,
bundle_status=bundle_status)
elif command == 'bundle':
#return jsonify(filepath=filepath, status="building")
filepath = os.path.join(project.repository_path, filepath)
if not os.path.exists(filepath):
@@ -150,7 +163,7 @@ class FileAPI(Resource):
import tempfile
# weak! (ignore original opened file)
filepath_zip = tempfile.mkstemp(suffix=".zip")
filepath_zip = tempfile.mkstemp(dir=app.config['STORAGE_BUNDLES'], suffix=".zip")
os.close(filepath_zip[0])
filepath_zip = filepath_zip[1]
@@ -162,13 +175,38 @@ class FileAPI(Resource):
report,
):
pass
# once done, send a message to the cloud and mark the download as available
# we will send the download form the cloud server
b = Bundle.query.filter_by(source_file_path=filepath).first()
if b:
b.bundle_path = filepath_zip
else:
b = Bundle(
source_file_path=filepath,
bundle_path=filepath_zip)
db.session.add(b)
b.status = "available"
db.session.commit()
# once done, we update the queue, as well as the status of the
# bundle in the table and serve the bundle_path
# return jsonify(filepath=filepath_zip)
# Check in database if file has been requested already
# Check in database if file has been requested already
b = Bundle.query.filter_by(source_file_path=filepath).first()
if b:
if b.status == "available":
# Check if archive is available on the filesystem
if os.path.isfile(b.bundle_path):
# serve the local path for the zip file
return jsonify(filepath=b.bundle_path, status="available")
else:
b.status = "building"
db.session.commit()
# build the bundle again
elif b.status == "building":
# we are waiting for the server to build the archive
filepath=None
return jsonify(filepath=filepath, status="building")
# If file not avaliable, start the bundling and return a None filepath,
# which the cloud will interpret as, no file is available at the moment
@@ -177,7 +215,6 @@ class FileAPI(Resource):
p.start()
filepath=None
return jsonify(filepath=filepath, status="building")
elif command == 'checkout':
@@ -349,7 +386,6 @@ class FileAPI(Resource):
"""
import os
from bam.blend import blendfile_pack
assert(os.path.exists(filepath) and not os.path.isdir(filepath))
log.info(" Source path: %r" % filepath)
log.info(" Zip path: %r" % filepath_zip)

28
webservice/bam/application/modules/resources/model.py Normal file
View File

@@ -0,0 +1,28 @@
import datetime
from application import db
class Bundle(db.Model):
"""Bundles are the results of a 'bam bundle' command. When running such command
for the first time we:
- create a task in the queue (see the queue module)
- create a bundle entry and set its status as 'waiting'
- executed the task (at due time)
- set the bundle entry status to 'building'
- once completed we set the status to 'available'
- serve the bundle_path
The bundle_path can be used but an application that shares access to the BAM
storage, as well as by the 'bam checkout' command itself (later on).
If this is not the case, we will provide a working download link via the 'bam info'.
"""
id = db.Column(db.Integer, primary_key=True)
source_file_path = db.Column(db.String(512), nullable=False)
bundle_path = db.Column(db.String(512))
status = db.Column(db.String(80))
creation_date = db.Column(db.DateTime(), default=datetime.datetime.now)
update_date = db.Column(db.DateTime(), default=datetime.datetime.now)
def __str__(self):
return str(self.name)

3
webservice/bam/config.py.example
View File

@@ -2,6 +2,5 @@ class Config:
DEBUG = True
class Development(Config):
STORAGE_PATH = "/Volumes/PROJECTS/storage"
UPLOAD_FOLDER = "/Volumes/PROJECTS/storage_staging"
STORAGE_BUNDLES = '/tmp/bam_storage_bundles'
ALLOWED_EXTENSIONS = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'}

34
webservice/bam/migrations/versions/34f45e6817a_bundles.py Normal file
View File

@@ -0,0 +1,34 @@
"""bundles
Revision ID: 34f45e6817a
Revises: 52d9e7b917f
Create Date: 2015-01-08 11:37:26.622883
"""
# revision identifiers, used by Alembic.
revision = '34f45e6817a'
down_revision = '52d9e7b917f'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('bundle',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('source_file_path', sa.String(length=512), nullable=False),
sa.Column('bundle_path', sa.String(length=512), nullable=True),
sa.Column('status', sa.String(length=80), nullable=True),
sa.Column('creation_date', sa.DateTime(), nullable=True),
sa.Column('update_date', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('bundle')
### end Alembic commands ###