diff --git a/tests/test_webserver.py b/tests/test_webserver.py index b1a7c22..1fbeebd 100644 --- a/tests/test_webserver.py +++ b/tests/test_webserver.py @@ -6,7 +6,7 @@ that all tests pass. Individual tests can be run with the following syntax: - python tests.py ServerTestCase.test_job_delete + python3 -m unittest test_webserver.ServerUsageTest.test_file_bundle """ @@ -33,6 +33,7 @@ from application import db from test_cli import svn_repo_create from test_cli import svn_repo_checkout from test_cli import file_quick_touch +from test_cli import file_quick_write from test_cli import run_check from test_cli import wait_for_input @@ -68,7 +69,9 @@ class ServerTestCase(unittest.TestCase): def setUp(self): - if not os.path.isdir(TMP_DIR): + if os.path.isdir(TMP_DIR): + shutil.rmtree(TMP_DIR) + else: os.makedirs(TMP_DIR) # Create remote storage (usually is on the server). @@ -93,8 +96,9 @@ class ServerTestCase(unittest.TestCase): if not svn_repo_checkout(path_svn_repo_url, path_svn_checkout): self.fail("svn_repo: checkout %r" % path_svn_repo_url) + file_data = b"goodbye cruel world!\n" + file_quick_write(path_svn_checkout, "file1", file_data) dummy_file = os.path.join(path_svn_checkout, "file1") - file_quick_touch(dummy_file) # adds all files recursively if not run_check(["svn", "add", dummy_file]): @@ -107,6 +111,8 @@ class ServerTestCase(unittest.TestCase): app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+ TMP_DIR +'/test.sqlite' app.config['TESTING'] = True db.create_all() + + # os.chmod(TMP_DIR + "/test.sqlite", 0o777) # Create a testing project, based on the global configuration (depends on a # correct initialization of the SVN repo and on the creation of a checkout) @@ -177,7 +183,7 @@ class ServerUsageTest(ServerTestCase): assert res.status_code == 200 d = json.loads(res.data.decode('utf-8')) - # print(d) + def test_file_info(self): @@ -197,7 +203,21 @@ class ServerUsageTest(ServerTestCase): assert res.status_code == 200 f = json.loads(res.data.decode('utf-8')) - print(f['filepath']) + assert f['status'] == 'building' + + # Not ideal, but we have to wait for the server to build the bundle + # and this happens in the background. + import time + time.sleep(2) + + res = self.open_with_auth('/{0}/file'.format(PROJECT_NAME), + 'GET', + data=dict(filepath='file1', command='bundle')) + f = json.loads(res.data.decode('utf-8')) + assert f['status'] == 'available' + + #input() + if __name__ == '__main__': diff --git a/webservice/bam/application/__init__.py b/webservice/bam/application/__init__.py index 6718b65..36b4d45 100644 --- a/webservice/bam/application/__init__.py +++ b/webservice/bam/application/__init__.py @@ -59,13 +59,11 @@ auth = HTTPBasicAuth() try: import config -except ImportError: - config = None - -if config is None: - app.config["ALLOWED_EXTENSIONS"] = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'} -else: app.config.from_object(config.Development) +except ImportError: + app.config["ALLOWED_EXTENSIONS"] = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'} + app.config["STORAGE_BUNDLES"] = "/tmp/bam_storage_bundles" + db = SQLAlchemy(app) log = logging.getLogger("webservice") diff --git a/webservice/bam/application/modules/resources/__init__.py b/webservice/bam/application/modules/resources/__init__.py index 66acbda..c019561 100644 --- a/webservice/bam/application/modules/resources/__init__.py +++ b/webservice/bam/application/modules/resources/__init__.py @@ -4,6 +4,7 @@ import svn.local import werkzeug import xml.etree.ElementTree import logging +from multiprocessing import Process from flask import Flask from flask import jsonify @@ -22,12 +23,14 @@ from flask.ext.restful import marshal from application import auth from application import app from application import log +from application import db from application.modules.admin import backend from application.modules.admin import settings from application.modules.projects import admin from application.modules.projects.model import Project from application.modules.projects.model import ProjectSetting +from application.modules.resources.model import Bundle class DirectoryAPI(Resource): @@ -129,12 +132,22 @@ class FileAPI(Resource): size = os.path.getsize(os.path.join(r.path, filepath)) + # Check bundle_status: (ready, in_progress) + full_filepath = os.path.join(project.repository_path, filepath) + b = Bundle.query.filter_by(source_file_path=full_filepath).first() + if b: + bundle_status = b.status + else: + bundle_status = None + return jsonify( filepath=filepath, log=svn_log, - size=size) + size=size, + bundle_status=bundle_status) elif command == 'bundle': + #return jsonify(filepath=filepath, status="building") filepath = os.path.join(project.repository_path, filepath) if not os.path.exists(filepath): @@ -150,7 +163,7 @@ class FileAPI(Resource): import tempfile # weak! (ignore original opened file) - filepath_zip = tempfile.mkstemp(suffix=".zip") + filepath_zip = tempfile.mkstemp(dir=app.config['STORAGE_BUNDLES'], suffix=".zip") os.close(filepath_zip[0]) filepath_zip = filepath_zip[1] @@ -162,13 +175,38 @@ class FileAPI(Resource): report, ): pass - # once done, send a message to the cloud and mark the download as available - # we will send the download form the cloud server + + b = Bundle.query.filter_by(source_file_path=filepath).first() + if b: + b.bundle_path = filepath_zip + else: + b = Bundle( + source_file_path=filepath, + bundle_path=filepath_zip) + db.session.add(b) + b.status = "available" + db.session.commit() + # once done, we update the queue, as well as the status of the + # bundle in the table and serve the bundle_path # return jsonify(filepath=filepath_zip) - # Check in database if file has been requested already - # Check if archive is available on the filesystem + # Check in database if file has been requested already + b = Bundle.query.filter_by(source_file_path=filepath).first() + if b: + if b.status == "available": + # Check if archive is available on the filesystem + if os.path.isfile(b.bundle_path): + # serve the local path for the zip file + return jsonify(filepath=b.bundle_path, status="available") + else: + b.status = "building" + db.session.commit() + # build the bundle again + elif b.status == "building": + # we are waiting for the server to build the archive + filepath=None + return jsonify(filepath=filepath, status="building") # If file not avaliable, start the bundling and return a None filepath, # which the cloud will interpret as, no file is available at the moment @@ -177,7 +215,6 @@ class FileAPI(Resource): p.start() filepath=None - return jsonify(filepath=filepath, status="building") elif command == 'checkout': @@ -349,7 +386,6 @@ class FileAPI(Resource): """ import os from bam.blend import blendfile_pack - assert(os.path.exists(filepath) and not os.path.isdir(filepath)) log.info(" Source path: %r" % filepath) log.info(" Zip path: %r" % filepath_zip) diff --git a/webservice/bam/application/modules/resources/model.py b/webservice/bam/application/modules/resources/model.py new file mode 100644 index 0000000..d147e6f --- /dev/null +++ b/webservice/bam/application/modules/resources/model.py @@ -0,0 +1,28 @@ +import datetime +from application import db + + +class Bundle(db.Model): + """Bundles are the results of a 'bam bundle' command. When running such command + for the first time we: + - create a task in the queue (see the queue module) + - create a bundle entry and set its status as 'waiting' + - executed the task (at due time) + - set the bundle entry status to 'building' + - once completed we set the status to 'available' + - serve the bundle_path + + The bundle_path can be used but an application that shares access to the BAM + storage, as well as by the 'bam checkout' command itself (later on). + + If this is not the case, we will provide a working download link via the 'bam info'. + """ + id = db.Column(db.Integer, primary_key=True) + source_file_path = db.Column(db.String(512), nullable=False) + bundle_path = db.Column(db.String(512)) + status = db.Column(db.String(80)) + creation_date = db.Column(db.DateTime(), default=datetime.datetime.now) + update_date = db.Column(db.DateTime(), default=datetime.datetime.now) + + def __str__(self): + return str(self.name) diff --git a/webservice/bam/config.py.example b/webservice/bam/config.py.example index 02ff6e5..e140c0e 100644 --- a/webservice/bam/config.py.example +++ b/webservice/bam/config.py.example @@ -2,6 +2,5 @@ class Config: DEBUG = True class Development(Config): - STORAGE_PATH = "/Volumes/PROJECTS/storage" - UPLOAD_FOLDER = "/Volumes/PROJECTS/storage_staging" + STORAGE_BUNDLES = '/tmp/bam_storage_bundles' ALLOWED_EXTENSIONS = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'} diff --git a/webservice/bam/migrations/versions/34f45e6817a_bundles.py b/webservice/bam/migrations/versions/34f45e6817a_bundles.py new file mode 100644 index 0000000..214e52a --- /dev/null +++ b/webservice/bam/migrations/versions/34f45e6817a_bundles.py @@ -0,0 +1,34 @@ +"""bundles + +Revision ID: 34f45e6817a +Revises: 52d9e7b917f +Create Date: 2015-01-08 11:37:26.622883 + +""" + +# revision identifiers, used by Alembic. +revision = '34f45e6817a' +down_revision = '52d9e7b917f' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('bundle', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_file_path', sa.String(length=512), nullable=False), + sa.Column('bundle_path', sa.String(length=512), nullable=True), + sa.Column('status', sa.String(length=80), nullable=True), + sa.Column('creation_date', sa.DateTime(), nullable=True), + sa.Column('update_date', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('bundle') + ### end Alembic commands ###