Support for the bundle command

This is primarily meant to be used via 3rd party applications (like the
Blender Cloud). The external software requires a bundle, and if was
already build it gets served a local filesystem path, that can be
further used. Otherwise the bundle is built.

tests/test_webserver.py

@@ -6,7 +6,7 @@ that all tests pass.
 
 Individual tests can be run with the following syntax:
 
-    python tests.py ServerTestCase.test_job_delete
+    python3 -m unittest test_webserver.ServerUsageTest.test_file_bundle
 
 """
 
@@ -33,6 +33,7 @@ from application import db
 from test_cli import svn_repo_create
 from test_cli import svn_repo_checkout
 from test_cli import file_quick_touch
+from test_cli import file_quick_write
 from test_cli import run_check
 from test_cli import wait_for_input
 
@@ -68,7 +69,9 @@ class ServerTestCase(unittest.TestCase):
 
     def setUp(self):
 
-        if not os.path.isdir(TMP_DIR):
+        if os.path.isdir(TMP_DIR):
+            shutil.rmtree(TMP_DIR)
+        else:
             os.makedirs(TMP_DIR)
 
         # Create remote storage (usually is on the server).
@@ -93,8 +96,9 @@ class ServerTestCase(unittest.TestCase):
         if not svn_repo_checkout(path_svn_repo_url, path_svn_checkout):
             self.fail("svn_repo: checkout %r" % path_svn_repo_url)
 
+        file_data = b"goodbye cruel world!\n"
+        file_quick_write(path_svn_checkout, "file1", file_data)
         dummy_file = os.path.join(path_svn_checkout, "file1")
-        file_quick_touch(dummy_file)
 
         # adds all files recursively
         if not run_check(["svn", "add", dummy_file]):
@@ -107,6 +111,8 @@ class ServerTestCase(unittest.TestCase):
         app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+ TMP_DIR +'/test.sqlite'
         app.config['TESTING'] = True
         db.create_all()
+
+        # os.chmod(TMP_DIR + "/test.sqlite", 0o777)
         # Create a testing project, based on the global configuration (depends on a
         # correct initialization of the SVN repo and on the creation of a checkout)
 
@@ -177,7 +183,7 @@ class ServerUsageTest(ServerTestCase):
 
         assert res.status_code == 200
         d = json.loads(res.data.decode('utf-8'))
-        # print(d)
+
 
 
     def test_file_info(self):
@@ -197,7 +203,21 @@ class ServerUsageTest(ServerTestCase):
 
         assert res.status_code == 200
         f = json.loads(res.data.decode('utf-8'))
-        print(f['filepath'])
+        assert f['status'] == 'building'
+
+        # Not ideal, but we have to wait for the server to build the bundle
+        # and this happens in the background.
+        import time
+        time.sleep(2)
+
+        res = self.open_with_auth('/{0}/file'.format(PROJECT_NAME), 
+            'GET', 
+            data=dict(filepath='file1', command='bundle'))
+        f = json.loads(res.data.decode('utf-8'))
+        assert f['status'] == 'available'
+
+        #input()
+
 
 
 if __name__ == '__main__':

webservice/bam/application/__init__.py

@@ -59,13 +59,11 @@ auth = HTTPBasicAuth()
 
 try:
     import config
-except ImportError:
-    config = None
-
-if config is None:
-    app.config["ALLOWED_EXTENSIONS"] = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'}
-else:
     app.config.from_object(config.Development)
+except ImportError:
+    app.config["ALLOWED_EXTENSIONS"] = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'}
+    app.config["STORAGE_BUNDLES"] = "/tmp/bam_storage_bundles"
+
 
 db = SQLAlchemy(app)
 log = logging.getLogger("webservice")

webservice/bam/application/modules/resources/__init__.py

@@ -4,6 +4,7 @@ import svn.local
 import werkzeug
 import xml.etree.ElementTree
 import logging
+from multiprocessing import Process
 
 from flask import Flask
 from flask import jsonify
@@ -22,12 +23,14 @@ from flask.ext.restful import marshal
 from application import auth
 from application import app
 from application import log
+from application import db
 
 from application.modules.admin import backend
 from application.modules.admin import settings
 from application.modules.projects import admin
 from application.modules.projects.model import Project
 from application.modules.projects.model import ProjectSetting
+from application.modules.resources.model import Bundle
 
 
 class DirectoryAPI(Resource):
@@ -129,12 +132,22 @@ class FileAPI(Resource):
 
             size = os.path.getsize(os.path.join(r.path, filepath))
 
+            # Check bundle_status: (ready, in_progress)
+            full_filepath = os.path.join(project.repository_path, filepath)
+            b = Bundle.query.filter_by(source_file_path=full_filepath).first()
+            if b:
+                bundle_status = b.status
+            else:
+                bundle_status = None
+
             return jsonify(
                 filepath=filepath,
                 log=svn_log,
-                size=size)
+                size=size,
+                bundle_status=bundle_status)
 
         elif command == 'bundle':
+            #return jsonify(filepath=filepath, status="building")
             filepath = os.path.join(project.repository_path, filepath)
 
             if not os.path.exists(filepath):
@@ -150,7 +163,7 @@ class FileAPI(Resource):
                 import tempfile
 
                 # weak! (ignore original opened file)
-                filepath_zip = tempfile.mkstemp(suffix=".zip")
+                filepath_zip = tempfile.mkstemp(dir=app.config['STORAGE_BUNDLES'], suffix=".zip")
                 os.close(filepath_zip[0])
                 filepath_zip = filepath_zip[1]
 
@@ -162,13 +175,38 @@ class FileAPI(Resource):
                         report,
                         ):
                     pass
-                # once done, send a message to the cloud and mark the download as available
+
-                # we will send the download form the cloud server
+                b = Bundle.query.filter_by(source_file_path=filepath).first()
+                if b:
+                    b.bundle_path = filepath_zip
+                else:
+                    b = Bundle(
+                        source_file_path=filepath,
+                        bundle_path=filepath_zip)
+                    db.session.add(b)
+                b.status = "available"
+                db.session.commit()
+                # once done, we update the queue, as well as the status of the
+                # bundle in the table and serve the bundle_path
                 # return jsonify(filepath=filepath_zip)
 
-            # Check in database if file has been requested already
 
-            # Check if archive is available on the filesystem
+            # Check in database if file has been requested already
+            b = Bundle.query.filter_by(source_file_path=filepath).first()
+            if b:
+                if b.status == "available":
+                    # Check if archive is available on the filesystem
+                    if os.path.isfile(b.bundle_path):
+                        # serve the local path for the zip file
+                        return jsonify(filepath=b.bundle_path, status="available")
+                    else:
+                        b.status = "building"
+                        db.session.commit()
+                        # build the bundle again
+                elif b.status == "building":
+                    # we are waiting for the server to build the archive
+                    filepath=None
+                    return jsonify(filepath=filepath, status="building")
 
             # If file not avaliable, start the bundling and return a None filepath,
             # which the cloud will interpret as, no file is available at the moment
@@ -177,7 +215,6 @@ class FileAPI(Resource):
             p.start()
 
             filepath=None
-
             return jsonify(filepath=filepath, status="building")
 
         elif command == 'checkout':
@@ -349,7 +386,6 @@ class FileAPI(Resource):
         """
         import os
         from bam.blend import blendfile_pack
-
         assert(os.path.exists(filepath) and not os.path.isdir(filepath))
         log.info("  Source path: %r" % filepath)
         log.info("  Zip path: %r" % filepath_zip)

webservice/bam/application/modules/resources/model.py

@@ -0,0 +1,28 @@
+import datetime
+from application import db
+
+
+class Bundle(db.Model):
+    """Bundles are the results of a 'bam bundle' command. When running such command
+    for the first time we:
+        - create a task in the queue (see the queue module)
+        - create a bundle entry and set its status as 'waiting'
+        - executed the task (at due time)
+        - set the bundle entry status to 'building'
+        - once completed we set the status to 'available'
+        - serve the bundle_path
+
+    The bundle_path can be used but an application that shares access to the BAM
+    storage, as well as by the 'bam checkout' command itself (later on).
+
+    If this is not the case, we will provide a working download link via the 'bam info'.
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    source_file_path = db.Column(db.String(512), nullable=False)
+    bundle_path = db.Column(db.String(512))
+    status = db.Column(db.String(80))
+    creation_date = db.Column(db.DateTime(), default=datetime.datetime.now)
+    update_date = db.Column(db.DateTime(), default=datetime.datetime.now)
+
+    def __str__(self):
+        return str(self.name)

webservice/bam/config.py.example

@@ -2,6 +2,5 @@ class Config:
     DEBUG = True
 
 class Development(Config):
-    STORAGE_PATH = "/Volumes/PROJECTS/storage"
+    STORAGE_BUNDLES = '/tmp/bam_storage_bundles'
-    UPLOAD_FOLDER = "/Volumes/PROJECTS/storage_staging"
     ALLOWED_EXTENSIONS = {'txt', 'mp4', 'png', 'jpg', 'jpeg', 'gif', 'blend', 'zip'}

webservice/bam/migrations/versions/34f45e6817a_bundles.py

@@ -0,0 +1,34 @@
+"""bundles
+
+Revision ID: 34f45e6817a
+Revises: 52d9e7b917f
+Create Date: 2015-01-08 11:37:26.622883
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '34f45e6817a'
+down_revision = '52d9e7b917f'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('bundle',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('source_file_path', sa.String(length=512), nullable=False),
+    sa.Column('bundle_path', sa.String(length=512), nullable=True),
+    sa.Column('status', sa.String(length=80), nullable=True),
+    sa.Column('creation_date', sa.DateTime(), nullable=True),
+    sa.Column('update_date', sa.DateTime(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('bundle')
+    ### end Alembic commands ###