Switched caching to CacheControl + pillar module usable without Blender

Making the blender_cloud.pillar and blender_cloud.cache modules usable
without Blender required some moving of the code, from __init__.py to
blender.py.

CacheControl requires the lockfile package, which increases the number
of bundled wheel files to 3. Those are now managed by
blender_cloud.wheels.load_wheels(). The wheels are only loaded if the
Python installation doesn't yet contain the required packages. This allows
development with virtualenv-installed packages, debugging in the IDE, etc.

blender_cloud/__init__.py

@@ -32,144 +32,59 @@ bl_info = {
     "support": "TESTING"
 }
 
+import logging
+
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)-15s %(levelname)8s %(name)s %(message)s')
+logging.getLogger('cachecontrol').setLevel(logging.DEBUG)
+logging.getLogger(__name__).setLevel(logging.DEBUG)
+
 # Support reloading
 if 'pillar' in locals():
     import importlib
 
+    wheels = importlib.reload(wheels)
+    wheels.load_wheels()
+
     pillar = importlib.reload(pillar)
-    async_loop = importlib.reload(async_loop)
-    gui = importlib.reload(gui)
     cache = importlib.reload(cache)
 else:
-    from . import pillar, async_loop, gui, cache
+    from . import wheels
+    wheels.load_wheels()
 
-import logging
+    from . import pillar, cache
-import os.path
-
-import bpy
-from bpy.types import AddonPreferences, Operator, WindowManager, Scene
-from bpy.props import StringProperty
-
-
-class BlenderCloudPreferences(AddonPreferences):
-    bl_idname = __name__
-
-    pillar_server = bpy.props.StringProperty(
-        name='Blender Cloud Server',
-        description='URL of the Blender Cloud backend server',
-        default='https://pillar.blender.org:5000/'
-    )
-
-    def draw(self, context):
-        layout = self.layout
-
-        # Carefully try and import the Blender ID addon
-        try:
-            import blender_id.profiles as blender_id_profiles
-        except ImportError:
-            blender_id_profiles = None
-            blender_id_profile = None
-        else:
-            blender_id_profile = blender_id_profiles.get_active_profile()
-
-        if blender_id_profiles is None:
-            blender_id_icon = 'ERROR'
-            blender_id_text = "This add-on requires Blender ID"
-            blender_id_help = "Make sure that the Blender ID add-on is installed and activated"
-        elif not blender_id_profile:
-            blender_id_icon = 'ERROR'
-            blender_id_text = "You are logged out."
-            blender_id_help = "To login, go to the Blender ID add-on preferences."
-        else:
-            blender_id_icon = 'WORLD_DATA'
-            blender_id_text = "You are logged in as %s." % blender_id_profile['username']
-            blender_id_help = "To logout or change profile, " \
-                              "go to the Blender ID add-on preferences."
-
-        sub = layout.column()
-        sub.label(text=blender_id_text, icon=blender_id_icon)
-        sub.label(text="* " + blender_id_help)
-
-        # options for Pillar
-        sub = layout.column()
-        sub.enabled = blender_id_icon != 'ERROR'
-        sub.prop(self, "pillar_server")
-        sub.operator("pillar.credentials_update")
-
-
-class PillarCredentialsUpdate(Operator):
-    """Updates the Pillar URL and tests the new URL."""
-    bl_idname = "pillar.credentials_update"
-    bl_label = "Update credentials"
-
-    @classmethod
-    def poll(cls, context):
-        # Only allow activation when the user is actually logged in.
-        return cls.is_logged_in(context)
-
-    @classmethod
-    def is_logged_in(cls, context):
-        active_user_id = getattr(context.window_manager, 'blender_id_active_profile', None)
-        return bool(active_user_id)
-
-    def execute(self, context):
-        # Only allow activation when the user is actually logged in.
-        if not self.is_logged_in(context):
-            self.report({'ERROR'}, "No active profile found")
-            return {'CANCELLED'}
-
-        # Test the new URL
-        endpoint = bpy.context.user_preferences.addons[__name__].preferences.pillar_server
-        pillar._pillar_api = None
-        try:
-            pillar.get_project_uuid('textures')  # Just any query will do.
-        except Exception as e:
-            print(e)
-            self.report({'ERROR'}, 'Failed connection to %s' % endpoint)
-            return {'FINISHED'}
-
-        self.report({'INFO'}, 'Updated cloud server address to %s' % endpoint)
-        return {'FINISHED'}
 
 
 def register():
-    bpy.utils.register_class(BlenderCloudPreferences)
+    """Late-loads and registers the Blender-dependent submodules."""
-    bpy.utils.register_class(PillarCredentialsUpdate)
 
-    WindowManager.thumbnails_cache = StringProperty(
+    import sys
-        name="Thumbnails cache",
-        subtype='DIR_PATH',
-        default=os.path.join(cache.cache_directory(), 'thumbnails'))
 
-    WindowManager.blender_cloud_project = StringProperty(
+    # Support reloading
-        name="Blender Cloud project UUID",
+    if '%s.blender' % __name__ in sys.modules:
-        default='5672beecc0261b2005ed1a33')  # TODO: don't hard-code this
+        import importlib
 
-    WindowManager.blender_cloud_node = StringProperty(
+        def reload_mod(name):
-        name="Blender Cloud node UUID",
+            modname = '%s.%s' % (__name__, name)
-        default='')  # empty == top-level of project
+            module = importlib.reload(sys.modules[modname])
+            sys.modules[modname] = module
+            return module
 
-    Scene.blender_cloud_dir = StringProperty(
+        blender = reload_mod('blender')
-        name='Blender Cloud texture storage directory',
+        gui = reload_mod('gui')
-        subtype='DIR_PATH',
+        async_loop = reload_mod('async_loop')
-        default='//textures')
+    else:
+        from . import blender, gui, async_loop
 
-    logging.basicConfig(level=logging.DEBUG,
-                        format='%(asctime)-15s %(levelname)8s %(name)s %(message)s')
     async_loop.setup_asyncio_executor()
+
+    blender.register()
     gui.register()
 
 
 def unregister():
+    from . import blender, gui
+
     gui.unregister()
+    blender.unregister()
 
-    bpy.utils.unregister_class(PillarCredentialsUpdate)
-    bpy.utils.unregister_class(BlenderCloudPreferences)
-
-    del WindowManager.blender_cloud_project
-    del WindowManager.blender_cloud_node
-    del WindowManager.blender_cloud_thumbnails
-
-
-if __name__ == "__main__":
-    register()

blender_cloud/blender.py

@@ -0,0 +1,133 @@
+"""Blender-specific code.
+
+Separated from __init__.py so that we can import & run from non-Blender environments.
+"""
+
+import os.path
+
+import bpy
+from bpy.types import AddonPreferences, Operator, WindowManager, Scene
+from bpy.props import StringProperty
+
+from . import pillar, gui, cache
+
+ADDON_NAME = 'blender_cloud'
+
+
+class BlenderCloudPreferences(AddonPreferences):
+    bl_idname = ADDON_NAME
+
+    pillar_server = bpy.props.StringProperty(
+        name='Blender Cloud Server',
+        description='URL of the Blender Cloud backend server',
+        default='https://pillar.blender.org:5000/'
+    )
+
+    def draw(self, context):
+        layout = self.layout
+
+        # Carefully try and import the Blender ID addon
+        try:
+            import blender_id.profiles as blender_id_profiles
+        except ImportError:
+            blender_id_profiles = None
+            blender_id_profile = None
+        else:
+            blender_id_profile = blender_id_profiles.get_active_profile()
+
+        if blender_id_profiles is None:
+            blender_id_icon = 'ERROR'
+            blender_id_text = "This add-on requires Blender ID"
+            blender_id_help = "Make sure that the Blender ID add-on is installed and activated"
+        elif not blender_id_profile:
+            blender_id_icon = 'ERROR'
+            blender_id_text = "You are logged out."
+            blender_id_help = "To login, go to the Blender ID add-on preferences."
+        else:
+            blender_id_icon = 'WORLD_DATA'
+            blender_id_text = "You are logged in as %s." % blender_id_profile['username']
+            blender_id_help = "To logout or change profile, " \
+                              "go to the Blender ID add-on preferences."
+
+        sub = layout.column()
+        sub.label(text=blender_id_text, icon=blender_id_icon)
+        sub.label(text="* " + blender_id_help)
+
+        # options for Pillar
+        sub = layout.column()
+        sub.enabled = blender_id_icon != 'ERROR'
+        sub.prop(self, "pillar_server")
+        sub.operator("pillar.credentials_update")
+
+
+class PillarCredentialsUpdate(Operator):
+    """Updates the Pillar URL and tests the new URL."""
+    bl_idname = "pillar.credentials_update"
+    bl_label = "Update credentials"
+
+    @classmethod
+    def poll(cls, context):
+        # Only allow activation when the user is actually logged in.
+        return cls.is_logged_in(context)
+
+    @classmethod
+    def is_logged_in(cls, context):
+        active_user_id = getattr(context.window_manager, 'blender_id_active_profile', None)
+        return bool(active_user_id)
+
+    def execute(self, context):
+        # Only allow activation when the user is actually logged in.
+        if not self.is_logged_in(context):
+            self.report({'ERROR'}, "No active profile found")
+            return {'CANCELLED'}
+
+        # Test the new URL
+        endpoint = bpy.context.user_preferences.addons[ADDON_NAME].preferences.pillar_server
+        pillar._pillar_api = None
+        try:
+            pillar.get_project_uuid('textures')  # Just any query will do.
+        except Exception as e:
+            print(e)
+            self.report({'ERROR'}, 'Failed connection to %s' % endpoint)
+            return {'FINISHED'}
+
+        self.report({'INFO'}, 'Updated cloud server address to %s' % endpoint)
+        return {'FINISHED'}
+
+
+def preferences() -> BlenderCloudPreferences:
+    return bpy.context.user_preferences.addons[ADDON_NAME].preferences
+
+
+def register():
+    bpy.utils.register_class(BlenderCloudPreferences)
+    bpy.utils.register_class(PillarCredentialsUpdate)
+
+    WindowManager.thumbnails_cache = StringProperty(
+        name="Thumbnails cache",
+        subtype='DIR_PATH',
+        default=os.path.join(cache.cache_directory(), 'thumbnails'))
+
+    WindowManager.blender_cloud_project = StringProperty(
+        name="Blender Cloud project UUID",
+        default='5672beecc0261b2005ed1a33')  # TODO: don't hard-code this
+
+    WindowManager.blender_cloud_node = StringProperty(
+        name="Blender Cloud node UUID",
+        default='')  # empty == top-level of project
+
+    Scene.blender_cloud_dir = StringProperty(
+        name='Blender Cloud texture storage directory',
+        subtype='DIR_PATH',
+        default='//textures')
+
+
+def unregister():
+    gui.unregister()
+
+    bpy.utils.unregister_class(PillarCredentialsUpdate)
+    bpy.utils.unregister_class(BlenderCloudPreferences)
+
+    del WindowManager.blender_cloud_project
+    del WindowManager.blender_cloud_node
+    del WindowManager.blender_cloud_thumbnails

blender_cloud/cache.py

@@ -1,24 +1,15 @@
 """Cache management."""
 
 import os
-import sys
 import logging
+import requests
+import cachecontrol
+from cachecontrol.caches import FileCache
 
 from . import appdirs
 
 log = logging.getLogger(__name__)
-
+_session = None  # requests.Session object that's set up for caching by requests_session().
-
-# Add our shipped Requests-Cache wheel to the Python path
-if not any('requests_cache' in path for path in sys.path):
-    import glob
-
-    # TODO: gracefully handle errors when the wheel cannot be found.
-    my_dir = os.path.dirname(__file__)
-    wheel = glob.glob(os.path.join(my_dir, 'requests_cache*.whl'))[0]
-    sys.path.append(wheel)
-
-import requests_cache
 
 
 def cache_directory() -> str:
@@ -30,37 +21,27 @@ def cache_directory() -> str:
 
     # TODO: just use bpy.utils.user_resource('CACHE', ...)
 
-    cache_dir = os.path.join(appdirs.user_cache_dir(appname='Blender', appauthor=False), 'blender-cloud')
+    cache_dir = os.path.join(appdirs.user_cache_dir(appname='Blender', appauthor=False), 'blender_cloud')
 
     os.makedirs(cache_dir, exist_ok=True)
 
     return cache_dir
 
 
-def requests_session() -> requests_cache.CachedSession:
+def requests_session() -> requests.Session:
     """Creates a Requests-Cache session object."""
 
+    global _session
+
+    if _session is not None:
+        return _session
+
     cache_dir = cache_directory()
-    cache_name = os.path.join(cache_dir, 'blender_cloud_cache')
+    cache_name = os.path.join(cache_dir, 'blender_cloud_http')
     log.info('Storing cache in %s' % cache_name)
 
-    req_sess = requests_cache.CachedSession(backend='sqlite',
+    _session = cachecontrol.CacheControl(sess=requests.session(),
-                                            cache_name=cache_name)
+                                         cache=FileCache(cache_name))
 
-    return req_sess
+    return _session
 
-
-def debug_show_responses():
-
-    req_sess = requests_session()
-
-    log.info('Cache type: %s', type(req_sess.cache))
-    log.info('Cached URLs:')
-    for key in req_sess.cache.keys_map:
-        value = req_sess.cache.keys_map[key]
-        log.info('  %s = %s' % (key, value))
-
-    log.info('Cached responses:')
-    for key in req_sess.cache.responses:
-        response, timekey = req_sess.cache.get_response_and_time(key)
-        log.info('  %s = %s' % (key, response.content))

blender_cloud/pillar.py

@@ -6,25 +6,17 @@ import logging
 from contextlib import closing
 
 import requests
-
-from . import cache
-
-# Add our shipped Pillar SDK wheel to the Python path
-if not any('pillar_sdk' in path for path in sys.path):
-    import glob
-
-    # TODO: gracefully handle errors when the wheel cannot be found.
-    my_dir = os.path.dirname(__file__)
-    pillar_wheel = glob.glob(os.path.join(my_dir, 'pillar_sdk*.whl'))[0]
-    sys.path.append(pillar_wheel)
-
 import pillarsdk
 import pillarsdk.exceptions
 import pillarsdk.utils
 
+from . import cache
+
 
 _pillar_api = None  # will become a pillarsdk.Api object.
 log = logging.getLogger(__name__)
+uncached_session = requests.session()
+_testing_blender_id_profile = None  # Just for testing, overrides what is returned by blender_id_profile.
 
 
 class UserNotLoggedInError(RuntimeError):
@@ -37,6 +29,10 @@ class UserNotLoggedInError(RuntimeError):
 def blender_id_profile() -> dict:
     """Returns the Blender ID profile of the currently logged in user."""
 
+    # Allow overriding before we import the bpy module.
+    if _testing_blender_id_profile is not None:
+        return _testing_blender_id_profile
+
     import bpy
 
     active_user_id = getattr(bpy.context.window_manager, 'blender_id_active_profile', None)
@@ -47,14 +43,16 @@ def blender_id_profile() -> dict:
     return blender_id.profiles.get_active_profile()
 
 
-def pillar_api() -> pillarsdk.Api:
+def pillar_api(pillar_endpoint: str=None) -> pillarsdk.Api:
     """Returns the Pillar SDK API object for the current user.
 
     The user must be logged in.
+
+    :param pillar_endpoint: URL of the Pillar server, for testing purposes. If not specified,
+        it will use the addon preferences.
     """
 
     global _pillar_api
-    import bpy
 
     # Only return the Pillar API object if the user is still logged in.
     profile = blender_id_profile()
@@ -62,9 +60,14 @@ def pillar_api() -> pillarsdk.Api:
         raise UserNotLoggedInError()
 
     if _pillar_api is None:
-        endpoint = bpy.context.user_preferences.addons['blender_cloud'].preferences.pillar_server
+        # Allow overriding the endpoint before importing Blender-specific stuff.
+        if pillar_endpoint is None:
+            from . import blender
+            pillar_endpoint = blender.preferences().pillar_server
+
         pillarsdk.Api.requests_session = cache.requests_session()
-        _pillar_api = pillarsdk.Api(endpoint=endpoint,
+
+        _pillar_api = pillarsdk.Api(endpoint=pillar_endpoint,
                                     username=profile['username'],
                                     password=None,
                                     token=profile['token'])
@@ -143,7 +146,7 @@ async def download_to_file(url, filename, chunk_size=100 * 1024, *, future: asyn
     # the download in between.
 
     def perform_get_request():
-        return requests.get(url, stream=True, verify=True)
+        return uncached_session.get(url, stream=True, verify=True)
 
     # Download the file in a different thread.
     def download_loop():

blender_cloud/wheels/__init__.py

@@ -0,0 +1,41 @@
+"""External dependencies loader."""
+
+import glob
+import os.path
+import sys
+import logging
+
+my_dir = os.path.join(os.path.dirname(__file__))
+log = logging.getLogger(__name__)
+
+
+def load_wheel(module_name, fname_prefix):
+    """Loads a wheel from 'fname_prefix*.whl', unless the named module can be imported.
+
+    This allows us to use system-installed packages before falling back to the shipped wheels.
+    This is useful for development, less so for deployment.
+    """
+
+    try:
+        module = __import__(module_name)
+    except ImportError:
+        pass
+    else:
+        log.debug('Was able to load %s from %s, no need to load wheel %s',
+                  module_name, module.__file__, fname_prefix)
+        return
+
+    path_pattern = os.path.join(my_dir, '%s*.whl' % fname_prefix)
+    wheels = glob.glob(path_pattern)
+    if not wheels:
+        raise RuntimeError('Unable to find wheel at %r' % path_pattern)
+
+    sys.path.append(wheels[0])
+    module = __import__(module_name)
+    log.debug('Loaded %s from %s', module_name, module.__file__)
+
+
+def load_wheels():
+    load_wheel('lockfile', 'lockfile')
+    load_wheel('cachecontrol', 'CacheControl')
+    load_wheel('pillarsdk', 'pillar_sdk')

requirements.txt

@@ -0,0 +1,2 @@
+CacheControl==0.11.6
+lockfile==0.12.2