Handle server-side pagination in /bcloud/texture-libraries

We keep looping over the pages until the last page is hit. As a result,
we can't forward the HTTP headers from Eve to the client.

pillar/application/modules/blender_cloud.py

@@ -3,17 +3,18 @@ import logging
 from flask import Blueprint, request, current_app, g
 from eve.methods.get import get
 from eve.utils import config as eve_config
+from werkzeug.datastructures import MultiDict
+from werkzeug.exceptions import InternalServerError
 
 from application import utils
 from application.utils.authorization import require_login
 
+TL_PROJECTION = utils.dumps({'name': 1, 'url': 1, 'permissions': 1,})
+TL_SORT = utils.dumps([('name', 1)])
+
 TEXTURE_LIBRARY_QUERY_ARGS = {
-    eve_config.QUERY_PROJECTION: utils.dumps({
-        'name': 1,
-        'url': 1,
-        'permissions': 1,
-    }),
-    eve_config.QUERY_SORT: utils.dumps([('name', 1)]),
+    eve_config.QUERY_PROJECTION: TL_PROJECTION,
+    eve_config.QUERY_SORT: TL_SORT,
     'max_results': 'null',  # this needs to be there, or we get a KeyError.
 }
 
@@ -21,29 +22,62 @@ blueprint = Blueprint('blender_cloud', __name__)
 log = logging.getLogger(__name__)
 
 
+def keep_fetching_texture_libraries(proj_filter):
+
+    groups = g.current_user['groups']
+    user_id = g.current_user['user_id']
+
+    page = 1
+    max_page = float('inf')
+
+    while page <= max_page:
+        request.args.setlist(eve_config.QUERY_PAGE, [page])
+
+        result, _, _, status, _ = get(
+            'projects',
+            {'$or': [
+                {'user': user_id},
+                {'permissions.groups.group': {'$in': groups}},
+                {'permissions.world': 'GET'}
+            ]})
+
+        if status != 200:
+            log.warning('Error fetching texture libraries: %s', result)
+            raise InternalServerError('Error fetching texture libraries')
+
+        for proj in result['_items']:
+            if proj_filter(proj):
+                yield proj
+
+        # Compute the last page number we should query.
+        meta = result['_meta']
+        max_page = meta['total'] // meta['max_results']
+        if meta['total'] % meta['max_results'] > 0:
+            max_page += 1
+
+        page += 1
+
+
 @blueprint.route('/texture-libraries')
 @require_login()
 def texture_libraries():
     # Use Eve method so that we get filtering on permissions for free.
     # This gives all the projects that contain the required node types.
-    request.args = TEXTURE_LIBRARY_QUERY_ARGS
-    groups = g.current_user['groups']
 
-    result, _, _, status, headers = get(
-        'projects',
-        {'$or': [
-            {'permissions.groups.group': {'$in': groups}},
-            {'permissions.world': 'GET'}
-        ]})
+    request.args = MultiDict(request.args)   # allow changes; it's an ImmutableMultiDict by default.
+    request.args.setlist(eve_config.QUERY_PROJECTION, [TL_PROJECTION])
+    request.args.setlist(eve_config.QUERY_SORT, [TL_SORT])
 
-    if status == 200:
-        # Filter those projects that don't contain a top-level texture or group_texture node.
-        result['_items'] = [proj for proj in result['_items']
-                            if has_texture_node(proj)]
+    # Construct eve-like response.
+    projects = list(keep_fetching_texture_libraries(has_texture_node))
+    result = {'_items': projects,
+              '_meta': {
+                  'max_results': len(projects),
+                  'page': 1,
+                  'total': len(projects),
+              }}
 
-    resp = utils.jsonify(result)
-    resp.headers.extend(headers)
-    return resp, status
+    return utils.jsonify(result)
 
 
 def has_texture_node(proj):