(un)delete on project also (un)delete file documents.

Note that undeleting files cannot be done via Eve, as it doesn't support
PATCHing collections. Instead, direct MongoDB modification is used to set
_deleted=False and provide new _etag and _updated values.

pillar/api/projects/__init__.py

@@ -6,10 +6,17 @@ def setup_app(app, api_prefix):
     app.on_replace_projects += hooks.override_is_private_field
     app.on_replace_projects += hooks.before_edit_check_permissions
     app.on_replace_projects += hooks.protect_sensitive_fields
+
+    app.on_replaced_projects += hooks.after_undelete_project
+    app.on_updated_projects += hooks.after_undelete_project
+
     app.on_update_projects += hooks.override_is_private_field
     app.on_update_projects += hooks.before_edit_check_permissions
     app.on_update_projects += hooks.protect_sensitive_fields
+
     app.on_delete_item_projects += hooks.before_delete_project
+    app.on_deleted_item_projects += hooks.after_delete_project
+
     app.on_insert_projects += hooks.before_inserting_override_is_private_field
     app.on_insert_projects += hooks.before_inserting_projects
     app.on_inserted_projects += hooks.after_inserting_projects

pillar/api/projects/hooks.py

@@ -1,8 +1,11 @@
 import copy
+import datetime
 import logging
 
-from flask import request, abort, current_app
+import bson.tz_util
+from flask import request, abort
 
+from pillar import current_app
 from pillar.api.node_types.asset import node_type_asset
 from pillar.api.node_types.comment import node_type_comment
 from pillar.api.node_types.group import node_type_group
@@ -10,8 +13,9 @@ from pillar.api.node_types.group_texture import node_type_group_texture
 from pillar.api.node_types.texture import node_type_texture
 from pillar.api.file_storage_backends import default_storage_backend
 from pillar.api.utils import authorization, authentication
-from pillar.api.utils import remove_private_keys
+from pillar.api.utils import remove_private_keys, random_etag
 from pillar.api.utils.authorization import user_has_role, check_permissions
+from pillar.auth import current_user
 from .utils import abort_with_error
 
 log = logging.getLogger(__name__)
@@ -64,6 +68,51 @@ def before_delete_project(document):
     """Checks permissions before we allow deletion"""
 
     check_permissions('projects', document, request.method)
+    log.info('Deleting project %s on behalf of user %s', document['_id'], current_user)
+
+
+def after_delete_project(project: dict):
+    """Perform delete on the project's files too."""
+
+    from eve.methods.delete import delete
+
+    pid = project['_id']
+    log.info('Project %s was deleted, also deleting its files.', pid)
+
+    r, _, _, status = delete('files', {'project': pid})
+    if status != 204:
+        log.warning('Unable to delete files of project %s: %s', pid, r)
+
+
+def after_undelete_project(project: dict, original: dict):
+    """Undelete the files that belong to this project.
+
+    We cannot do this via Eve, as it doesn't support PATCHing collections, so
+    direct MongoDB modification is used to set _deleted=False and provide
+    new _etag and _updated values.
+    """
+
+    if not original:
+        return
+
+    was_deleted = original.get('_deleted', False)
+    now_deleted = project.get('_deleted', False)
+    if not was_deleted or now_deleted:
+        # This is not an undelete.
+        return
+
+    pid = project['_id']
+    new_etag = random_etag()
+    now = datetime.datetime.now(tz=bson.tz_util.utc)
+
+    files_coll = current_app.db('files')
+    update_result = files_coll.update_many(
+        {'project': pid},
+        {'$set': {'_deleted': False,
+                  '_etag': new_etag,
+                  '_updated': now}})
+    log.info('undeleted %d of %d file documents of project %s',
+             update_result.modified_count, update_result.matched_count, pid)
 
 
 def protect_sensitive_fields(document, original):
@@ -225,5 +274,3 @@ def project_node_type_has_method(response):
 def projects_node_type_has_method(response):
     for project in response['_items']:
         project_node_type_has_method(project)
-
-

pillar/api/utils/__init__.py

@@ -1,12 +1,13 @@
+import base64
 import copy
-import hashlib
-import json
-import typing
-import urllib.request, urllib.parse, urllib.error
-
 import datetime
 import functools
+import hashlib
+import json
 import logging
+import random
+import typing
+import urllib.request, urllib.parse, urllib.error
 
 import bson.objectid
 from eve import RFC1123_DATE_FORMAT
@@ -192,3 +193,10 @@ def doc_diff(doc1, doc2, falsey_is_equal=True):
             continue
 
         yield key, val1, val2
+
+
+def random_etag() -> str:
+    """Random string usable as etag."""
+
+    randbytes = random.getrandbits(256).to_bytes(32, 'big')
+    return base64.b64encode(randbytes)[:-1].decode()

tests/test_api/test_bcloud_home_project.py

@@ -295,22 +295,17 @@ class HomeProjectTest(AbstractHomeProjectTest):
         self._create_user_with_token(roles={'subscriber'}, token='token')
 
         # Create home project by getting it.
-        resp = self.client.get('/api/bcloud/home-project',
+        resp = self.get('/api/bcloud/home-project', auth_token='token')
-                               headers={'Authorization': self.make_header('token')})
+        before_delete_json_proj = resp.json()
-        self.assertEqual(200, resp.status_code, resp.data)
-        before_delete_json_proj = json.loads(resp.data)
 
         # Delete the project.
-        resp = self.client.delete('/api/projects/%s' % before_delete_json_proj['_id'],
+        self.delete(f'/api/projects/{before_delete_json_proj["_id"]}',
-                                  headers={'Authorization': self.make_header('token'),
+                    auth_token='token', etag=before_delete_json_proj['_etag'],
-                                           'If-Match': before_delete_json_proj['_etag']})
+                    expected_status=204)
-        self.assertEqual(204, resp.status_code, resp.data)
 
         # Recreate home project by getting it.
-        resp = self.client.get('/api/bcloud/home-project',
+        resp = self.get('/api/bcloud/home-project', auth_token='token')
-                               headers={'Authorization': self.make_header('token')})
+        after_delete_json_proj = resp.json()
-        self.assertEqual(200, resp.status_code, resp.data)
-        after_delete_json_proj = json.loads(resp.data)
 
         self.assertEqual(before_delete_json_proj['_id'],
                          after_delete_json_proj['_id'])

tests/test_api/test_project_management.py

@@ -1,13 +1,15 @@
-# -*- encoding: utf-8 -*-
-
 """Unit tests for creating and editing projects_blueprint."""
 
+import datetime
 import functools
 import json
 import logging
 import urllib.request, urllib.parse, urllib.error
 
 from bson import ObjectId
+import bson.tz_util
+from pymongo.collection import ReturnDocument
+
 from pillar.tests import AbstractPillarTest
 
 log = logging.getLogger(__name__)
@@ -370,6 +372,68 @@ class ProjectEditTest(AbstractProjectTest):
                                            'If-Match': project_info['_etag']})
         self.assertEqual(204, resp.status_code, resp.data)
 
+    def test_delete_files_too(self):
+        # Create test project with a file.
+        project_info = self._create_user_and_project(['subscriber'])
+        project_id = project_info['_id']
+
+        fid, _ = self.ensure_file_exists({'project': ObjectId(project_id)})
+        with self.app.app_context():
+            files_coll = self.app.db('files')
+            nowish = datetime.datetime.now(tz=bson.tz_util.utc) - datetime.timedelta(seconds=5)
+            db_file_before = files_coll.find_one_and_update({'_id': fid},
+                                                            {'$set': {'_updated': nowish}},
+                                                            return_document=ReturnDocument.AFTER)
+
+        # DELETE by owner should also soft-delete the file documents.
+        self.delete(f'/api/projects/{project_id}',
+                    auth_token='token', etag=project_info['_etag'],
+                    expected_status=204)
+
+        resp = self.get(f'/api/files/{fid}', expected_status=404)
+        self.assertEqual(str(fid), resp.json()['_id'])
+
+        with self.app.app_context():
+            db_file_after = files_coll.find_one(fid)
+        self.assertGreater(db_file_after['_updated'], db_file_before['_updated'])
+        self.assertNotEqual(db_file_after['_etag'], db_file_before['_etag'])
+
+    def test_undelete_with_put__files_too(self):
+        from pillar.api.utils import remove_private_keys
+
+        # Create test project with a file.
+        project_info = self._create_user_and_project(['subscriber'])
+        project_id = project_info['_id']
+
+        fid, _ = self.ensure_file_exists({'project': ObjectId(project_id)})
+
+        # DELETE by owner should also soft-delete the file documents.
+        proj_url = f'/api/projects/{project_id}'
+        self.delete(proj_url, auth_token='token', etag=project_info['_etag'],
+                    expected_status=204)
+
+        resp = self.get(proj_url, auth_token='token', expected_status=404)
+        etag = resp.json()['_etag']
+
+        with self.app.app_context():
+            files_coll = self.app.db('files')
+            now = datetime.datetime.now(tz=bson.tz_util.utc) - datetime.timedelta(seconds=5)
+            db_file_before = files_coll.find_one_and_update({'_id': fid},
+                                                            {'$set': {'_updated': now}},
+                                                            return_document=ReturnDocument.AFTER)
+
+        # PUT on the project should also restore the files.
+        self.put(proj_url, auth_token='token', etag=etag,
+                 json=remove_private_keys(project_info))
+
+        resp = self.get(f'/api/files/{fid}')
+        self.assertEqual(str(fid), resp.json()['_id'])
+
+        with self.app.app_context():
+            db_file_after = files_coll.find_one(fid)
+        self.assertGreater(db_file_after['_updated'], db_file_before['_updated'])
+        self.assertNotEqual(db_file_after['_etag'], db_file_before['_etag'])
+
 
 class ProjectNodeAccess(AbstractProjectTest):
     def setUp(self, **kwargs):