From f8ff30fb4de6162b0e650c53fea41492921b4b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 30 Jan 2018 17:29:28 +0100 Subject: [PATCH] (un)delete on project also (un)delete file documents. Note that undeleting files cannot be done via Eve, as it doesn't support PATCHing collections. Instead, direct MongoDB modification is used to set _deleted=False and provide new _etag and _updated values. --- pillar/api/projects/__init__.py | 7 +++ pillar/api/projects/hooks.py | 55 +++++++++++++++-- pillar/api/utils/__init__.py | 18 ++++-- tests/test_api/test_bcloud_home_project.py | 19 +++--- tests/test_api/test_project_management.py | 68 +++++++++++++++++++++- 5 files changed, 144 insertions(+), 23 deletions(-) diff --git a/pillar/api/projects/__init__.py b/pillar/api/projects/__init__.py index f4240736..808a83e0 100644 --- a/pillar/api/projects/__init__.py +++ b/pillar/api/projects/__init__.py @@ -6,10 +6,17 @@ def setup_app(app, api_prefix): app.on_replace_projects += hooks.override_is_private_field app.on_replace_projects += hooks.before_edit_check_permissions app.on_replace_projects += hooks.protect_sensitive_fields + + app.on_replaced_projects += hooks.after_undelete_project + app.on_updated_projects += hooks.after_undelete_project + app.on_update_projects += hooks.override_is_private_field app.on_update_projects += hooks.before_edit_check_permissions app.on_update_projects += hooks.protect_sensitive_fields + app.on_delete_item_projects += hooks.before_delete_project + app.on_deleted_item_projects += hooks.after_delete_project + app.on_insert_projects += hooks.before_inserting_override_is_private_field app.on_insert_projects += hooks.before_inserting_projects app.on_inserted_projects += hooks.after_inserting_projects diff --git a/pillar/api/projects/hooks.py b/pillar/api/projects/hooks.py index 0365e1d5..4a269c80 100644 --- a/pillar/api/projects/hooks.py +++ b/pillar/api/projects/hooks.py @@ -1,8 +1,11 @@ import copy +import datetime import logging -from flask import request, abort, current_app +import bson.tz_util +from flask import request, abort +from pillar import current_app from pillar.api.node_types.asset import node_type_asset from pillar.api.node_types.comment import node_type_comment from pillar.api.node_types.group import node_type_group @@ -10,8 +13,9 @@ from pillar.api.node_types.group_texture import node_type_group_texture from pillar.api.node_types.texture import node_type_texture from pillar.api.file_storage_backends import default_storage_backend from pillar.api.utils import authorization, authentication -from pillar.api.utils import remove_private_keys +from pillar.api.utils import remove_private_keys, random_etag from pillar.api.utils.authorization import user_has_role, check_permissions +from pillar.auth import current_user from .utils import abort_with_error log = logging.getLogger(__name__) @@ -64,6 +68,51 @@ def before_delete_project(document): """Checks permissions before we allow deletion""" check_permissions('projects', document, request.method) + log.info('Deleting project %s on behalf of user %s', document['_id'], current_user) + + +def after_delete_project(project: dict): + """Perform delete on the project's files too.""" + + from eve.methods.delete import delete + + pid = project['_id'] + log.info('Project %s was deleted, also deleting its files.', pid) + + r, _, _, status = delete('files', {'project': pid}) + if status != 204: + log.warning('Unable to delete files of project %s: %s', pid, r) + + +def after_undelete_project(project: dict, original: dict): + """Undelete the files that belong to this project. + + We cannot do this via Eve, as it doesn't support PATCHing collections, so + direct MongoDB modification is used to set _deleted=False and provide + new _etag and _updated values. + """ + + if not original: + return + + was_deleted = original.get('_deleted', False) + now_deleted = project.get('_deleted', False) + if not was_deleted or now_deleted: + # This is not an undelete. + return + + pid = project['_id'] + new_etag = random_etag() + now = datetime.datetime.now(tz=bson.tz_util.utc) + + files_coll = current_app.db('files') + update_result = files_coll.update_many( + {'project': pid}, + {'$set': {'_deleted': False, + '_etag': new_etag, + '_updated': now}}) + log.info('undeleted %d of %d file documents of project %s', + update_result.modified_count, update_result.matched_count, pid) def protect_sensitive_fields(document, original): @@ -225,5 +274,3 @@ def project_node_type_has_method(response): def projects_node_type_has_method(response): for project in response['_items']: project_node_type_has_method(project) - - diff --git a/pillar/api/utils/__init__.py b/pillar/api/utils/__init__.py index b355a330..f2f8b78f 100644 --- a/pillar/api/utils/__init__.py +++ b/pillar/api/utils/__init__.py @@ -1,12 +1,13 @@ +import base64 import copy -import hashlib -import json -import typing -import urllib.request, urllib.parse, urllib.error - import datetime import functools +import hashlib +import json import logging +import random +import typing +import urllib.request, urllib.parse, urllib.error import bson.objectid from eve import RFC1123_DATE_FORMAT @@ -192,3 +193,10 @@ def doc_diff(doc1, doc2, falsey_is_equal=True): continue yield key, val1, val2 + + +def random_etag() -> str: + """Random string usable as etag.""" + + randbytes = random.getrandbits(256).to_bytes(32, 'big') + return base64.b64encode(randbytes)[:-1].decode() diff --git a/tests/test_api/test_bcloud_home_project.py b/tests/test_api/test_bcloud_home_project.py index 90532d3e..9730618e 100644 --- a/tests/test_api/test_bcloud_home_project.py +++ b/tests/test_api/test_bcloud_home_project.py @@ -295,22 +295,17 @@ class HomeProjectTest(AbstractHomeProjectTest): self._create_user_with_token(roles={'subscriber'}, token='token') # Create home project by getting it. - resp = self.client.get('/api/bcloud/home-project', - headers={'Authorization': self.make_header('token')}) - self.assertEqual(200, resp.status_code, resp.data) - before_delete_json_proj = json.loads(resp.data) + resp = self.get('/api/bcloud/home-project', auth_token='token') + before_delete_json_proj = resp.json() # Delete the project. - resp = self.client.delete('/api/projects/%s' % before_delete_json_proj['_id'], - headers={'Authorization': self.make_header('token'), - 'If-Match': before_delete_json_proj['_etag']}) - self.assertEqual(204, resp.status_code, resp.data) + self.delete(f'/api/projects/{before_delete_json_proj["_id"]}', + auth_token='token', etag=before_delete_json_proj['_etag'], + expected_status=204) # Recreate home project by getting it. - resp = self.client.get('/api/bcloud/home-project', - headers={'Authorization': self.make_header('token')}) - self.assertEqual(200, resp.status_code, resp.data) - after_delete_json_proj = json.loads(resp.data) + resp = self.get('/api/bcloud/home-project', auth_token='token') + after_delete_json_proj = resp.json() self.assertEqual(before_delete_json_proj['_id'], after_delete_json_proj['_id']) diff --git a/tests/test_api/test_project_management.py b/tests/test_api/test_project_management.py index d35126a1..feee7073 100644 --- a/tests/test_api/test_project_management.py +++ b/tests/test_api/test_project_management.py @@ -1,13 +1,15 @@ -# -*- encoding: utf-8 -*- - """Unit tests for creating and editing projects_blueprint.""" +import datetime import functools import json import logging import urllib.request, urllib.parse, urllib.error from bson import ObjectId +import bson.tz_util +from pymongo.collection import ReturnDocument + from pillar.tests import AbstractPillarTest log = logging.getLogger(__name__) @@ -370,6 +372,68 @@ class ProjectEditTest(AbstractProjectTest): 'If-Match': project_info['_etag']}) self.assertEqual(204, resp.status_code, resp.data) + def test_delete_files_too(self): + # Create test project with a file. + project_info = self._create_user_and_project(['subscriber']) + project_id = project_info['_id'] + + fid, _ = self.ensure_file_exists({'project': ObjectId(project_id)}) + with self.app.app_context(): + files_coll = self.app.db('files') + nowish = datetime.datetime.now(tz=bson.tz_util.utc) - datetime.timedelta(seconds=5) + db_file_before = files_coll.find_one_and_update({'_id': fid}, + {'$set': {'_updated': nowish}}, + return_document=ReturnDocument.AFTER) + + # DELETE by owner should also soft-delete the file documents. + self.delete(f'/api/projects/{project_id}', + auth_token='token', etag=project_info['_etag'], + expected_status=204) + + resp = self.get(f'/api/files/{fid}', expected_status=404) + self.assertEqual(str(fid), resp.json()['_id']) + + with self.app.app_context(): + db_file_after = files_coll.find_one(fid) + self.assertGreater(db_file_after['_updated'], db_file_before['_updated']) + self.assertNotEqual(db_file_after['_etag'], db_file_before['_etag']) + + def test_undelete_with_put__files_too(self): + from pillar.api.utils import remove_private_keys + + # Create test project with a file. + project_info = self._create_user_and_project(['subscriber']) + project_id = project_info['_id'] + + fid, _ = self.ensure_file_exists({'project': ObjectId(project_id)}) + + # DELETE by owner should also soft-delete the file documents. + proj_url = f'/api/projects/{project_id}' + self.delete(proj_url, auth_token='token', etag=project_info['_etag'], + expected_status=204) + + resp = self.get(proj_url, auth_token='token', expected_status=404) + etag = resp.json()['_etag'] + + with self.app.app_context(): + files_coll = self.app.db('files') + now = datetime.datetime.now(tz=bson.tz_util.utc) - datetime.timedelta(seconds=5) + db_file_before = files_coll.find_one_and_update({'_id': fid}, + {'$set': {'_updated': now}}, + return_document=ReturnDocument.AFTER) + + # PUT on the project should also restore the files. + self.put(proj_url, auth_token='token', etag=etag, + json=remove_private_keys(project_info)) + + resp = self.get(f'/api/files/{fid}') + self.assertEqual(str(fid), resp.json()['_id']) + + with self.app.app_context(): + db_file_after = files_coll.find_one(fid) + self.assertGreater(db_file_after['_updated'], db_file_before['_updated']) + self.assertNotEqual(db_file_after['_etag'], db_file_before['_etag']) + class ProjectNodeAccess(AbstractProjectTest): def setUp(self, **kwargs):