archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

(un)delete on project also (un)delete file documents.

Browse Source 
Note that undeleting files cannot be done via Eve, as it doesn't support
PATCHing collections. Instead, direct MongoDB modification is used to set
_deleted=False and provide new _etag and _updated values.
This commit is contained in:
Sybren A. Stüvel 2018-01-30 17:29:28 +01:00
parent 7d1b08bf58
commit f8ff30fb4d
5 changed files with 144 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pillar/api/projects/__init__.py
View File

@ -6,10 +6,17 @@ def setup_app(app, api_prefix):
app.on_replace_projects += hooks.override_is_private_field
app.on_replace_projects += hooks.before_edit_check_permissions
app.on_replace_projects += hooks.protect_sensitive_fields
app.on_replaced_projects += hooks.after_undelete_project
app.on_updated_projects += hooks.after_undelete_project
app.on_update_projects += hooks.override_is_private_field
app.on_update_projects += hooks.before_edit_check_permissions
app.on_update_projects += hooks.protect_sensitive_fields
app.on_delete_item_projects += hooks.before_delete_project
app.on_deleted_item_projects += hooks.after_delete_project
app.on_insert_projects += hooks.before_inserting_override_is_private_field
app.on_insert_projects += hooks.before_inserting_projects
app.on_inserted_projects += hooks.after_inserting_projects

55
pillar/api/projects/hooks.py
View File

@ -1,8 +1,11 @@
import copy
import datetime
import logging
from flask import request, abort, current_app
import bson.tz_util
from flask import request, abort
from pillar import current_app
from pillar.api.node_types.asset import node_type_asset
from pillar.api.node_types.comment import node_type_comment
from pillar.api.node_types.group import node_type_group
@ -10,8 +13,9 @@ from pillar.api.node_types.group_texture import node_type_group_texture
from pillar.api.node_types.texture import node_type_texture
from pillar.api.file_storage_backends import default_storage_backend
from pillar.api.utils import authorization, authentication
from pillar.api.utils import remove_private_keys
from pillar.api.utils import remove_private_keys, random_etag
from pillar.api.utils.authorization import user_has_role, check_permissions
from pillar.auth import current_user
from .utils import abort_with_error
log = logging.getLogger(__name__)
@ -64,6 +68,51 @@ def before_delete_project(document):
"""Checks permissions before we allow deletion"""
check_permissions('projects', document, request.method)
log.info('Deleting project %s on behalf of user %s', document['_id'], current_user)
def after_delete_project(project: dict):
"""Perform delete on the project's files too."""
from eve.methods.delete import delete
pid = project['_id']
log.info('Project %s was deleted, also deleting its files.', pid)
r, _, _, status = delete('files', {'project': pid})
if status != 204:
log.warning('Unable to delete files of project %s: %s', pid, r)
def after_undelete_project(project: dict, original: dict):
"""Undelete the files that belong to this project.
We cannot do this via Eve, as it doesn't support PATCHing collections, so
direct MongoDB modification is used to set _deleted=False and provide
new _etag and _updated values.
"""
if not original:
return
was_deleted = original.get('_deleted', False)
now_deleted = project.get('_deleted', False)
if not was_deleted or now_deleted:
# This is not an undelete.
return
pid = project['_id']
new_etag = random_etag()
now = datetime.datetime.now(tz=bson.tz_util.utc)
files_coll = current_app.db('files')
update_result = files_coll.update_many(
{'project': pid},
{'$set': {'_deleted': False,
'_etag': new_etag,
'_updated': now}})
log.info('undeleted %d of %d file documents of project %s',
update_result.modified_count, update_result.matched_count, pid)
def protect_sensitive_fields(document, original):
@ -225,5 +274,3 @@ def project_node_type_has_method(response):
def projects_node_type_has_method(response):
for project in response['_items']:
project_node_type_has_method(project)

18
pillar/api/utils/__init__.py
View File

@ -1,12 +1,13 @@
import base64
import copy
import hashlib
import json
import typing
import urllib.request, urllib.parse, urllib.error
import datetime
import functools
import hashlib
import json
import logging
import random
import typing
import urllib.request, urllib.parse, urllib.error
import bson.objectid
from eve import RFC1123_DATE_FORMAT
@ -192,3 +193,10 @@ def doc_diff(doc1, doc2, falsey_is_equal=True):
continue
yield key, val1, val2
def random_etag() -> str:
"""Random string usable as etag."""
randbytes = random.getrandbits(256).to_bytes(32, 'big')
return base64.b64encode(randbytes)[:-1].decode()

19
tests/test_api/test_bcloud_home_project.py
View File

@ -295,22 +295,17 @@ class HomeProjectTest(AbstractHomeProjectTest):
self._create_user_with_token(roles={'subscriber'}, token='token')
# Create home project by getting it.
resp = self.client.get('/api/bcloud/home-project',
headers={'Authorization': self.make_header('token')})
self.assertEqual(200, resp.status_code, resp.data)
before_delete_json_proj = json.loads(resp.data)
resp = self.get('/api/bcloud/home-project', auth_token='token')
before_delete_json_proj = resp.json()
# Delete the project.
resp = self.client.delete('/api/projects/%s' % before_delete_json_proj['_id'],
headers={'Authorization': self.make_header('token'),
'If-Match': before_delete_json_proj['_etag']})
self.assertEqual(204, resp.status_code, resp.data)
self.delete(f'/api/projects/{before_delete_json_proj["_id"]}',
auth_token='token', etag=before_delete_json_proj['_etag'],
expected_status=204)
# Recreate home project by getting it.
resp = self.client.get('/api/bcloud/home-project',
headers={'Authorization': self.make_header('token')})
self.assertEqual(200, resp.status_code, resp.data)
after_delete_json_proj = json.loads(resp.data)
resp = self.get('/api/bcloud/home-project', auth_token='token')
after_delete_json_proj = resp.json()
self.assertEqual(before_delete_json_proj['_id'],
after_delete_json_proj['_id'])

68
tests/test_api/test_project_management.py
View File

@ -1,13 +1,15 @@
# -*- encoding: utf-8 -*-
"""Unit tests for creating and editing projects_blueprint."""
import datetime
import functools
import json
import logging
import urllib.request, urllib.parse, urllib.error
from bson import ObjectId
import bson.tz_util
from pymongo.collection import ReturnDocument
from pillar.tests import AbstractPillarTest
log = logging.getLogger(__name__)
@ -370,6 +372,68 @@ class ProjectEditTest(AbstractProjectTest):
'If-Match': project_info['_etag']})
self.assertEqual(204, resp.status_code, resp.data)
def test_delete_files_too(self):
# Create test project with a file.
project_info = self._create_user_and_project(['subscriber'])
project_id = project_info['_id']
fid, _ = self.ensure_file_exists({'project': ObjectId(project_id)})
with self.app.app_context():
files_coll = self.app.db('files')
nowish = datetime.datetime.now(tz=bson.tz_util.utc) - datetime.timedelta(seconds=5)
db_file_before = files_coll.find_one_and_update({'_id': fid},
{'$set': {'_updated': nowish}},
return_document=ReturnDocument.AFTER)
# DELETE by owner should also soft-delete the file documents.
self.delete(f'/api/projects/{project_id}',
auth_token='token', etag=project_info['_etag'],
expected_status=204)
resp = self.get(f'/api/files/{fid}', expected_status=404)
self.assertEqual(str(fid), resp.json()['_id'])
with self.app.app_context():
db_file_after = files_coll.find_one(fid)
self.assertGreater(db_file_after['_updated'], db_file_before['_updated'])
self.assertNotEqual(db_file_after['_etag'], db_file_before['_etag'])
def test_undelete_with_put__files_too(self):
from pillar.api.utils import remove_private_keys
# Create test project with a file.
project_info = self._create_user_and_project(['subscriber'])
project_id = project_info['_id']
fid, _ = self.ensure_file_exists({'project': ObjectId(project_id)})
# DELETE by owner should also soft-delete the file documents.
proj_url = f'/api/projects/{project_id}'
self.delete(proj_url, auth_token='token', etag=project_info['_etag'],
expected_status=204)
resp = self.get(proj_url, auth_token='token', expected_status=404)
etag = resp.json()['_etag']
with self.app.app_context():
files_coll = self.app.db('files')
now = datetime.datetime.now(tz=bson.tz_util.utc) - datetime.timedelta(seconds=5)
db_file_before = files_coll.find_one_and_update({'_id': fid},
{'$set': {'_updated': now}},
return_document=ReturnDocument.AFTER)
# PUT on the project should also restore the files.
self.put(proj_url, auth_token='token', etag=etag,
json=remove_private_keys(project_info))
resp = self.get(f'/api/files/{fid}')
self.assertEqual(str(fid), resp.json()['_id'])
with self.app.app_context():
db_file_after = files_coll.find_one(fid)
self.assertGreater(db_file_after['_updated'], db_file_before['_updated'])
self.assertNotEqual(db_file_after['_etag'], db_file_before['_etag'])
class ProjectNodeAccess(AbstractProjectTest):
def setUp(self, **kwargs):