Added Organization Manager.

This is a Flamenco/Attract-style Manager object that's instantiated by
the PillarApplication. It can create Organizations and assign/remove
users.

Also I updated the Organization schema to reflect the currently desired
design.

NOTA BENE: this does not include any security/authorisation checks on Eve's
organizations collection.

pillar/__init__.py

@@ -39,6 +39,7 @@ import pillar.web.jinja
 from . import api
 from . import web
 from . import auth
+import pillar.api.organizations
 
 empty_settings = {
     # Use a random URL prefix when booting Eve, to ensure that any
@@ -121,6 +122,8 @@ class PillarServer(Eve):
         # Celery itself is configured after all extensions have loaded.
         self.celery: Celery = None
 
+        self.org_manager = pillar.api.organizations.OrgManager()
+
         self.before_first_request(self.setup_db_indices)
 
     def _load_flask_config(self):

pillar/api/eve_settings.py

@@ -138,14 +138,9 @@ organizations_schema = {
         'maxlength': 128,
         'required': True
     },
-    'email': {
-        'type': 'string'
-    },
     'url': {
         'type': 'string',
-        'minlength': 1,
         'maxlength': 128,
-        'required': True
     },
     'description': {
         'type': 'string',
@@ -162,7 +157,15 @@ organizations_schema = {
     'picture': dict(
         nullable=True,
         **_file_embedded_schema),
-    'users': {
+    'admin_uid': {
+        'type': 'objectid',
+        'data_relation': {
+            'resource': 'users',
+            'field': '_id',
+        },
+        'required': True,
+    },
+    'members': {
         'type': 'list',
         'default': [],
         'schema': {
@@ -170,50 +173,37 @@ organizations_schema = {
             'data_relation': {
                 'resource': 'users',
                 'field': '_id',
-                'embeddable': True
             }
         }
     },
-    'teams': {
+    'unknown_members': {
+        'type': 'list',  # of email addresses of yet-to-register users.
+        'default': [],
+        'schema': {
+            'type': 'string',
+        },
+    },
+
+    # Maximum size of the organization, i.e. len(members) + len(unknown_members) may
+    # not exceed this.
+    'seat_count': {
+        'type': 'integer',
+        'required': True,
+    },
+
+    # Roles that the members of this organization automatically get.
+    'org_roles': {
         'type': 'list',
         'default': [],
         'schema': {
-            'type': 'dict',
+            'type': 'string',
-            'schema': {
+        },
-                # Team name
+    },
-                'name': {
+
-                    'type': 'string',
+    # Identification of the subscription that pays for this organisation
-                    'minlength': 1,
+    # in an external subscription/payment management system.
-                    'maxlength': 128,
+    'payment_subscription_id': {
-                    'required': True
+        'type': 'string',
-                },
-                # List of user ids for the team
-                'users': {
-                    'type': 'list',
-                    'default': [],
-                    'schema': {
-                        'type': 'objectid',
-                        'data_relation': {
-                            'resource': 'users',
-                            'field': '_id',
-                        }
-                    }
-                },
-                # List of groups assigned to the team (this will automatically
-                # update the groups property of each user in the team)
-                'groups': {
-                    'type': 'list',
-                    'default': [],
-                    'schema': {
-                        'type': 'objectid',
-                        'data_relation': {
-                            'resource': 'groups',
-                            'field': '_id',
-                        }
-                    }
-                }
-            }
-        }
     }
 }
 
@@ -751,8 +741,6 @@ groups = {
 
 organizations = {
     'schema': organizations_schema,
-    'public_item_methods': ['GET'],
-    'public_methods': ['GET']
 }
 
 projects = {

pillar/api/organizations/__init__.py

@@ -0,0 +1,244 @@
+"""Organization management.
+
+Assumes role names that are given to users by organization membership
+start with the string "org-".
+"""
+
+import enum
+import logging
+import typing
+
+import attr
+import bson
+
+from pillar import attrs_extra, current_app
+from pillar.api.utils import remove_private_keys
+
+
+class OrganizationError(Exception):
+    """Superclass for all Organization-related errors."""
+
+
+@attr.s
+class NotEnoughSeats(OrganizationError):
+    """Thrown when trying to add too many members to the organization."""
+
+    org_id = attr.ib(validator=attr.validators.instance_of(bson.ObjectId))
+    seat_count = attr.ib(validator=attr.validators.instance_of(int))
+    attempted_seat_count = attr.ib(validator=attr.validators.instance_of(int))
+
+
+@attr.s
+class OrgManager:
+    """Organization manager.
+
+    Performs actions on an Organization. Does *NOT* test user permissions -- the caller
+    is responsible for that.
+    """
+
+    _log = attrs_extra.log('%s.OrgManager' % __name__)
+
+    def create_new_org(self,
+                       name: str,
+                       admin_uid: bson.ObjectId,
+                       seat_count: int,
+                       *,
+                       org_roles: typing.Iterable[str] = None) -> dict:
+        """Creates a new Organization.
+
+        Returns the new organization document.
+        """
+
+        assert isinstance(admin_uid, bson.ObjectId)
+
+        org_doc = {
+            'name': name,
+            'admin_uid': admin_uid,
+            'seat_count': seat_count,
+        }
+
+        if org_roles:
+            org_doc['org_roles'] = list(org_roles)
+
+        r, _, _, status = current_app.post_internal('organizations', org_doc)
+        if status != 201:
+            self._log.error('Error creating organization; status should be 201, not %i: %s',
+                            status, r)
+            raise ValueError(f'Unable to create organization, status code {status}')
+
+        org_doc.update(r)
+        return org_doc
+
+    def assign_users(self,
+                     org_id: bson.ObjectId,
+                     emails: typing.List[str]) -> dict:
+        """Assigns users to the organization.
+
+        Checks the seat count and throws a NotEnoughSeats exception when the
+        seat count is not sufficient to assign the requested users.
+
+        Users are looked up by email address, and known users are
+        automatically mapped.
+
+        :returns: the new organization document.
+        """
+
+        self._log.info('Adding %i new members to organization %s', len(emails), org_id)
+
+        users_coll = current_app.db('users')
+        existing_user_docs = list(users_coll.find({'email': {'$in': emails}},
+                                                  projection={'_id': 1, 'email': 1}))
+        unknown_users = set(emails) - {user['email'] for user in existing_user_docs}
+        existing_users = {user['_id'] for user in existing_user_docs}
+
+        if self._log.isEnabledFor(logging.INFO):
+            self._log.info('  - found users: %s', ', '.join(str(uid) for uid in existing_users))
+            self._log.info('  - unknown users: %s', ', '.join(unknown_users))
+
+        org_doc = self._get_org(org_id)
+
+        # Compute the new members.
+        members = set(org_doc.get('members') or []) | existing_users
+        unknown_members = set(org_doc.get('unknown_members')) | unknown_users
+
+        # Make sure we don't exceed the current seat count.
+        new_seat_count = len(members) + len(unknown_members)
+        if new_seat_count > org_doc['seat_count']:
+            self._log.warning('assign_users(%s, ...): Trying to increase seats to %i, '
+                              'but org only has %i seats.',
+                              org_id, new_seat_count, org_doc['seat_count'])
+            raise NotEnoughSeats(org_id, org_doc['seat_count'], new_seat_count)
+
+        # Update the organization.
+        org_doc['members'] = list(members)
+        org_doc['unknown_members'] = list(unknown_members)
+
+        r, _, _, status = current_app.put_internal('organizations',
+                                                   remove_private_keys(org_doc),
+                                                   _id=org_id)
+        if status != 200:
+            self._log.error('Error updating organization; status should be 200, not %i: %s',
+                            status, r)
+            raise ValueError(f'Unable to update organization, status code {status}')
+        org_doc.update(r)
+
+        # Update the roles for the affected members
+        for uid in existing_users:
+            self.refresh_roles(uid)
+
+        return org_doc
+
+    def remove_user(self,
+                    org_id: bson.ObjectId,
+                    *,
+                    user_id: bson.ObjectId = None,
+                    email: str = None) -> dict:
+        """Removes a user from the organization.
+
+        The user can be identified by either user ID or email.
+
+        Returns the new organization document.
+        """
+
+        users_coll = current_app.db('users')
+
+        assert user_id or email
+
+        # Collect the email address if not given. This ensures the removal
+        # if the email was accidentally in the unknown_members list.
+        if email is None:
+            user_doc = users_coll.find_one(user_id, projection={'email': 1})
+            if user_doc is not None:
+                email = user_doc['email']
+
+        # See if we know this user.
+        if user_id is None:
+            user_doc = users_coll.find_one({'email': email}, projection={'_id': 1})
+            if user_doc is not None:
+                user_id = user_doc['_id']
+
+        self._log.info('Removing user %s / %s from organization %s', user_id, email, org_id)
+
+        org_doc = self._get_org(org_id)
+
+        # Compute the new members.
+        if user_id:
+            members = set(org_doc.get('members') or []) - {user_id}
+            org_doc['members'] = list(members)
+
+        if email:
+            unknown_members = set(org_doc.get('unknown_members')) - {email}
+            org_doc['unknown_members'] = list(unknown_members)
+
+        r, _, _, status = current_app.put_internal('organizations',
+                                                   remove_private_keys(org_doc),
+                                                   _id=org_id)
+        if status != 200:
+            self._log.error('Error updating organization; status should be 200, not %i: %s',
+                            status, r)
+            raise ValueError(f'Unable to update organization, status code {status}')
+        org_doc.update(r)
+
+        # Update the roles for the affected member.
+        if user_id:
+            self.refresh_roles(user_id)
+
+        return org_doc
+
+    def _get_org(self, org_id: bson.ObjectId):
+        """Returns the organization, or raises a ValueError."""
+
+        assert isinstance(org_id, bson.ObjectId)
+
+        org_coll = current_app.db('organizations')
+        org = org_coll.find_one(org_id)
+        if org is None:
+            raise ValueError(f'Organization {org_id} not found')
+        return org
+
+    def refresh_roles(self, user_id: bson.ObjectId):
+        """Refreshes the user's roles to own roles + organizations' roles."""
+
+        from pillar.api.service import do_badger
+
+        org_coll = current_app.db('organizations')
+
+        # Aggregate all org-given roles for this user.
+        query = org_coll.aggregate([
+            {'$match': {'members': user_id}},
+            {'$project': {'org_roles': 1}},
+            {'$unwind': {'path': '$org_roles'}},
+            {'$group': {
+                '_id': None,
+                'org_roles': {'$addToSet': '$org_roles'},
+            }}])
+
+        # If the user has no organizations at all, the query will have no results.
+        try:
+            org_roles_doc = query.next()
+        except StopIteration:
+            org_roles = set()
+        else:
+            org_roles = set(org_roles_doc['org_roles'])
+
+        users_coll = current_app.db('users')
+        user_doc = users_coll.find_one(user_id, projection={'roles': 1})
+
+        all_user_roles = set(user_doc.get('roles') or [])
+        existing_org_roles = {role for role in all_user_roles
+                              if role.startswith('org-')}
+
+        grant_roles = org_roles - all_user_roles
+        revoke_roles = existing_org_roles - org_roles
+
+        if grant_roles:
+            do_badger('grant', roles=grant_roles, user_id=user_id)
+        if revoke_roles:
+            do_badger('revoke', roles=revoke_roles, user_id=user_id)
+
+# def setup_app(app):
+#     from . import eve_hooks, api, patch
+#
+#     eve_hooks.setup_app(app)
+#     api.setup_app(app)
+#     patch.setup_app(app)

tests/test_api/test_organizations.py

@@ -0,0 +1,158 @@
+from pillar.tests import AbstractPillarTest
+
+import bson
+
+
+class OrganizationCruTest(AbstractPillarTest):
+    """Test creating and updating organizations."""
+
+    def test_create_org(self):
+        self.enter_app_context()
+
+        # There should be no organizations to begin with.
+        db = self.app.db('organizations')
+        self.assertEqual(0, db.count())
+
+        admin_uid = self.create_user(24 * 'a')
+        org_doc = self.app.org_manager.create_new_org('Хакеры', admin_uid, 25)
+
+        self.assertIsNotNone(db.find_one(org_doc['_id']))
+        self.assertEqual(bson.ObjectId(24 * 'a'), org_doc['admin_uid'])
+        self.assertEqual('Хакеры', org_doc['name'])
+        self.assertEqual(25, org_doc['seat_count'])
+
+    def test_assign_users(self):
+        self.enter_app_context()
+
+        admin_uid = self.create_user(24 * 'a')
+        member1_uid = self.create_user(24 * 'b', email='member1@example.com')
+
+        om = self.app.org_manager
+        org_doc = om.create_new_org('Хакеры', admin_uid, 25)
+
+        new_org_doc = om.assign_users(
+            org_doc['_id'],
+            ['member1@example.com', 'member2@example.com'])
+
+        db = self.app.db('organizations')
+        db_org = db.find_one(org_doc['_id'])
+
+        self.assertEqual([member1_uid], db_org['members'])
+        self.assertEqual(['member2@example.com'], db_org['unknown_members'])
+
+        self.assertEqual([member1_uid], new_org_doc['members'])
+        self.assertEqual(['member2@example.com'], new_org_doc['unknown_members'])
+
+    def test_remove_users(self):
+        self.enter_app_context()
+        om = self.app.org_manager
+
+        admin_uid = self.create_user(24 * 'a')
+        self.create_user(24 * 'b', email='member1@example.com')
+        org_doc = om.create_new_org('Хакеры', admin_uid, 25)
+
+        om.assign_users(
+            org_doc['_id'],
+            ['member1@example.com', 'member2@example.com'])
+
+        new_org_doc = None  # to prevent 'might not be assigned' warning later on.
+        for email in ('member1@example.com', 'member2@example.com'):
+            new_org_doc = om.remove_user(org_doc['_id'], email=email)
+
+        db = self.app.db('organizations')
+        db_org = db.find_one(org_doc['_id'])
+
+        self.assertEqual([], db_org['members'])
+        self.assertEqual([], db_org['unknown_members'])
+
+        self.assertEqual([], new_org_doc['members'])
+        self.assertEqual([], new_org_doc['unknown_members'])
+
+    def test_assign_user_roles(self):
+        self.enter_app_context()
+
+        admin_uid = self.create_user(24 * 'a')
+        member1_uid = self.create_user(24 * 'b',
+                                       email='member1@example.com',
+                                       roles={'subscriber', 'monkeyhead'})
+        om = self.app.org_manager
+        org_doc = om.create_new_org('Хакеры', admin_uid, 25,
+                                    org_roles=['org-xакеры'])
+
+        new_org_doc = om.assign_users(org_doc['_id'], ['member1@example.com'])
+        self.assertEqual(['org-xакеры'], new_org_doc['org_roles'])
+
+        users_coll = self.app.db('users')
+
+        member1_doc = users_coll.find_one(member1_uid)
+        self.assertEqual(set(member1_doc['roles']), {'subscriber', 'monkeyhead', 'org-xакеры'})
+
+    def test_revoke_user_roles_simple(self):
+        self.enter_app_context()
+
+        admin_uid = self.create_user(24 * 'a')
+        member1_uid = self.create_user(24 * 'b',
+                                       email='member1@example.com',
+                                       roles={'subscriber', 'monkeyhead'})
+        om = self.app.org_manager
+        org_doc = om.create_new_org('Хакеры', admin_uid, 25, org_roles=['org-xакеры'])
+
+        om.assign_users(org_doc['_id'], ['member1@example.com'])
+        om.remove_user(org_doc['_id'], email='member1@example.com')
+
+        users_coll = self.app.db('users')
+
+        member1_doc = users_coll.find_one(member1_uid)
+        self.assertEqual(set(member1_doc['roles']), {'subscriber', 'monkeyhead'})
+
+    def test_revoke_user_roles_multiorg_by_email(self):
+        self.enter_app_context()
+
+        admin_uid = self.create_user(24 * 'a')
+        member1_uid = self.create_user(24 * 'b',
+                                       email='member1@example.com',
+                                       roles={'subscriber', 'monkeyhead'})
+        om = self.app.org_manager
+        org1 = om.create_new_org('Хакеры', admin_uid, 25, org_roles=['org-xакеры', 'org-subs'])
+        org2 = om.create_new_org('अजिङ्गर', admin_uid, 25, org_roles=['org-अजिङ्गर', 'org-subs'])
+
+        om.assign_users(org1['_id'], ['member1@example.com'])
+        om.assign_users(org2['_id'], ['member1@example.com'])
+        om.remove_user(org1['_id'], email='member1@example.com')
+
+        users_coll = self.app.db('users')
+
+        member1_doc = users_coll.find_one(member1_uid)
+        self.assertEqual(set(member1_doc['roles']),
+                         {'subscriber', 'monkeyhead', 'org-subs', 'org-अजिङ्गर'})
+
+    def test_revoke_user_roles_multiorg_by_user_id(self):
+        self.enter_app_context()
+
+        admin_uid = self.create_user(24 * 'a')
+        member1_uid = self.create_user(24 * 'b',
+                                       email='member1@example.com',
+                                       roles={'subscriber', 'monkeyhead'})
+        om = self.app.org_manager
+        org1 = om.create_new_org('Хакеры', admin_uid, 25, org_roles=['org-xакеры', 'org-subs'])
+        org2 = om.create_new_org('अजिङ्गर', admin_uid, 25, org_roles=['org-अजिङ्गर', 'org-subs'])
+
+        # Hack the DB to add the member as "unknown member" too, even though we know this user.
+        # This has to be handled cleanly by the removal too.
+        orgs_coll = self.app.db('organizations')
+        orgs_coll.update_one({'_id': org1['_id']},
+                             {'$set': {'unknown_members': ['member1@example.com']}})
+
+        om.assign_users(org1['_id'], ['member1@example.com'])
+        om.assign_users(org2['_id'], ['member1@example.com'])
+        om.remove_user(org1['_id'], user_id=member1_uid)
+
+        users_coll = self.app.db('users')
+
+        member1_doc = users_coll.find_one(member1_uid)
+        self.assertEqual(set(member1_doc['roles']),
+                         {'subscriber', 'monkeyhead', 'org-subs', 'org-अजिङ्गर'})
+
+        # The unknown members list should be empty.
+        db_org1 = orgs_coll.find_one(org1['_id'])
+        self.assertEqual(db_org1['unknown_members'], [])