diff --git a/pillar/__init__.py b/pillar/__init__.py index f0959d65..fc2e01d7 100644 --- a/pillar/__init__.py +++ b/pillar/__init__.py @@ -739,6 +739,11 @@ class PillarServer(BlinkerCompatibleEve): coll.create_index([('is_private', pymongo.ASCENDING)]) coll.create_index([('category', pymongo.ASCENDING)]) + coll = db['organizations'] + coll.create_index([('ip_ranges.start', pymongo.ASCENDING)]) + coll.create_index([('ip_ranges.end', pymongo.ASCENDING)]) + self.log.debug('Created database indices') + def register_api_blueprint(self, blueprint, url_prefix): # TODO: use Eve config variable instead of hard-coded '/api' self.register_blueprint(blueprint, url_prefix='/api' + url_prefix) diff --git a/pillar/api/custom_field_validation.py b/pillar/api/custom_field_validation.py index 773acb82..d53e1dfd 100644 --- a/pillar/api/custom_field_validation.py +++ b/pillar/api/custom_field_validation.py @@ -125,3 +125,30 @@ class ValidateCustomFields(Validator): if not value: self._error(field, "Value is required once the document was created") + + def _validate_type_iprange(self, field_name: str, value: str): + """Ensure the field contains a valid IP address. + + Supports both IPv6 and IPv4 ranges. Requires the IPy module. + """ + + from IPy import IP + + try: + ip = IP(value, make_net=True) + except ValueError as ex: + self._error(field_name, str(ex)) + return + + if ip.prefixlen() == 0: + self._error(field_name, 'Zero-length prefix is not allowed') + + def _validate_type_binary(self, field_name: str, value: bytes): + """Add support for binary type. + + This type was actually introduced in Cerberus 1.0, so we can drop + support for this once Eve starts using that version (or newer). + """ + + if not isinstance(value, (bytes, bytearray)): + self._error(field_name, f'wrong value type {type(value)}, expected bytes or bytearray') diff --git a/pillar/api/eve_settings.py b/pillar/api/eve_settings.py index 2e6aeaa0..88606a95 100644 --- a/pillar/api/eve_settings.py +++ b/pillar/api/eve_settings.py @@ -214,7 +214,21 @@ organizations_schema = { # in an external subscription/payment management system. 'payment_subscription_id': { 'type': 'string', - } + }, + + 'ip_ranges': { + 'type': 'list', + 'schema': { + 'type': 'dict', + 'schema': { + # see _validate_type_{typename} in ValidateCustomFields: + 'start': {'type': 'binary', 'required': True}, + 'end': {'type': 'binary', 'required': True}, + 'prefix': {'type': 'integer', 'required': True}, + 'human': {'type': 'iprange', 'required': True}, + } + }, + }, } permissions_embedded_schema = { diff --git a/pillar/api/organizations/__init__.py b/pillar/api/organizations/__init__.py index 1c80d981..5fa6fb7e 100644 --- a/pillar/api/organizations/__init__.py +++ b/pillar/api/organizations/__init__.py @@ -383,6 +383,20 @@ class OrgManager: org_count = org_coll.count({'unknown_members': member_email}) return bool(org_count) + def roles_for_ip_address(self, remote_addr: str) -> typing.Set[str]: + """Find the roles given to the user via org IP range definitions.""" + + from . import ip_ranges + + org_coll = current_app.db('organizations') + orgs = org_coll.find( + {'ip_ranges': ip_ranges.query(remote_addr)}, + projection={'org_roles': True}, + ) + return set(role + for org in orgs + for role in org.get('org_roles', [])) + def setup_app(app): from . import patch, hooks diff --git a/pillar/api/organizations/ip_ranges.py b/pillar/api/organizations/ip_ranges.py new file mode 100644 index 00000000..a6637736 --- /dev/null +++ b/pillar/api/organizations/ip_ranges.py @@ -0,0 +1,75 @@ +"""IP range support for Organizations.""" + +from IPy import IP + +# 128 bits all set to 1 +ONES_128 = 2 ** 128 - 1 + + +def doc(iprange: str, min_prefixlen6: int=0, min_prefixlen4: int=0) -> dict: + """Convert a human-readable string like '1.2.3.4/24' to a Mongo document. + + This converts the address to IPv6 and computes the start/end addresses + of the range. The address, its prefix size, and start and end address, + are returned as a dict. + + Addresses are stored as big-endian binary data because MongoDB doesn't + support 128 bits integers. + + :param iprange: the IP address and mask size, can be IPv6 or IPv4. + :param min_prefixlen6: if given, causes a ValuError when the mask size + is too low. Note that the mask size is always + evaluated only for IPv6 addresses. + :param min_prefixlen4: if given, causes a ValuError when the mask size + is too low. Note that the mask size is always + evaluated only for IPv4 addresses. + :returns: a dict like: { + 'start': b'xxxxx' with the lowest IP address in the range. + 'end': b'yyyyy' with the highest IP address in the range. + 'human': 'aaaa:bbbb::cc00/120' with the human-readable representation. + 'prefix': 120, the prefix length of the netmask in bits. + } + """ + + ip = IP(iprange, make_net=True) + prefixlen = ip.prefixlen() + if ip.version() == 4: + if prefixlen < min_prefixlen4: + raise ValueError(f'Prefix length {prefixlen} smaller than allowed {min_prefixlen4}') + ip = ip.v46map() + else: + if prefixlen < min_prefixlen6: + raise ValueError(f'Prefix length {prefixlen} smaller than allowed {min_prefixlen6}') + + addr = ip.int() + + # Set all address bits to 1 where the mask is 0 to obtain the largest address. + end = addr | (ONES_128 % ip.netmask().int()) + + # This ensures that even a single host is represented as /128 in the human-readable form. + ip.NoPrefixForSingleIp = False + + return { + 'start': addr.to_bytes(16, 'big'), + 'end': end.to_bytes(16, 'big'), + 'human': ip.strCompressed(), + 'prefix': ip.prefixlen(), + } + + +def query(address: str) -> dict: + """Return a dict usable for querying all organizations whose IP range matches the given one. + + :returns: a dict like: + {$elemMatch: {'start': {$lte: b'xxxxx'}, 'end': {$gte: b'xxxxx'}}} + """ + + ip = IP(address) + if ip.version() == 4: + ip = ip.v46map() + for_mongo = ip.ip.to_bytes(16, 'big') + + return {'$elemMatch': { + 'start': {'$lte': for_mongo}, + 'end': {'$gte': for_mongo}, + }} diff --git a/pillar/api/organizations/patch.py b/pillar/api/organizations/patch.py index 2e69fe6e..00a9563d 100644 --- a/pillar/api/organizations/patch.py +++ b/pillar/api/organizations/patch.py @@ -135,9 +135,20 @@ class OrganizationPatchHandler(patch_handler.AbstractPatchHandler): @authorization.require_login() def patch_edit_from_web(self, org_id: bson.ObjectId, patch: dict): - """Updates Organization fields from the web.""" + """Updates Organization fields from the web. + + The PATCH command supports the following payload. The 'name' field must + be set, all other fields are optional. When an optional field is + ommitted it will be handled as an instruction to clear that field. + {'name': str, + 'description': str, + 'website': str, + 'location': str, + 'ip_ranges': list of human-readable IP ranges} + """ from pymongo.results import UpdateResult + from . import ip_ranges self._assert_is_admin(org_id) user = current_user() @@ -150,6 +161,21 @@ class OrganizationPatchHandler(patch_handler.AbstractPatchHandler): 'website': patch.get('website', '').strip(), 'location': patch.get('location', '').strip(), } + unset = {} + + # Special transformation for IP ranges + iprs = patch.get('ip_ranges') + if iprs: + ipr_docs = [] + for r in iprs: + try: + doc = ip_ranges.doc(r, min_prefixlen6=48, min_prefixlen4=8) + except ValueError as ex: + raise wz_exceptions.UnprocessableEntity(f'Invalid IP range {r!r}: {ex}') + ipr_docs.append(doc) + update['ip_ranges'] = ipr_docs + else: + unset['ip_ranges'] = True refresh_user_roles = False if user.has_cap('admin'): @@ -177,11 +203,13 @@ class OrganizationPatchHandler(patch_handler.AbstractPatchHandler): resp.status_code = 422 return resp + # Figure out what to set and what to unset + for_mongo = {'$set': update} + if unset: + for_mongo['$unset'] = unset + organizations_coll = current_app.db('organizations') - result: UpdateResult = organizations_coll.update_one( - {'_id': org_id}, - {'$set': update} - ) + result: UpdateResult = organizations_coll.update_one({'_id': org_id}, for_mongo) if result.matched_count != 1: self.log.warning('User %s edits Organization %s but update matched %i items', diff --git a/requirements.txt b/requirements.txt index 90e21060..02ce9cc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ Flask-WTF==0.12 gcloud==0.12.0 google-apitools==0.4.11 httplib2==0.9.2 +IPy==0.83 MarkupSafe==0.23 ndg-httpsclient==0.4.0 Pillow==4.1.1 diff --git a/tests/test_api/test_organizations.py b/tests/test_api/test_organizations.py index 62c6106e..a6747abe 100644 --- a/tests/test_api/test_organizations.py +++ b/tests/test_api/test_organizations.py @@ -7,6 +7,14 @@ from pillar.tests import AbstractPillarTest from pillar.api.utils import remove_private_keys +class AbstractOrgTest(AbstractPillarTest): + def setUp(self, **kwargs): + super().setUp(**kwargs) + + self.enter_app_context() + self.om = self.app.org_manager + + class OrganizationCruTest(AbstractPillarTest): """Test creating and updating organizations.""" @@ -470,15 +478,12 @@ class OrganizationPatchTest(AbstractPillarTest): self.assertEqual(admin_uid, db_org['admin_uid']) -class OrganizationResourceEveTest(AbstractPillarTest): +class OrganizationResourceEveTest(AbstractOrgTest): """Test GET/POST/PUT access to Organization resource""" def setUp(self, **kwargs): super().setUp(**kwargs) - self.enter_app_context() - self.om = self.app.org_manager - # Pillar admin self.create_user(24 * '1', roles={'admin'}, token='uberadmin') @@ -764,3 +769,179 @@ class UserCreationTest(AbstractPillarTest): db_org = self._from_db() self.assertEqual([], db_org['unknown_members']) self.assertEqual([my_id], db_org['members']) + + +class IPRangeTest(AbstractOrgTest): + + def setUp(self, **kwargs): + super().setUp(**kwargs) + self.uid = self.create_user(24 * 'a', token='token') + self.org_roles = {'org-subscriber', 'org-phabricator'} + self.org = self.app.org_manager.create_new_org('Хакеры', self.uid, 25, + org_roles=self.org_roles) + self.org_id = self.org['_id'] + + def _patch(self, payload: dict, expected_status=204) -> dict: + self.patch(f'/api/organizations/{self.org_id}', + json={ + 'op': 'edit-from-web', + 'name': self.org['name'], + **payload, + }, + auth_token='token', + expected_status=expected_status) + db_org = self.om._get_org(self.org_id) + return db_org + + def test_ipranges_doc(self): + from pillar.api.organizations import ip_ranges + + doc = ip_ranges.doc('2a03:b0c0:0:1010::8fe:6ef1/120') + self.assertEqual(0x2a03b0c0000010100000000008fe6e00.to_bytes(16, 'big'), doc['start']) + self.assertEqual(0x2a03b0c0000010100000000008fe6eff.to_bytes(16, 'big'), doc['end']) + self.assertEqual('2a03:b0c0:0:1010::8fe:6e00/120', doc['human']) + self.assertEqual(120, doc['prefix']) + self.assertEqual({'prefix', 'human', 'start', 'end'}, set(doc.keys())) + + def test_ipranges_query(self): + from pillar.api.organizations import ip_ranges + + doc = ip_ranges.query('2a03:b0c0:0:1010::8fe:6ef1') + addr = 0x2a03b0c0000010100000000008fe6ef1.to_bytes(16, 'big') + + self.assertEqual(addr, doc['$elemMatch']['start']['$lte']) + self.assertEqual(addr, doc['$elemMatch']['end']['$gte']) + + def test_patch_set_ip_ranges_happy(self): + from pillar.api.organizations import ip_ranges + + # IP ranges should be saved as integers for fast matching. + db_org = self._patch({'ip_ranges': [ + '192.168.3.0/24', + '192.168.3.1/32', + '2a03:b0c0:0:1010::8fe:6ef1/120', + ]}) + + self.assertEqual([ + ip_ranges.doc('192.168.3.0/24'), + ip_ranges.doc('192.168.3.1/32'), + ip_ranges.doc('2a03:b0c0:0:1010::8fe:6ef1/120'), + ], db_org['ip_ranges']) + + def test_patch_unset_ip_ranges_happy(self): + """Setting to empty list should just delete the entire key.""" + ipranges = [ + '192.168.3.0/24', + '48.44.12.35/32', + '2a01:7c8:aab9:3b::0/64', + ] + self._patch({'ip_ranges': ipranges}) + db_org = self._patch({'ip_ranges': []}) + self.assertNotIn('ip_ranges', db_org) + + def test_single_host(self): + from pillar.api.organizations import ip_ranges + + # A host is a valid range by itself. + db_org = self._patch({'ip_ranges': ['192.168.3.5', '3abe:0412:0000::f00d']}) + self.assertEqual([ + ip_ranges.doc('192.168.3.5/32'), + ip_ranges.doc('3abe:0412:0000::f00d/128'), + ], db_org['ip_ranges']) + + def test_patch_set_ip_ranges_invalid(self): + db_org = self._patch({'ip_ranges': ['www.example.com']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + db_org = self._patch({'ip_ranges': ['127,0,0,0/16']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + def test_patch_set_ip_ranges_invalid_ipv4(self): + # zero range should not be allowed + db_org = self._patch({'ip_ranges': ['0.0.0.0/0']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + # range too large should not be allowed + db_org = self._patch({'ip_ranges': ['192.168.3.5/64']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + # Hollywood-style IP address + db_org = self._patch({'ip_ranges': ['555.168.3.5/24']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + def test_patch_set_ip_ranges_invalid_ipv6(self): + # small range should not be allowed + db_org = self._patch({'ip_ranges': ['::/0']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + db_org = self._patch({'ip_ranges': ['123::/16']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + # range too large should not be allowed + db_org = self._patch({'ip_ranges': ['2a03:b0c0:0:1010::8fe:6ef1/192']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + # Hollywood-style IP address + db_org = self._patch({'ip_ranges': ['555:555:555:555:b0c0:0:1010:fe:6ef1/64']}, + expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + # Non-hex IP address + db_org = self._patch({'ip_ranges': ['boo:foo::1010::8fe:6ef1/64']}, expected_status=422) + self.assertNotIn('ip_ranges', db_org) + + +class IPRangeQueryTest(AbstractOrgTest): + def setUp(self, **kwargs): + super().setUp(**kwargs) + self.uid = self.create_user(24 * 'a', token='token') + + def _patch(self, org_id: bson.ObjectId, payload: dict, expected_status=204) -> dict: + self.patch(f'/api/organizations/{org_id}', + json={ + 'op': 'edit-from-web', + **payload, + }, + auth_token='token', + expected_status=expected_status) + db_org = self.om._get_org(org_id) + return db_org + + def test_happy(self): + # Set up a few organisations. A and B have overlapping IPv4 ranges, B and C on IPv6. + org_roles_a = {'org-roleA1', 'org-roleA2'} + org_a = self.app.org_manager.create_new_org('Хакеры', self.uid, 25, org_roles=org_roles_a) + self._patch(org_a['_id'], { + 'name': 'Хакеры', + 'ip_ranges': [ + '192.168.0.0/16', + '2a03:b0c0:0:1010::8fe:6ef1/120', + ]}) + + org_roles_b = {'org-roleB'} + org_b = self.app.org_manager.create_new_org('ヤクザ', self.uid, 25, org_roles=org_roles_b) + self._patch(org_b['_id'], { + 'name': 'ヤクザ', + 'ip_ranges': [ + '192.168.9.0/24', + '2a03:b0c0:beef:1010::/64', + ]}) + + org_roles_c = {'org-roleC'} + org_c = self.app.org_manager.create_new_org('ਘਰ ਵਿਚ ਨ', self.uid, 25, org_roles=org_roles_c) + self._patch(org_c['_id'], { + 'name': 'ਘਰ ਵਿਚ ਨ', + 'ip_ranges': [ + '172.168.9.0/24', + '2a03:b0c0:beef::/48', + ]}) + + self.assertEqual(org_roles_a, self.om.roles_for_ip_address('192.168.3.255')) + self.assertEqual(org_roles_a.union(org_roles_b), + self.om.roles_for_ip_address('192.168.9.16')) + self.assertEqual(org_roles_a, self.om.roles_for_ip_address('2a03:b0c0:0:1010::8fe:6e47')) + self.assertEqual(org_roles_b.union(org_roles_c), + self.om.roles_for_ip_address('2a03:b0c0:beef:1010::8fe:6e47')) + self.assertEqual(org_roles_c, self.om.roles_for_ip_address('2a03:b0c0:beef:d00d::8fe:6e47')) + + self.assertEqual(set(), self.om.roles_for_ip_address('1111:ffff::1')) + self.assertEqual(set(), self.om.roles_for_ip_address('::1'))