Backend support for organization IP ranges.
We can now store IP ranges with Organizations. The aim is to have any user logging in with a remote IP address within such a race will get the organization roles assigned to the user object stored in the Flask session. This commit just contains the MongoDB storage and querying, and not yet the updates to the user.
This commit is contained in:
parent
9bd41ed5d7
commit
c44f0489bc
@ -739,6 +739,11 @@ class PillarServer(BlinkerCompatibleEve):
|
||||
coll.create_index([('is_private', pymongo.ASCENDING)])
|
||||
coll.create_index([('category', pymongo.ASCENDING)])
|
||||
|
||||
coll = db['organizations']
|
||||
coll.create_index([('ip_ranges.start', pymongo.ASCENDING)])
|
||||
coll.create_index([('ip_ranges.end', pymongo.ASCENDING)])
|
||||
self.log.debug('Created database indices')
|
||||
|
||||
def register_api_blueprint(self, blueprint, url_prefix):
|
||||
# TODO: use Eve config variable instead of hard-coded '/api'
|
||||
self.register_blueprint(blueprint, url_prefix='/api' + url_prefix)
|
||||
|
||||
@ -125,3 +125,30 @@ class ValidateCustomFields(Validator):
|
||||
|
||||
if not value:
|
||||
self._error(field, "Value is required once the document was created")
|
||||
|
||||
def _validate_type_iprange(self, field_name: str, value: str):
|
||||
"""Ensure the field contains a valid IP address.
|
||||
|
||||
Supports both IPv6 and IPv4 ranges. Requires the IPy module.
|
||||
"""
|
||||
|
||||
from IPy import IP
|
||||
|
||||
try:
|
||||
ip = IP(value, make_net=True)
|
||||
except ValueError as ex:
|
||||
self._error(field_name, str(ex))
|
||||
return
|
||||
|
||||
if ip.prefixlen() == 0:
|
||||
self._error(field_name, 'Zero-length prefix is not allowed')
|
||||
|
||||
def _validate_type_binary(self, field_name: str, value: bytes):
|
||||
"""Add support for binary type.
|
||||
|
||||
This type was actually introduced in Cerberus 1.0, so we can drop
|
||||
support for this once Eve starts using that version (or newer).
|
||||
"""
|
||||
|
||||
if not isinstance(value, (bytes, bytearray)):
|
||||
self._error(field_name, f'wrong value type {type(value)}, expected bytes or bytearray')
|
||||
|
||||
@ -214,7 +214,21 @@ organizations_schema = {
|
||||
# in an external subscription/payment management system.
|
||||
'payment_subscription_id': {
|
||||
'type': 'string',
|
||||
},
|
||||
|
||||
'ip_ranges': {
|
||||
'type': 'list',
|
||||
'schema': {
|
||||
'type': 'dict',
|
||||
'schema': {
|
||||
# see _validate_type_{typename} in ValidateCustomFields:
|
||||
'start': {'type': 'binary', 'required': True},
|
||||
'end': {'type': 'binary', 'required': True},
|
||||
'prefix': {'type': 'integer', 'required': True},
|
||||
'human': {'type': 'iprange', 'required': True},
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
permissions_embedded_schema = {
|
||||
|
||||
@ -383,6 +383,20 @@ class OrgManager:
|
||||
org_count = org_coll.count({'unknown_members': member_email})
|
||||
return bool(org_count)
|
||||
|
||||
def roles_for_ip_address(self, remote_addr: str) -> typing.Set[str]:
|
||||
"""Find the roles given to the user via org IP range definitions."""
|
||||
|
||||
from . import ip_ranges
|
||||
|
||||
org_coll = current_app.db('organizations')
|
||||
orgs = org_coll.find(
|
||||
{'ip_ranges': ip_ranges.query(remote_addr)},
|
||||
projection={'org_roles': True},
|
||||
)
|
||||
return set(role
|
||||
for org in orgs
|
||||
for role in org.get('org_roles', []))
|
||||
|
||||
|
||||
def setup_app(app):
|
||||
from . import patch, hooks
|
||||
|
||||
75
pillar/api/organizations/ip_ranges.py
Normal file
75
pillar/api/organizations/ip_ranges.py
Normal file
@ -0,0 +1,75 @@
|
||||
"""IP range support for Organizations."""
|
||||
|
||||
from IPy import IP
|
||||
|
||||
# 128 bits all set to 1
|
||||
ONES_128 = 2 ** 128 - 1
|
||||
|
||||
|
||||
def doc(iprange: str, min_prefixlen6: int=0, min_prefixlen4: int=0) -> dict:
|
||||
"""Convert a human-readable string like '1.2.3.4/24' to a Mongo document.
|
||||
|
||||
This converts the address to IPv6 and computes the start/end addresses
|
||||
of the range. The address, its prefix size, and start and end address,
|
||||
are returned as a dict.
|
||||
|
||||
Addresses are stored as big-endian binary data because MongoDB doesn't
|
||||
support 128 bits integers.
|
||||
|
||||
:param iprange: the IP address and mask size, can be IPv6 or IPv4.
|
||||
:param min_prefixlen6: if given, causes a ValuError when the mask size
|
||||
is too low. Note that the mask size is always
|
||||
evaluated only for IPv6 addresses.
|
||||
:param min_prefixlen4: if given, causes a ValuError when the mask size
|
||||
is too low. Note that the mask size is always
|
||||
evaluated only for IPv4 addresses.
|
||||
:returns: a dict like: {
|
||||
'start': b'xxxxx' with the lowest IP address in the range.
|
||||
'end': b'yyyyy' with the highest IP address in the range.
|
||||
'human': 'aaaa:bbbb::cc00/120' with the human-readable representation.
|
||||
'prefix': 120, the prefix length of the netmask in bits.
|
||||
}
|
||||
"""
|
||||
|
||||
ip = IP(iprange, make_net=True)
|
||||
prefixlen = ip.prefixlen()
|
||||
if ip.version() == 4:
|
||||
if prefixlen < min_prefixlen4:
|
||||
raise ValueError(f'Prefix length {prefixlen} smaller than allowed {min_prefixlen4}')
|
||||
ip = ip.v46map()
|
||||
else:
|
||||
if prefixlen < min_prefixlen6:
|
||||
raise ValueError(f'Prefix length {prefixlen} smaller than allowed {min_prefixlen6}')
|
||||
|
||||
addr = ip.int()
|
||||
|
||||
# Set all address bits to 1 where the mask is 0 to obtain the largest address.
|
||||
end = addr | (ONES_128 % ip.netmask().int())
|
||||
|
||||
# This ensures that even a single host is represented as /128 in the human-readable form.
|
||||
ip.NoPrefixForSingleIp = False
|
||||
|
||||
return {
|
||||
'start': addr.to_bytes(16, 'big'),
|
||||
'end': end.to_bytes(16, 'big'),
|
||||
'human': ip.strCompressed(),
|
||||
'prefix': ip.prefixlen(),
|
||||
}
|
||||
|
||||
|
||||
def query(address: str) -> dict:
|
||||
"""Return a dict usable for querying all organizations whose IP range matches the given one.
|
||||
|
||||
:returns: a dict like:
|
||||
{$elemMatch: {'start': {$lte: b'xxxxx'}, 'end': {$gte: b'xxxxx'}}}
|
||||
"""
|
||||
|
||||
ip = IP(address)
|
||||
if ip.version() == 4:
|
||||
ip = ip.v46map()
|
||||
for_mongo = ip.ip.to_bytes(16, 'big')
|
||||
|
||||
return {'$elemMatch': {
|
||||
'start': {'$lte': for_mongo},
|
||||
'end': {'$gte': for_mongo},
|
||||
}}
|
||||
@ -135,9 +135,20 @@ class OrganizationPatchHandler(patch_handler.AbstractPatchHandler):
|
||||
|
||||
@authorization.require_login()
|
||||
def patch_edit_from_web(self, org_id: bson.ObjectId, patch: dict):
|
||||
"""Updates Organization fields from the web."""
|
||||
"""Updates Organization fields from the web.
|
||||
|
||||
The PATCH command supports the following payload. The 'name' field must
|
||||
be set, all other fields are optional. When an optional field is
|
||||
ommitted it will be handled as an instruction to clear that field.
|
||||
{'name': str,
|
||||
'description': str,
|
||||
'website': str,
|
||||
'location': str,
|
||||
'ip_ranges': list of human-readable IP ranges}
|
||||
"""
|
||||
|
||||
from pymongo.results import UpdateResult
|
||||
from . import ip_ranges
|
||||
|
||||
self._assert_is_admin(org_id)
|
||||
user = current_user()
|
||||
@ -150,6 +161,21 @@ class OrganizationPatchHandler(patch_handler.AbstractPatchHandler):
|
||||
'website': patch.get('website', '').strip(),
|
||||
'location': patch.get('location', '').strip(),
|
||||
}
|
||||
unset = {}
|
||||
|
||||
# Special transformation for IP ranges
|
||||
iprs = patch.get('ip_ranges')
|
||||
if iprs:
|
||||
ipr_docs = []
|
||||
for r in iprs:
|
||||
try:
|
||||
doc = ip_ranges.doc(r, min_prefixlen6=48, min_prefixlen4=8)
|
||||
except ValueError as ex:
|
||||
raise wz_exceptions.UnprocessableEntity(f'Invalid IP range {r!r}: {ex}')
|
||||
ipr_docs.append(doc)
|
||||
update['ip_ranges'] = ipr_docs
|
||||
else:
|
||||
unset['ip_ranges'] = True
|
||||
|
||||
refresh_user_roles = False
|
||||
if user.has_cap('admin'):
|
||||
@ -177,11 +203,13 @@ class OrganizationPatchHandler(patch_handler.AbstractPatchHandler):
|
||||
resp.status_code = 422
|
||||
return resp
|
||||
|
||||
# Figure out what to set and what to unset
|
||||
for_mongo = {'$set': update}
|
||||
if unset:
|
||||
for_mongo['$unset'] = unset
|
||||
|
||||
organizations_coll = current_app.db('organizations')
|
||||
result: UpdateResult = organizations_coll.update_one(
|
||||
{'_id': org_id},
|
||||
{'$set': update}
|
||||
)
|
||||
result: UpdateResult = organizations_coll.update_one({'_id': org_id}, for_mongo)
|
||||
|
||||
if result.matched_count != 1:
|
||||
self.log.warning('User %s edits Organization %s but update matched %i items',
|
||||
|
||||
@ -20,6 +20,7 @@ Flask-WTF==0.12
|
||||
gcloud==0.12.0
|
||||
google-apitools==0.4.11
|
||||
httplib2==0.9.2
|
||||
IPy==0.83
|
||||
MarkupSafe==0.23
|
||||
ndg-httpsclient==0.4.0
|
||||
Pillow==4.1.1
|
||||
|
||||
@ -7,6 +7,14 @@ from pillar.tests import AbstractPillarTest
|
||||
from pillar.api.utils import remove_private_keys
|
||||
|
||||
|
||||
class AbstractOrgTest(AbstractPillarTest):
|
||||
def setUp(self, **kwargs):
|
||||
super().setUp(**kwargs)
|
||||
|
||||
self.enter_app_context()
|
||||
self.om = self.app.org_manager
|
||||
|
||||
|
||||
class OrganizationCruTest(AbstractPillarTest):
|
||||
"""Test creating and updating organizations."""
|
||||
|
||||
@ -470,15 +478,12 @@ class OrganizationPatchTest(AbstractPillarTest):
|
||||
self.assertEqual(admin_uid, db_org['admin_uid'])
|
||||
|
||||
|
||||
class OrganizationResourceEveTest(AbstractPillarTest):
|
||||
class OrganizationResourceEveTest(AbstractOrgTest):
|
||||
"""Test GET/POST/PUT access to Organization resource"""
|
||||
|
||||
def setUp(self, **kwargs):
|
||||
super().setUp(**kwargs)
|
||||
|
||||
self.enter_app_context()
|
||||
self.om = self.app.org_manager
|
||||
|
||||
# Pillar admin
|
||||
self.create_user(24 * '1', roles={'admin'}, token='uberadmin')
|
||||
|
||||
@ -764,3 +769,179 @@ class UserCreationTest(AbstractPillarTest):
|
||||
db_org = self._from_db()
|
||||
self.assertEqual([], db_org['unknown_members'])
|
||||
self.assertEqual([my_id], db_org['members'])
|
||||
|
||||
|
||||
class IPRangeTest(AbstractOrgTest):
|
||||
|
||||
def setUp(self, **kwargs):
|
||||
super().setUp(**kwargs)
|
||||
self.uid = self.create_user(24 * 'a', token='token')
|
||||
self.org_roles = {'org-subscriber', 'org-phabricator'}
|
||||
self.org = self.app.org_manager.create_new_org('Хакеры', self.uid, 25,
|
||||
org_roles=self.org_roles)
|
||||
self.org_id = self.org['_id']
|
||||
|
||||
def _patch(self, payload: dict, expected_status=204) -> dict:
|
||||
self.patch(f'/api/organizations/{self.org_id}',
|
||||
json={
|
||||
'op': 'edit-from-web',
|
||||
'name': self.org['name'],
|
||||
**payload,
|
||||
},
|
||||
auth_token='token',
|
||||
expected_status=expected_status)
|
||||
db_org = self.om._get_org(self.org_id)
|
||||
return db_org
|
||||
|
||||
def test_ipranges_doc(self):
|
||||
from pillar.api.organizations import ip_ranges
|
||||
|
||||
doc = ip_ranges.doc('2a03:b0c0:0:1010::8fe:6ef1/120')
|
||||
self.assertEqual(0x2a03b0c0000010100000000008fe6e00.to_bytes(16, 'big'), doc['start'])
|
||||
self.assertEqual(0x2a03b0c0000010100000000008fe6eff.to_bytes(16, 'big'), doc['end'])
|
||||
self.assertEqual('2a03:b0c0:0:1010::8fe:6e00/120', doc['human'])
|
||||
self.assertEqual(120, doc['prefix'])
|
||||
self.assertEqual({'prefix', 'human', 'start', 'end'}, set(doc.keys()))
|
||||
|
||||
def test_ipranges_query(self):
|
||||
from pillar.api.organizations import ip_ranges
|
||||
|
||||
doc = ip_ranges.query('2a03:b0c0:0:1010::8fe:6ef1')
|
||||
addr = 0x2a03b0c0000010100000000008fe6ef1.to_bytes(16, 'big')
|
||||
|
||||
self.assertEqual(addr, doc['$elemMatch']['start']['$lte'])
|
||||
self.assertEqual(addr, doc['$elemMatch']['end']['$gte'])
|
||||
|
||||
def test_patch_set_ip_ranges_happy(self):
|
||||
from pillar.api.organizations import ip_ranges
|
||||
|
||||
# IP ranges should be saved as integers for fast matching.
|
||||
db_org = self._patch({'ip_ranges': [
|
||||
'192.168.3.0/24',
|
||||
'192.168.3.1/32',
|
||||
'2a03:b0c0:0:1010::8fe:6ef1/120',
|
||||
]})
|
||||
|
||||
self.assertEqual([
|
||||
ip_ranges.doc('192.168.3.0/24'),
|
||||
ip_ranges.doc('192.168.3.1/32'),
|
||||
ip_ranges.doc('2a03:b0c0:0:1010::8fe:6ef1/120'),
|
||||
], db_org['ip_ranges'])
|
||||
|
||||
def test_patch_unset_ip_ranges_happy(self):
|
||||
"""Setting to empty list should just delete the entire key."""
|
||||
ipranges = [
|
||||
'192.168.3.0/24',
|
||||
'48.44.12.35/32',
|
||||
'2a01:7c8:aab9:3b::0/64',
|
||||
]
|
||||
self._patch({'ip_ranges': ipranges})
|
||||
db_org = self._patch({'ip_ranges': []})
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
def test_single_host(self):
|
||||
from pillar.api.organizations import ip_ranges
|
||||
|
||||
# A host is a valid range by itself.
|
||||
db_org = self._patch({'ip_ranges': ['192.168.3.5', '3abe:0412:0000::f00d']})
|
||||
self.assertEqual([
|
||||
ip_ranges.doc('192.168.3.5/32'),
|
||||
ip_ranges.doc('3abe:0412:0000::f00d/128'),
|
||||
], db_org['ip_ranges'])
|
||||
|
||||
def test_patch_set_ip_ranges_invalid(self):
|
||||
db_org = self._patch({'ip_ranges': ['www.example.com']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
db_org = self._patch({'ip_ranges': ['127,0,0,0/16']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
def test_patch_set_ip_ranges_invalid_ipv4(self):
|
||||
# zero range should not be allowed
|
||||
db_org = self._patch({'ip_ranges': ['0.0.0.0/0']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
# range too large should not be allowed
|
||||
db_org = self._patch({'ip_ranges': ['192.168.3.5/64']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
# Hollywood-style IP address
|
||||
db_org = self._patch({'ip_ranges': ['555.168.3.5/24']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
def test_patch_set_ip_ranges_invalid_ipv6(self):
|
||||
# small range should not be allowed
|
||||
db_org = self._patch({'ip_ranges': ['::/0']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
db_org = self._patch({'ip_ranges': ['123::/16']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
# range too large should not be allowed
|
||||
db_org = self._patch({'ip_ranges': ['2a03:b0c0:0:1010::8fe:6ef1/192']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
# Hollywood-style IP address
|
||||
db_org = self._patch({'ip_ranges': ['555:555:555:555:b0c0:0:1010:fe:6ef1/64']},
|
||||
expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
# Non-hex IP address
|
||||
db_org = self._patch({'ip_ranges': ['boo:foo::1010::8fe:6ef1/64']}, expected_status=422)
|
||||
self.assertNotIn('ip_ranges', db_org)
|
||||
|
||||
|
||||
class IPRangeQueryTest(AbstractOrgTest):
|
||||
def setUp(self, **kwargs):
|
||||
super().setUp(**kwargs)
|
||||
self.uid = self.create_user(24 * 'a', token='token')
|
||||
|
||||
def _patch(self, org_id: bson.ObjectId, payload: dict, expected_status=204) -> dict:
|
||||
self.patch(f'/api/organizations/{org_id}',
|
||||
json={
|
||||
'op': 'edit-from-web',
|
||||
**payload,
|
||||
},
|
||||
auth_token='token',
|
||||
expected_status=expected_status)
|
||||
db_org = self.om._get_org(org_id)
|
||||
return db_org
|
||||
|
||||
def test_happy(self):
|
||||
# Set up a few organisations. A and B have overlapping IPv4 ranges, B and C on IPv6.
|
||||
org_roles_a = {'org-roleA1', 'org-roleA2'}
|
||||
org_a = self.app.org_manager.create_new_org('Хакеры', self.uid, 25, org_roles=org_roles_a)
|
||||
self._patch(org_a['_id'], {
|
||||
'name': 'Хакеры',
|
||||
'ip_ranges': [
|
||||
'192.168.0.0/16',
|
||||
'2a03:b0c0:0:1010::8fe:6ef1/120',
|
||||
]})
|
||||
|
||||
org_roles_b = {'org-roleB'}
|
||||
org_b = self.app.org_manager.create_new_org('ヤクザ', self.uid, 25, org_roles=org_roles_b)
|
||||
self._patch(org_b['_id'], {
|
||||
'name': 'ヤクザ',
|
||||
'ip_ranges': [
|
||||
'192.168.9.0/24',
|
||||
'2a03:b0c0:beef:1010::/64',
|
||||
]})
|
||||
|
||||
org_roles_c = {'org-roleC'}
|
||||
org_c = self.app.org_manager.create_new_org('ਘਰ ਵਿਚ ਨ', self.uid, 25, org_roles=org_roles_c)
|
||||
self._patch(org_c['_id'], {
|
||||
'name': 'ਘਰ ਵਿਚ ਨ',
|
||||
'ip_ranges': [
|
||||
'172.168.9.0/24',
|
||||
'2a03:b0c0:beef::/48',
|
||||
]})
|
||||
|
||||
self.assertEqual(org_roles_a, self.om.roles_for_ip_address('192.168.3.255'))
|
||||
self.assertEqual(org_roles_a.union(org_roles_b),
|
||||
self.om.roles_for_ip_address('192.168.9.16'))
|
||||
self.assertEqual(org_roles_a, self.om.roles_for_ip_address('2a03:b0c0:0:1010::8fe:6e47'))
|
||||
self.assertEqual(org_roles_b.union(org_roles_c),
|
||||
self.om.roles_for_ip_address('2a03:b0c0:beef:1010::8fe:6e47'))
|
||||
self.assertEqual(org_roles_c, self.om.roles_for_ip_address('2a03:b0c0:beef:d00d::8fe:6e47'))
|
||||
|
||||
self.assertEqual(set(), self.om.roles_for_ip_address('1111:ffff::1'))
|
||||
self.assertEqual(set(), self.om.roles_for_ip_address('::1'))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user