Python 3.6 compatibility: bytes vs strings stuff

These changes mostly revolve around the change in ObjectId constructor
when running on Python 3.6. Where on 2.7 the constructor would accept
12- and 24-byte strings, now only 12-byte bytes and 24-character strings
are accepted. Good thing, but required some changes in our code.

Other changes include hashing of strings, which isn't supported, so they
are converted to bytes first, and sometimes converted back afterwards.

pillar/api/file_storage/__init__.py

@@ -296,7 +296,7 @@ def delete_file(file_item):
     process_file_delete(file_item)
 
 
-def generate_link(backend, file_path, project_id=None, is_public=False):
+def generate_link(backend, file_path: str, project_id=None, is_public=False):
     """Hook to check the backend of a file resource, to build an appropriate link
     that can be used by the client to retrieve the actual file.
     """
@@ -328,7 +328,7 @@ def generate_link(backend, file_path, project_id=None, is_public=False):
     if backend == 'cdnsun':
         return hash_file_path(file_path, None)
     if backend == 'unittest':
-        return 'https://unit.test/%s' % md5(file_path).hexdigest()
+        return 'https://unit.test/%s' % md5(file_path.encode()).hexdigest()
 
     log.warning('generate_link(): Unknown backend %r, returning empty string '
                 'as new link.',

pillar/api/local_auth.py

@@ -1,6 +1,7 @@
 import base64
 import hashlib
 import logging
+import typing
 
 import bcrypt
 import datetime
@@ -67,12 +68,12 @@ def make_token():
     return jsonify(token=token['token'])
 
 
-def generate_and_store_token(user_id, days=15, prefix=''):
+def generate_and_store_token(user_id, days=15, prefix=b''):
     """Generates token based on random bits.
 
     :param user_id: ObjectId of the owning user.
     :param days: token will expire in this many days.
-    :param prefix: the token will be prefixed by this string, for easy identification.
+    :param prefix: the token will be prefixed by these bytes, for easy identification.
     :return: the token document.
     """
 
@@ -80,17 +81,22 @@ def generate_and_store_token(user_id, days=15, prefix=''):
 
     # Use 'xy' as altargs to prevent + and / characters from appearing.
     # We never have to b64decode the string anyway.
-    token = prefix + base64.b64encode(random_bits, altchars='xy').strip('=')
+    token = prefix + base64.b64encode(random_bits, altchars=b'xy').strip(b'=')
 
     token_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(days=days)
-    return store_token(user_id, token, token_expiry)
+    return store_token(user_id, token.decode('ascii'), token_expiry)
 
 
-def hash_password(password, salt):
+def hash_password(password: str, salt: typing.Union[str, bytes]) -> str:
+    password = password.encode()
+
     if isinstance(salt, str):
         salt = salt.encode('utf-8')
-    encoded_password = base64.b64encode(hashlib.sha256(password).digest())
-    return bcrypt.hashpw(encoded_password, salt)
+
+    hash = hashlib.sha256(password).digest()
+    encoded_password = base64.b64encode(hash)
+    hashed_password = bcrypt.hashpw(encoded_password, salt)
+    return hashed_password.decode('ascii')
 
 
 def setup_app(app, url_prefix):

pillar/api/service.py

@@ -221,7 +221,7 @@ def create_service_account(email, roles, service, update_existing=None):
     user.update(result)
 
     # Create an authentication token that won't expire for a long time.
-    token = local_auth.generate_and_store_token(user['_id'], days=36500, prefix='SRV')
+    token = local_auth.generate_and_store_token(user['_id'], days=36500, prefix=b'SRV')
 
     return user, token
 

pillar/api/utils/__init__.py

@@ -136,7 +136,7 @@ def str2id(document_id):
 
     try:
         return bson.ObjectId(document_id)
-    except bson.objectid.InvalidId:
+    except (bson.objectid.InvalidId, TypeError):
         log.debug('str2id(%r): Invalid Object ID', document_id)
         raise wz_exceptions.BadRequest('Invalid object ID %r' % document_id)
 

pillar/api/utils/authentication.py

@@ -118,7 +118,7 @@ def find_token(token, is_subclient_token=False, **extra_filters):
     return db_token
 
 
-def store_token(user_id, token, token_expiry, oauth_subclient_id=False):
+def store_token(user_id, token: str, token_expiry, oauth_subclient_id=False):
     """Stores an authentication token.
 
     :returns: the token document from MongoDB