Added pillar.api.utils.utcnow() which returns a datetime for 'now'

This replaces pillar.web.utils.datetime_now() and can be used in a wider
setting (since we don't import web stuff in the api, but we do vice versa).

pillar/api/blender_cloud/home_project.py

@@ -1,12 +1,11 @@
 import copy
 import logging
 
-import datetime
+from bson import ObjectId
-from bson import ObjectId, tz_util
 from eve.methods.get import get
 from flask import Blueprint, current_app, request
 from pillar.api import utils
-from pillar.api.utils import authentication, authorization
+from pillar.api.utils import authentication, authorization, utcnow
 from werkzeug import exceptions as wz_exceptions
 
 from pillar.api.projects import utils as proj_utils
@@ -282,7 +281,7 @@ def is_home_project(project_id, user_id):
 def mark_node_updated(node_id):
     """Uses pymongo to set the node's _updated to "now"."""
 
-    now = datetime.datetime.now(tz=tz_util.utc)
+    now = utcnow()
     nodes_coll = current_app.data.driver.db['nodes']
 
     return nodes_coll.update_one({'_id': node_id},

pillar/api/blender_id.py

@@ -15,7 +15,7 @@ from requests.adapters import HTTPAdapter
 
 from pillar import current_app
 from pillar.api import service
-from pillar.api.utils import authentication
+from pillar.api.utils import authentication, utcnow
 from pillar.api.utils.authentication import find_user_in_db, upsert_user
 
 blender_id = Blueprint('blender_id', __name__)
@@ -171,7 +171,7 @@ def _compute_token_expiry(token_expires_string):
 
     blid_expiry = parser.parse(token_expires_string)
     blid_expiry = blid_expiry.astimezone(tz_util.utc)
-    our_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(hours=1)
+    our_expiry = utcnow() + datetime.timedelta(hours=1)
 
     return min(blid_expiry, our_expiry)
 

pillar/api/encoding.py

@@ -3,7 +3,7 @@ import json
 import logging
 import os
 
-from bson import ObjectId, tz_util
+from bson import ObjectId
 from flask import Blueprint
 from flask import abort
 from flask import current_app
@@ -161,7 +161,7 @@ def zencoder_notifications():
     file_doc['status'] = 'complete'
 
     # Force an update of the links on the next load of the file.
-    file_doc['link_expires'] = datetime.datetime.now(tz=tz_util.utc) - datetime.timedelta(days=1)
+    file_doc['link_expires'] = utils.utcnow() - datetime.timedelta(days=1)
 
     r, _, _, status = current_app.put_internal('files', file_doc, _id=file_id)
     if status != 200:

pillar/api/file_storage/__init__.py

@@ -9,7 +9,6 @@ import typing
 import uuid
 from hashlib import md5
 
-import bson.tz_util
 import eve.utils
 import pymongo
 import werkzeug.exceptions as wz_exceptions
@@ -27,7 +26,7 @@ from pillar.api import utils
 from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket, \
     GoogleCloudStorageBlob
 from pillar.api.utils import remove_private_keys
-from pillar.api.utils.authorization import require_login, user_has_role, \
+from pillar.api.utils.authorization import require_login, \
     user_matches_roles
 from pillar.api.utils.cdn import hash_file_path
 from pillar.api.utils.encoding import Encoder
@@ -419,7 +418,7 @@ def ensure_valid_link(response):
     # log.debug('Inspecting link for file %s', response['_id'])
 
     # Check link expiry.
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     if 'link_expires' in response:
         link_expires = response['link_expires']
         if now < link_expires:
@@ -502,7 +501,7 @@ def on_pre_get_files(_, lookup):
         return
 
     # Only fetch it if the date got expired.
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     lookup_expired = lookup.copy()
     lookup_expired['link_expires'] = {'$lte': now}
 
@@ -527,7 +526,7 @@ def refresh_links_for_project(project_uuid, chunk_size, expiry_seconds):
     # Retrieve expired links.
     files_collection = current_app.data.driver.db['files']
 
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     expire_before = now + datetime.timedelta(seconds=expiry_seconds)
     log.info('Limiting to links that expire before %s', expire_before)
 
@@ -556,7 +555,7 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds):
     files_collection = current_app.data.driver.db['files']
     proj_coll = current_app.data.driver.db['projects']
 
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    now = utils.utcnow()
     expire_before = now + datetime.timedelta(seconds=expiry_seconds)
     my_log.info('Limiting to links that expire before %s', expire_before)
 

pillar/api/file_storage/moving.py

@@ -1,16 +1,15 @@
 """Code for moving files between backends."""
 
-import datetime
 import logging
 import os
 import tempfile
 
-import bson.tz_util
 import requests
 import requests.exceptions
 from bson import ObjectId
 from flask import current_app
 
+from pillar.api import utils
 from . import stream_to_gcs, generate_all_links, ensure_valid_link
 
 __all__ = ['PrerequisiteNotMetError', 'change_file_storage_backend']
@@ -74,8 +73,7 @@ def change_file_storage_backend(file_id, dest_backend):
     # Generate new links for the file & all variations. This also saves
     # the new backend we set here.
     f['backend'] = dest_backend
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    generate_all_links(f, utils.utcnow())
-    generate_all_links(f, now)
 
 
 def copy_file_to_backend(file_id, project_id, file_or_var, src_backend, dest_backend):
@@ -190,4 +188,4 @@ def gcs_move_to_bucket(file_id, dest_project_id, skip_gcs=False):
 
     # Regenerate the links for this file
     f['project'] = dest_project_id
-    generate_all_links(f, now=datetime.datetime.now(tz=bson.tz_util.utc))
+    generate_all_links(f, now=utils.utcnow())

pillar/api/local_auth.py

@@ -1,15 +1,16 @@
 import base64
+import datetime
 import hashlib
 import logging
 import typing
 
 import bcrypt
-import datetime
+
-from bson import tz_util
 from flask import abort, Blueprint, current_app, jsonify, request
 from pillar.api.utils.authentication import create_new_user_document
 from pillar.api.utils.authentication import make_unique_username
 from pillar.api.utils.authentication import store_token
+from pillar.api.utils import utcnow
 
 blueprint = Blueprint('authentication', __name__)
 log = logging.getLogger(__name__)
@@ -96,7 +97,7 @@ def generate_and_store_token(user_id, days=15, prefix=b'') -> dict:
     token_bytes = prefix + base64.b64encode(random_bits, altchars=b'xy').strip(b'=')
     token = token_bytes.decode('ascii')
 
-    token_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(days=days)
+    token_expiry = utcnow() + datetime.timedelta(days=days)
     token_data = store_token(user_id, token, token_expiry)
 
     # Include the token in the returned document so that it can be stored client-side,

pillar/api/organizations/__init__.py

@@ -4,7 +4,6 @@ Assumes role names that are given to users by organization membership
 start with the string "org-".
 """
 
-import datetime
 import logging
 import typing
 
@@ -14,7 +13,7 @@ import flask
 import werkzeug.exceptions as wz_exceptions
 
 from pillar import attrs_extra, current_app
-from pillar.api.utils import remove_private_keys
+from pillar.api.utils import remove_private_keys, utcnow
 
 
 class OrganizationError(Exception):
@@ -281,10 +280,9 @@ class OrgManager:
         # Join all organization-given roles and roles from the tokens collection.
         org_roles = aggr_roles(org_coll, {'members': user_id})
         self._log.debug('Organization-given roles for user %s: %s', user_id, org_roles)
-        now = datetime.datetime.now(bson.tz_util.utc)
         token_roles = aggr_roles(tokens_coll, {
             'user': user_id,
-            'expire_time': {"$gt": now},
+            'expire_time': {"$gt": utcnow()},
         })
         self._log.debug('Token-given roles for user %s: %s', user_id, token_roles)
         org_roles.update(token_roles)

pillar/api/projects/patch.py

@@ -1,16 +1,14 @@
 """Project patching support."""
 
-import datetime
 import logging
 
-import bson.tz_util
 import flask
 from flask import Blueprint, request
 import werkzeug.exceptions as wz_exceptions
 
 from pillar import current_app
 from pillar.auth import current_user
-from pillar.api.utils import random_etag, str2id
+from pillar.api.utils import random_etag, str2id, utcnow
 from pillar.api.utils import authorization
 
 log = logging.getLogger(__name__)
@@ -60,7 +58,6 @@ def patch_project(project_id: str):
     # PATCHing collections, so direct MongoDB modification is used to set
     # _deleted=False and provide new _etag and _updated values.
     new_etag = random_etag()
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
 
     log.debug('undeleting files before undeleting project %s', pid)
     files_coll = current_app.db('files')
@@ -68,7 +65,7 @@ def patch_project(project_id: str):
         {'project': pid},
         {'$set': {'_deleted': False,
                   '_etag': new_etag,
-                  '_updated': now}})
+                  '_updated': utcnow()}})
     log.info('undeleted %d of %d file documents of project %s',
              update_result.modified_count, update_result.matched_count, pid)
 

pillar/api/utils/__init__.py

@@ -10,6 +10,7 @@ import typing
 import urllib.request, urllib.parse, urllib.error
 
 import bson.objectid
+import bson.tz_util
 from eve import RFC1123_DATE_FORMAT
 from flask import current_app
 from werkzeug import exceptions as wz_exceptions
@@ -200,3 +201,7 @@ def random_etag() -> str:
 
     randbytes = random.getrandbits(256).to_bytes(32, 'big')
     return base64.b64encode(randbytes)[:-1].decode()
+
+
+def utcnow() -> datetime.datetime:
+    return datetime.datetime.now(tz=bson.tz_util.utc)

pillar/api/utils/authentication.py

@@ -13,12 +13,11 @@ import logging
 import typing
 
 import bson
-from bson import tz_util
 from flask import g, current_app
 from flask import request
 from werkzeug import exceptions as wz_exceptions
 
-from pillar.api.utils import remove_private_keys
+from pillar.api.utils import remove_private_keys, utcnow
 
 log = logging.getLogger(__name__)
 
@@ -209,7 +208,7 @@ def find_token(token, is_subclient_token=False, **extra_filters):
     # TODO: remove matching on unhashed tokens once all tokens have been hashed.
     lookup = {'$or': [{'token': token}, {'token_hashed': token_hashed}],
               'is_subclient_token': True if is_subclient_token else {'$in': [False, None]},
-              'expire_time': {"$gt": datetime.datetime.now(tz=tz_util.utc)}}
+              'expire_time': {"$gt": utcnow()}}
     lookup.update(extra_filters)
 
     db_token = tokens_coll.find_one(lookup)
@@ -333,9 +332,7 @@ def _delete_expired_tokens():
 
     token_coll = current_app.data.driver.db['tokens']
 
-    now = datetime.datetime.now(tz_util.utc)
+    expiry_date = utcnow() - datetime.timedelta(days=7)
-    expiry_date = now - datetime.timedelta(days=7)
-
     result = token_coll.delete_many({'expire_time': {"$lt": expiry_date}})
     # log.debug('Deleted %i expired authentication tokens', result.deleted_count)
 

pillar/cli/maintenance.py

@@ -391,13 +391,11 @@ def expire_all_project_links(project_uuid):
     """
 
     import datetime
-    import bson.tz_util
+    from pillar.api.utils import utcnow
 
     files_collection = current_app.data.driver.db['files']
 
-    now = datetime.datetime.now(tz=bson.tz_util.utc)
+    expires = utcnow() - datetime.timedelta(days=1)
-    expires = now - datetime.timedelta(days=1)
-
     result = files_collection.update_many(
         {'project': ObjectId(project_uuid)},
         {'$set': {'link_expires': expires}}

pillar/tests/__init__.py

@@ -327,8 +327,9 @@ class AbstractPillarTest(TestMinimal):
         return user
 
     def create_valid_auth_token(self, user_id, token='token'):
-        now = datetime.datetime.now(tz_util.utc)
+        from pillar.api.utils import utcnow
-        future = now + datetime.timedelta(days=1)
+
+        future = utcnow() + datetime.timedelta(days=1)
 
         with self.app.test_request_context():
             from pillar.api.utils import authentication as auth

pillar/web/nodes/custom/comments.py

@@ -10,10 +10,11 @@ from pillarsdk import Node
 from pillarsdk import Project
 import werkzeug.exceptions as wz_exceptions
 
+from pillar.api.utils import utcnow
 from pillar.web import subquery
 from pillar.web.nodes.routes import blueprint
 from pillar.web.utils import gravatar
-from pillar.web.utils import pretty_date, datetime_now
+from pillar.web.utils import pretty_date
 from pillar.web.utils import system_util
 
 log = logging.getLogger(__name__)
@@ -111,7 +112,7 @@ def format_comment(comment, is_reply=False, is_team=False, replies=None):
 
     return dict(_id=comment._id,
                 gravatar=gravatar(comment.user.email, size=32),
-                time_published=pretty_date(comment._created or datetime_now(), detail=True),
+                time_published=pretty_date(comment._created or utcnow(), detail=True),
                 rating=comment.properties.rating_positive - comment.properties.rating_negative,
                 author=comment.user.full_name,
                 author_username=comment.user.username,

pillar/web/projects/routes.py

@@ -21,6 +21,7 @@ from flask_login import login_required, current_user
 import werkzeug.exceptions as wz_exceptions
 
 from pillar import current_app
+from pillar.api.utils import utcnow
 from pillar.web import system_util
 from pillar.web import utils
 from pillar.web.utils.jstree import jstree_get_children
@@ -82,7 +83,7 @@ def index():
 
     show_deleted_projects = request.args.get('deleted') is not None
     if show_deleted_projects:
-        timeframe = utils.datetime_now() - datetime.timedelta(days=31)
+        timeframe = utcnow() - datetime.timedelta(days=31)
         projects_deleted = Project.all({
             'where': {'user': current_user.objectid,
                       'category': {'$ne': 'home'},

pillar/web/utils/__init__.py

@@ -114,12 +114,6 @@ def gravatar(email: str, size=64):
     return api_gravatar(email, size)
 
 
-def datetime_now():
-    """Returns a datetime.datetime that represents 'now' in UTC."""
-
-    return datetime.datetime.now(tz=pillarsdk.utils.utc)
-
-
 def pretty_date(time, detail=False, now=None):
     """Get a datetime object or a int() Epoch timestamp and return a
     pretty string like 'an hour ago', 'Yesterday', '3 months ago',