diff --git a/pillar/api/blender_cloud/home_project.py b/pillar/api/blender_cloud/home_project.py index 94fb8a5d..90d005ce 100644 --- a/pillar/api/blender_cloud/home_project.py +++ b/pillar/api/blender_cloud/home_project.py @@ -1,12 +1,11 @@ import copy import logging -import datetime -from bson import ObjectId, tz_util +from bson import ObjectId from eve.methods.get import get from flask import Blueprint, current_app, request from pillar.api import utils -from pillar.api.utils import authentication, authorization +from pillar.api.utils import authentication, authorization, utcnow from werkzeug import exceptions as wz_exceptions from pillar.api.projects import utils as proj_utils @@ -282,7 +281,7 @@ def is_home_project(project_id, user_id): def mark_node_updated(node_id): """Uses pymongo to set the node's _updated to "now".""" - now = datetime.datetime.now(tz=tz_util.utc) + now = utcnow() nodes_coll = current_app.data.driver.db['nodes'] return nodes_coll.update_one({'_id': node_id}, diff --git a/pillar/api/blender_id.py b/pillar/api/blender_id.py index c8d66942..9df1387a 100644 --- a/pillar/api/blender_id.py +++ b/pillar/api/blender_id.py @@ -15,7 +15,7 @@ from requests.adapters import HTTPAdapter from pillar import current_app from pillar.api import service -from pillar.api.utils import authentication +from pillar.api.utils import authentication, utcnow from pillar.api.utils.authentication import find_user_in_db, upsert_user blender_id = Blueprint('blender_id', __name__) @@ -171,7 +171,7 @@ def _compute_token_expiry(token_expires_string): blid_expiry = parser.parse(token_expires_string) blid_expiry = blid_expiry.astimezone(tz_util.utc) - our_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(hours=1) + our_expiry = utcnow() + datetime.timedelta(hours=1) return min(blid_expiry, our_expiry) diff --git a/pillar/api/encoding.py b/pillar/api/encoding.py index 24e67b8d..72d82173 100644 --- a/pillar/api/encoding.py +++ b/pillar/api/encoding.py @@ -3,7 +3,7 @@ import json import logging import os -from bson import ObjectId, tz_util +from bson import ObjectId from flask import Blueprint from flask import abort from flask import current_app @@ -161,7 +161,7 @@ def zencoder_notifications(): file_doc['status'] = 'complete' # Force an update of the links on the next load of the file. - file_doc['link_expires'] = datetime.datetime.now(tz=tz_util.utc) - datetime.timedelta(days=1) + file_doc['link_expires'] = utils.utcnow() - datetime.timedelta(days=1) r, _, _, status = current_app.put_internal('files', file_doc, _id=file_id) if status != 200: diff --git a/pillar/api/file_storage/__init__.py b/pillar/api/file_storage/__init__.py index c405cb52..e14e4711 100644 --- a/pillar/api/file_storage/__init__.py +++ b/pillar/api/file_storage/__init__.py @@ -9,7 +9,6 @@ import typing import uuid from hashlib import md5 -import bson.tz_util import eve.utils import pymongo import werkzeug.exceptions as wz_exceptions @@ -27,7 +26,7 @@ from pillar.api import utils from pillar.api.file_storage_backends.gcs import GoogleCloudStorageBucket, \ GoogleCloudStorageBlob from pillar.api.utils import remove_private_keys -from pillar.api.utils.authorization import require_login, user_has_role, \ +from pillar.api.utils.authorization import require_login, \ user_matches_roles from pillar.api.utils.cdn import hash_file_path from pillar.api.utils.encoding import Encoder @@ -419,7 +418,7 @@ def ensure_valid_link(response): # log.debug('Inspecting link for file %s', response['_id']) # Check link expiry. - now = datetime.datetime.now(tz=bson.tz_util.utc) + now = utils.utcnow() if 'link_expires' in response: link_expires = response['link_expires'] if now < link_expires: @@ -502,7 +501,7 @@ def on_pre_get_files(_, lookup): return # Only fetch it if the date got expired. - now = datetime.datetime.now(tz=bson.tz_util.utc) + now = utils.utcnow() lookup_expired = lookup.copy() lookup_expired['link_expires'] = {'$lte': now} @@ -527,7 +526,7 @@ def refresh_links_for_project(project_uuid, chunk_size, expiry_seconds): # Retrieve expired links. files_collection = current_app.data.driver.db['files'] - now = datetime.datetime.now(tz=bson.tz_util.utc) + now = utils.utcnow() expire_before = now + datetime.timedelta(seconds=expiry_seconds) log.info('Limiting to links that expire before %s', expire_before) @@ -556,7 +555,7 @@ def refresh_links_for_backend(backend_name, chunk_size, expiry_seconds): files_collection = current_app.data.driver.db['files'] proj_coll = current_app.data.driver.db['projects'] - now = datetime.datetime.now(tz=bson.tz_util.utc) + now = utils.utcnow() expire_before = now + datetime.timedelta(seconds=expiry_seconds) my_log.info('Limiting to links that expire before %s', expire_before) diff --git a/pillar/api/file_storage/moving.py b/pillar/api/file_storage/moving.py index 5c37831a..d0d6613d 100644 --- a/pillar/api/file_storage/moving.py +++ b/pillar/api/file_storage/moving.py @@ -1,16 +1,15 @@ """Code for moving files between backends.""" -import datetime import logging import os import tempfile -import bson.tz_util import requests import requests.exceptions from bson import ObjectId from flask import current_app +from pillar.api import utils from . import stream_to_gcs, generate_all_links, ensure_valid_link __all__ = ['PrerequisiteNotMetError', 'change_file_storage_backend'] @@ -74,8 +73,7 @@ def change_file_storage_backend(file_id, dest_backend): # Generate new links for the file & all variations. This also saves # the new backend we set here. f['backend'] = dest_backend - now = datetime.datetime.now(tz=bson.tz_util.utc) - generate_all_links(f, now) + generate_all_links(f, utils.utcnow()) def copy_file_to_backend(file_id, project_id, file_or_var, src_backend, dest_backend): @@ -190,4 +188,4 @@ def gcs_move_to_bucket(file_id, dest_project_id, skip_gcs=False): # Regenerate the links for this file f['project'] = dest_project_id - generate_all_links(f, now=datetime.datetime.now(tz=bson.tz_util.utc)) + generate_all_links(f, now=utils.utcnow()) diff --git a/pillar/api/local_auth.py b/pillar/api/local_auth.py index 456cb383..8b3fe1bc 100644 --- a/pillar/api/local_auth.py +++ b/pillar/api/local_auth.py @@ -1,15 +1,16 @@ import base64 +import datetime import hashlib import logging import typing import bcrypt -import datetime -from bson import tz_util + from flask import abort, Blueprint, current_app, jsonify, request from pillar.api.utils.authentication import create_new_user_document from pillar.api.utils.authentication import make_unique_username from pillar.api.utils.authentication import store_token +from pillar.api.utils import utcnow blueprint = Blueprint('authentication', __name__) log = logging.getLogger(__name__) @@ -96,7 +97,7 @@ def generate_and_store_token(user_id, days=15, prefix=b'') -> dict: token_bytes = prefix + base64.b64encode(random_bits, altchars=b'xy').strip(b'=') token = token_bytes.decode('ascii') - token_expiry = datetime.datetime.now(tz=tz_util.utc) + datetime.timedelta(days=days) + token_expiry = utcnow() + datetime.timedelta(days=days) token_data = store_token(user_id, token, token_expiry) # Include the token in the returned document so that it can be stored client-side, diff --git a/pillar/api/organizations/__init__.py b/pillar/api/organizations/__init__.py index c5c9de8f..6ab7059a 100644 --- a/pillar/api/organizations/__init__.py +++ b/pillar/api/organizations/__init__.py @@ -4,7 +4,6 @@ Assumes role names that are given to users by organization membership start with the string "org-". """ -import datetime import logging import typing @@ -14,7 +13,7 @@ import flask import werkzeug.exceptions as wz_exceptions from pillar import attrs_extra, current_app -from pillar.api.utils import remove_private_keys +from pillar.api.utils import remove_private_keys, utcnow class OrganizationError(Exception): @@ -281,10 +280,9 @@ class OrgManager: # Join all organization-given roles and roles from the tokens collection. org_roles = aggr_roles(org_coll, {'members': user_id}) self._log.debug('Organization-given roles for user %s: %s', user_id, org_roles) - now = datetime.datetime.now(bson.tz_util.utc) token_roles = aggr_roles(tokens_coll, { 'user': user_id, - 'expire_time': {"$gt": now}, + 'expire_time': {"$gt": utcnow()}, }) self._log.debug('Token-given roles for user %s: %s', user_id, token_roles) org_roles.update(token_roles) diff --git a/pillar/api/projects/patch.py b/pillar/api/projects/patch.py index e523363e..68c90f1d 100644 --- a/pillar/api/projects/patch.py +++ b/pillar/api/projects/patch.py @@ -1,16 +1,14 @@ """Project patching support.""" -import datetime import logging -import bson.tz_util import flask from flask import Blueprint, request import werkzeug.exceptions as wz_exceptions from pillar import current_app from pillar.auth import current_user -from pillar.api.utils import random_etag, str2id +from pillar.api.utils import random_etag, str2id, utcnow from pillar.api.utils import authorization log = logging.getLogger(__name__) @@ -60,7 +58,6 @@ def patch_project(project_id: str): # PATCHing collections, so direct MongoDB modification is used to set # _deleted=False and provide new _etag and _updated values. new_etag = random_etag() - now = datetime.datetime.now(tz=bson.tz_util.utc) log.debug('undeleting files before undeleting project %s', pid) files_coll = current_app.db('files') @@ -68,7 +65,7 @@ def patch_project(project_id: str): {'project': pid}, {'$set': {'_deleted': False, '_etag': new_etag, - '_updated': now}}) + '_updated': utcnow()}}) log.info('undeleted %d of %d file documents of project %s', update_result.modified_count, update_result.matched_count, pid) diff --git a/pillar/api/utils/__init__.py b/pillar/api/utils/__init__.py index f2f8b78f..b605c4ca 100644 --- a/pillar/api/utils/__init__.py +++ b/pillar/api/utils/__init__.py @@ -10,6 +10,7 @@ import typing import urllib.request, urllib.parse, urllib.error import bson.objectid +import bson.tz_util from eve import RFC1123_DATE_FORMAT from flask import current_app from werkzeug import exceptions as wz_exceptions @@ -200,3 +201,7 @@ def random_etag() -> str: randbytes = random.getrandbits(256).to_bytes(32, 'big') return base64.b64encode(randbytes)[:-1].decode() + + +def utcnow() -> datetime.datetime: + return datetime.datetime.now(tz=bson.tz_util.utc) diff --git a/pillar/api/utils/authentication.py b/pillar/api/utils/authentication.py index 5e2767c7..f75b96f9 100644 --- a/pillar/api/utils/authentication.py +++ b/pillar/api/utils/authentication.py @@ -13,12 +13,11 @@ import logging import typing import bson -from bson import tz_util from flask import g, current_app from flask import request from werkzeug import exceptions as wz_exceptions -from pillar.api.utils import remove_private_keys +from pillar.api.utils import remove_private_keys, utcnow log = logging.getLogger(__name__) @@ -209,7 +208,7 @@ def find_token(token, is_subclient_token=False, **extra_filters): # TODO: remove matching on unhashed tokens once all tokens have been hashed. lookup = {'$or': [{'token': token}, {'token_hashed': token_hashed}], 'is_subclient_token': True if is_subclient_token else {'$in': [False, None]}, - 'expire_time': {"$gt": datetime.datetime.now(tz=tz_util.utc)}} + 'expire_time': {"$gt": utcnow()}} lookup.update(extra_filters) db_token = tokens_coll.find_one(lookup) @@ -333,9 +332,7 @@ def _delete_expired_tokens(): token_coll = current_app.data.driver.db['tokens'] - now = datetime.datetime.now(tz_util.utc) - expiry_date = now - datetime.timedelta(days=7) - + expiry_date = utcnow() - datetime.timedelta(days=7) result = token_coll.delete_many({'expire_time': {"$lt": expiry_date}}) # log.debug('Deleted %i expired authentication tokens', result.deleted_count) diff --git a/pillar/cli/maintenance.py b/pillar/cli/maintenance.py index 67799cad..659dda11 100644 --- a/pillar/cli/maintenance.py +++ b/pillar/cli/maintenance.py @@ -391,13 +391,11 @@ def expire_all_project_links(project_uuid): """ import datetime - import bson.tz_util + from pillar.api.utils import utcnow files_collection = current_app.data.driver.db['files'] - now = datetime.datetime.now(tz=bson.tz_util.utc) - expires = now - datetime.timedelta(days=1) - + expires = utcnow() - datetime.timedelta(days=1) result = files_collection.update_many( {'project': ObjectId(project_uuid)}, {'$set': {'link_expires': expires}} diff --git a/pillar/tests/__init__.py b/pillar/tests/__init__.py index 396207ba..1d7677be 100644 --- a/pillar/tests/__init__.py +++ b/pillar/tests/__init__.py @@ -327,8 +327,9 @@ class AbstractPillarTest(TestMinimal): return user def create_valid_auth_token(self, user_id, token='token'): - now = datetime.datetime.now(tz_util.utc) - future = now + datetime.timedelta(days=1) + from pillar.api.utils import utcnow + + future = utcnow() + datetime.timedelta(days=1) with self.app.test_request_context(): from pillar.api.utils import authentication as auth diff --git a/pillar/web/nodes/custom/comments.py b/pillar/web/nodes/custom/comments.py index 6693a8ac..901b5ebc 100644 --- a/pillar/web/nodes/custom/comments.py +++ b/pillar/web/nodes/custom/comments.py @@ -10,10 +10,11 @@ from pillarsdk import Node from pillarsdk import Project import werkzeug.exceptions as wz_exceptions +from pillar.api.utils import utcnow from pillar.web import subquery from pillar.web.nodes.routes import blueprint from pillar.web.utils import gravatar -from pillar.web.utils import pretty_date, datetime_now +from pillar.web.utils import pretty_date from pillar.web.utils import system_util log = logging.getLogger(__name__) @@ -111,7 +112,7 @@ def format_comment(comment, is_reply=False, is_team=False, replies=None): return dict(_id=comment._id, gravatar=gravatar(comment.user.email, size=32), - time_published=pretty_date(comment._created or datetime_now(), detail=True), + time_published=pretty_date(comment._created or utcnow(), detail=True), rating=comment.properties.rating_positive - comment.properties.rating_negative, author=comment.user.full_name, author_username=comment.user.username, diff --git a/pillar/web/projects/routes.py b/pillar/web/projects/routes.py index e931efa9..c7032bf8 100644 --- a/pillar/web/projects/routes.py +++ b/pillar/web/projects/routes.py @@ -21,6 +21,7 @@ from flask_login import login_required, current_user import werkzeug.exceptions as wz_exceptions from pillar import current_app +from pillar.api.utils import utcnow from pillar.web import system_util from pillar.web import utils from pillar.web.utils.jstree import jstree_get_children @@ -82,7 +83,7 @@ def index(): show_deleted_projects = request.args.get('deleted') is not None if show_deleted_projects: - timeframe = utils.datetime_now() - datetime.timedelta(days=31) + timeframe = utcnow() - datetime.timedelta(days=31) projects_deleted = Project.all({ 'where': {'user': current_user.objectid, 'category': {'$ne': 'home'}, diff --git a/pillar/web/utils/__init__.py b/pillar/web/utils/__init__.py index b166be12..3b80c1ef 100644 --- a/pillar/web/utils/__init__.py +++ b/pillar/web/utils/__init__.py @@ -114,12 +114,6 @@ def gravatar(email: str, size=64): return api_gravatar(email, size) -def datetime_now(): - """Returns a datetime.datetime that represents 'now' in UTC.""" - - return datetime.datetime.now(tz=pillarsdk.utils.utc) - - def pretty_date(time, detail=False, now=None): """Get a datetime object or a int() Epoch timestamp and return a pretty string like 'an hour ago', 'Yesterday', '3 months ago',