diff --git a/pillar/api/file_storage/__init__.py b/pillar/api/file_storage/__init__.py index 4c48d55f..af31b763 100644 --- a/pillar/api/file_storage/__init__.py +++ b/pillar/api/file_storage/__init__.py @@ -130,6 +130,67 @@ def _process_image(bucket: Bucket, src_file['status'] = 'complete' +def _video_duration_seconds(filename: pathlib.Path) -> typing.Optional[int]: + """Get the duration of a video file using ffprobe + https://superuser.com/questions/650291/how-to-get-video-duration-in-seconds + + :param filename: file path to video + :return: video duration in seconds + """ + import subprocess + + def run(cli_args): + if log.isEnabledFor(logging.INFO): + import shlex + cmd = ' '.join(shlex.quote(s) for s in cli_args) + log.info('Calling %s', cmd) + + ffprobe = subprocess.run( + cli_args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=10, # seconds + ) + + if ffprobe.returncode: + import shlex + cmd = ' '.join(shlex.quote(s) for s in cli_args) + log.error('Error running %s: stopped with return code %i', + cmd, ffprobe.returncode) + log.error('Output was: %s', ffprobe.stdout) + return None + + try: + return int(float(ffprobe.stdout)) + except ValueError as e: + log.exception('ffprobe produced invalid number: %s', ffprobe.stdout) + return None + + ffprobe_from_container_args = [ + current_app.config['BIN_FFPROBE'], + '-v', 'error', + '-show_entries', 'format=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', + str(filename), + ] + + ffprobe_from_stream_args = [ + current_app.config['BIN_FFPROBE'], + '-v', 'error', + '-hide_banner', + '-select_streams', 'v:0', # we only care about the first video stream + '-show_entries', 'stream=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', + str(filename), + ] + + duration = run(ffprobe_from_stream_args) or\ + run(ffprobe_from_container_args) or\ + None + return duration + + def _video_size_pixels(filename: pathlib.Path) -> typing.Tuple[int, int]: """Figures out the size (in pixels) of the video file. @@ -220,8 +281,10 @@ def _process_video(gcs, # by determining the video size here we already have this information in the file # document before Zencoder calls our notification URL. It also opens up possibilities # for other encoding backends that don't support this functionality. - video_width, video_height = _video_size_pixels(pathlib.Path(local_file.name)) + video_path = pathlib.Path(local_file.name) + video_width, video_height = _video_size_pixels(video_path) capped_video_width, capped_video_height = _video_cap_at_1080(video_width, video_height) + video_duration = _video_duration_seconds(video_path) # Create variations root, _ = os.path.splitext(src_file['file_path']) @@ -234,12 +297,13 @@ def _process_video(gcs, content_type='video/{}'.format(v), file_path='{}-{}.{}'.format(root, v, v), size='', - duration=0, width=capped_video_width, height=capped_video_height, length=0, md5='', ) + if video_duration: + file_variation['duration'] = video_duration # Append file variation. Originally mp4 and webm were the available options, # that's why we build a list. src_file['variations'].append(file_variation) diff --git a/pillar/api/latest.py b/pillar/api/latest.py index 3a0879d0..33076929 100644 --- a/pillar/api/latest.py +++ b/pillar/api/latest.py @@ -70,6 +70,7 @@ def latest_assets(): {'name': 1, 'node_type': 1, 'parent': 1, 'picture': 1, 'properties.status': 1, 'properties.content_type': 1, + 'properties.duration_seconds': 1, 'permissions.world': 1}, 12) diff --git a/pillar/api/node_types/asset.py b/pillar/api/node_types/asset.py index 407ab05e..8eba132c 100644 --- a/pillar/api/node_types/asset.py +++ b/pillar/api/node_types/asset.py @@ -24,6 +24,10 @@ node_type_asset = { 'content_type': { 'type': 'string' }, + # The duration of a video asset in seconds. + 'duration_seconds': { + 'type': 'integer' + }, # We point to the original file (and use it to extract any relevant # variation useful for our scope). 'file': _file_embedded_schema, @@ -58,6 +62,7 @@ node_type_asset = { }, 'form_schema': { 'content_type': {'visible': False}, + 'duration_seconds': {'visible': False}, 'order': {'visible': False}, 'tags': {'visible': False}, 'categories': {'visible': False}, diff --git a/pillar/api/nodes/__init__.py b/pillar/api/nodes/__init__.py index 06ced783..240d1f7f 100644 --- a/pillar/api/nodes/__init__.py +++ b/pillar/api/nodes/__init__.py @@ -66,8 +66,8 @@ def tagged(tag=''): agg_list = _tagged(tag) for node in agg_list: - if node.get('video_duration_seconds'): - node['video_duration'] = datetime.timedelta(seconds=node['video_duration_seconds']) + if node['properties'].get('duration_seconds'): + node['properties']['duration'] = datetime.timedelta(seconds=node['properties']['duration_seconds']) if node.get('_created') is not None: node['pretty_created'] = pretty_date(node['_created']) @@ -108,26 +108,16 @@ def _tagged(tag: str): 'foreignField': '_id', 'as': '_project', }}, - {'$lookup': { - 'from': 'files', - 'localField': 'properties.file', - 'foreignField': '_id', - 'as': '_file', - }}, - {'$unwind': '$_file'}, {'$unwind': '$_project'}, {'$match': {'_project.is_private': False}}, {'$addFields': { 'project._id': '$_project._id', 'project.name': '$_project.name', 'project.url': '$_project.url', - 'video_duration_seconds': {'$arrayElemAt': ['$_file.variations.duration', 0]}, }}, # Don't return the entire project/file for each node. - {'$project': {'_project': False, - '_file': False} - }, + {'$project': {'_project': False}}, {'$sort': {'_created': -1}} ]) @@ -224,13 +214,13 @@ def setup_app(app, url_prefix): app.on_replace_nodes += eve_hooks.before_replacing_node app.on_replace_nodes += eve_hooks.parse_markdown app.on_replace_nodes += eve_hooks.texture_sort_files - app.on_replace_nodes += eve_hooks.deduct_content_type + app.on_replace_nodes += eve_hooks.deduct_content_type_and_duration app.on_replace_nodes += eve_hooks.node_set_default_picture app.on_replaced_nodes += eve_hooks.after_replacing_node app.on_insert_nodes += eve_hooks.before_inserting_nodes app.on_insert_nodes += eve_hooks.parse_markdowns - app.on_insert_nodes += eve_hooks.nodes_deduct_content_type + app.on_insert_nodes += eve_hooks.nodes_deduct_content_type_and_duration app.on_insert_nodes += eve_hooks.nodes_set_default_picture app.on_insert_nodes += eve_hooks.textures_sort_files app.on_inserted_nodes += eve_hooks.after_inserting_nodes diff --git a/pillar/api/nodes/eve_hooks.py b/pillar/api/nodes/eve_hooks.py index a11f434a..ba131f87 100644 --- a/pillar/api/nodes/eve_hooks.py +++ b/pillar/api/nodes/eve_hooks.py @@ -165,7 +165,7 @@ def after_inserting_nodes(items): ) -def deduct_content_type(node_doc, original=None): +def deduct_content_type_and_duration(node_doc, original=None): """Deduct the content type from the attached file, if any.""" if node_doc['node_type'] != 'asset': @@ -184,7 +184,8 @@ def deduct_content_type(node_doc, original=None): files = current_app.data.driver.db['files'] file_doc = files.find_one({'_id': file_id}, - {'content_type': 1}) + {'content_type': 1, + 'variations': 1}) if not file_doc: log.warning('deduct_content_type: Node %s refers to non-existing file %s, rejecting.', node_id, file_id) @@ -201,10 +202,17 @@ def deduct_content_type(node_doc, original=None): node_doc['properties']['content_type'] = content_type + if content_type == 'video': + duration = file_doc['variations'][0].get('duration') + if duration: + node_doc['properties']['duration_seconds'] = duration + else: + log.warning('Video file %s has no duration', file_id) -def nodes_deduct_content_type(nodes): + +def nodes_deduct_content_type_and_duration(nodes): for node in nodes: - deduct_content_type(node) + deduct_content_type_and_duration(node) def node_set_default_picture(node, original=None): diff --git a/pillar/api/utils/__init__.py b/pillar/api/utils/__init__.py index 91bbe47c..df279287 100644 --- a/pillar/api/utils/__init__.py +++ b/pillar/api/utils/__init__.py @@ -57,6 +57,18 @@ def remove_private_keys(document): return doc_copy +def pretty_duration(seconds): + if seconds is None: + return '' + seconds = round(seconds) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + if hours > 0: + return f'{hours:02}:{minutes:02}:{seconds:02}' + else: + return f'{minutes:02}:{seconds:02}' + + class PillarJSONEncoder(json.JSONEncoder): """JSON encoder with support for Pillar resources.""" @@ -65,12 +77,7 @@ class PillarJSONEncoder(json.JSONEncoder): return obj.strftime(RFC1123_DATE_FORMAT) if isinstance(obj, datetime.timedelta): - hours, seconds = divmod(obj.seconds, 3600) - minutes, seconds = divmod(seconds, 60) - if hours > 0: - return f'{hours:02}:{minutes:02}:{seconds:02}' - else: - return f'{minutes:02}:{seconds:02}' + return pretty_duration(obj.total_seconds()) if isinstance(obj, bson.ObjectId): return str(obj) diff --git a/pillar/cli/maintenance.py b/pillar/cli/maintenance.py index a88a489e..c31a3490 100644 --- a/pillar/cli/maintenance.py +++ b/pillar/cli/maintenance.py @@ -1022,3 +1022,156 @@ def delete_orphan_files(): log.warning('Soft-deletion modified %d of %d files', res.modified_count, file_count) log.info('%d files have been soft-deleted', res.modified_count) + + +@manager_maintenance.command +def find_video_files_without_duration(): + """Finds video files without any duration + + This is a heavy operation. Use with care. + """ + from pathlib import Path + + output_fpath = Path(current_app.config['STORAGE_DIR']) / 'video_files_without_duration.txt' + if output_fpath.exists(): + log.error('Output filename %s already exists, remove it first.', output_fpath) + return 1 + + start_timestamp = datetime.datetime.now() + files_coll = current_app.db('files') + starts_with_video = re.compile("^video", re.IGNORECASE) + aggr = files_coll.aggregate([ + {'$match': {'content_type': starts_with_video, + '_deleted': {'$ne': True}}}, + {'$unwind': '$variations'}, + {'$match': { + 'variations.duration': {'$not': {'$gt': 0}} + }}, + {'$project': {'_id': 1}} + ]) + + file_ids = [str(f['_id']) for f in aggr] + nbr_files = len(file_ids) + log.info('Total nbr video files without duration: %d', nbr_files) + + end_timestamp = datetime.datetime.now() + duration = end_timestamp - start_timestamp + log.info('Finding files took %s', duration) + + log.info('Writing Object IDs to %s', output_fpath) + with output_fpath.open('w', encoding='ascii') as outfile: + outfile.write('\n'.join(sorted(file_ids))) + +@manager_maintenance.command +def find_video_nodes_without_duration(): + """Finds video nodes without any duration + + This is a heavy operation. Use with care. + """ + from pathlib import Path + + output_fpath = Path(current_app.config['STORAGE_DIR']) / 'video_nodes_without_duration.txt' + if output_fpath.exists(): + log.error('Output filename %s already exists, remove it first.', output_fpath) + return 1 + + start_timestamp = datetime.datetime.now() + nodes_coll = current_app.db('nodes') + + aggr = nodes_coll.aggregate([ + {'$match': {'node_type': 'asset', + 'properties.content_type': 'video', + '_deleted': {'$ne': True}, + 'properties.duration_seconds': {'$not': {'$gt': 0}}}}, + {'$project': {'_id': 1}} + ]) + + file_ids = [str(f['_id']) for f in aggr] + nbr_files = len(file_ids) + log.info('Total nbr video nodes without duration: %d', nbr_files) + + end_timestamp = datetime.datetime.now() + duration = end_timestamp - start_timestamp + log.info('Finding nodes took %s', duration) + + log.info('Writing Object IDs to %s', output_fpath) + with output_fpath.open('w', encoding='ascii') as outfile: + outfile.write('\n'.join(sorted(file_ids))) + + +@manager_maintenance.option('-n', '--nodes', dest='nodes_to_update', nargs='*', + help='List of nodes to update') +@manager_maintenance.option('-a', '--all', dest='all_nodes', action='store_true', default=False, + help='Update on all video nodes.') +@manager_maintenance.option('-g', '--go', dest='go', action='store_true', default=False, + help='Actually perform the changes (otherwise just show as dry-run).') +def reconcile_node_video_duration(nodes_to_update=None, all_nodes=False, go=False): + """Copy video duration from file.variations.duration to node.properties.duraion_seconds + + This is a heavy operation. Use with care. + """ + from pillar.api.utils import random_etag, utcnow + + if bool(nodes_to_update) == all_nodes: + log.error('Use either --nodes or --all.') + return 1 + + start_timestamp = datetime.datetime.now() + + nodes_coll = current_app.db('nodes') + node_subset = [] + if nodes_to_update: + node_subset = [{'$match': {'_id': {'$in': [ObjectId(nid) for nid in nodes_to_update]}}}] + files = nodes_coll.aggregate( + [ + *node_subset, + {'$match': { + 'node_type': 'asset', + 'properties.content_type': 'video', + '_deleted': {'$ne': True}} + }, + {'$lookup': { + 'from': 'files', + 'localField': 'properties.file', + 'foreignField': '_id', + 'as': '_files', + }}, + {'$unwind': '$_files'}, + {'$unwind': '$_files.variations'}, + {'$match': {'_files.variations.duration': {'$gt': 0}}}, + {'$addFields': { + 'need_update': {'$ne': ['$_files.variations.duration', '$properties.duration_seconds']} + }}, + {'$match': {'need_update': True}}, + {'$project': { + '_id': 1, + 'duration': '$_files.variations.duration', + }}] + ) + + if not go: + log.info('Would try to update %d nodes', len(list(files))) + return 0 + + modified_count = 0 + for f in files: + log.debug('Updating node %s with duration %d', f['_id'], f['duration']) + new_etag = random_etag() + now = utcnow() + resp = nodes_coll.update_one( + {'_id': f['_id']}, + {'$set': { + 'properties.duration_seconds': f['duration'], + '_etag': new_etag, + '_updated': now, + }} + ) + if resp.modified_count == 0: + log.debug('Node %s was already up to date', f['_id']) + modified_count += resp.modified_count + + log.info('Updated %d nodes', modified_count) + end_timestamp = datetime.datetime.now() + duration = end_timestamp - start_timestamp + log.info('Operation took %s', duration) + return 0 diff --git a/pillar/web/jinja.py b/pillar/web/jinja.py index 171f4dee..bbb6ef3e 100644 --- a/pillar/web/jinja.py +++ b/pillar/web/jinja.py @@ -13,6 +13,7 @@ import werkzeug.exceptions as wz_exceptions import pillarsdk import pillar.api.utils +from pillar.api.utils import pretty_duration from pillar.web.utils import pretty_date from pillar.web.nodes.routes import url_for_node import pillar.markdown @@ -28,6 +29,10 @@ def format_pretty_date_time(d): return pretty_date(d, detail=True) +def format_pretty_duration(s): + return pretty_duration(s) + + def format_undertitle(s): """Underscore-replacing title filter. @@ -203,6 +208,7 @@ def do_yesno(value, arg=None): def setup_jinja_env(jinja_env, app_config: dict): jinja_env.filters['pretty_date'] = format_pretty_date jinja_env.filters['pretty_date_time'] = format_pretty_date_time + jinja_env.filters['pretty_duration'] = format_pretty_duration jinja_env.filters['undertitle'] = format_undertitle jinja_env.filters['hide_none'] = do_hide_none jinja_env.filters['pluralize'] = do_pluralize diff --git a/pillar/web/projects/routes.py b/pillar/web/projects/routes.py index 53e98852..589611f1 100644 --- a/pillar/web/projects/routes.py +++ b/pillar/web/projects/routes.py @@ -361,6 +361,7 @@ def render_project(project, api, extra_context=None, template_name=None): # Construct query parameters outside the loop. projection = {'name': 1, 'user': 1, 'node_type': 1, 'project': 1, 'properties.url': 1, 'properties.content_type': 1, + 'properties.duration_seconds': 1, 'picture': 1} params = {'projection': projection, 'embedded': {'user': 1}} diff --git a/src/templates/_macros/_asset_list_item.pug b/src/templates/_macros/_asset_list_item.pug index d8f09fc8..a03a3c8f 100644 --- a/src/templates/_macros/_asset_list_item.pug +++ b/src/templates/_macros/_asset_list_item.pug @@ -37,6 +37,9 @@ a.card.asset.card-image-fade.pr-0.mx-0.mb-2( .card-label WATCHED | {% endif %} | {% endif %} {# endif progress #} + | {% if asset.properties.duration_seconds %} + .card-label.right {{ asset.properties.duration_seconds | pretty_duration }} + | {% endif %} | {% endif %} {# endif video #} diff --git a/src/templates/nodes/view_base.pug b/src/templates/nodes/view_base.pug index 8d9c85c1..f93991a6 100644 --- a/src/templates/nodes/view_base.pug +++ b/src/templates/nodes/view_base.pug @@ -62,6 +62,10 @@ section.node-details-meta.pl-4.pr-2.py-2.border-bottom li.ml-auto + | {% if node.properties.duration_seconds %} + li.px-2(title="Duration") + | {{ node.properties.duration_seconds | pretty_duration }} + | {% endif %} | {% if node.file %} li.px-2(title="File size") | {{ node.file.length | filesizeformat }} diff --git a/tests/test_api/test_file_storage.py b/tests/test_api/test_file_storage.py index bcc87515..9d3a8e09 100644 --- a/tests/test_api/test_file_storage.py +++ b/tests/test_api/test_file_storage.py @@ -280,3 +280,23 @@ class VideoSizeTest(AbstractPillarTest): size = file_storage._video_cap_at_1080(2048, 2000) self.assertIsInstance(size[0], int) self.assertIsInstance(size[1], int) + + +class VideoDurationTest(AbstractPillarTest): + def test_video_duration_from_container(self): + from pillar.api import file_storage + from pathlib import Path + + with self.app.test_request_context(): + fname = Path(__file__).with_name('video-tiny.mkv') + + self.assertEqual(1, file_storage._video_duration_seconds(fname)) + + def test_video_duration_from_stream(self): + from pillar.api import file_storage + from pathlib import Path + + with self.app.test_request_context(): + fname = Path(__file__).with_name('video-tiny.mp4') + + self.assertEqual(2, file_storage._video_duration_seconds(fname)) diff --git a/tests/test_api/test_latest.py b/tests/test_api/test_latest.py new file mode 100644 index 00000000..2619ea12 --- /dev/null +++ b/tests/test_api/test_latest.py @@ -0,0 +1,303 @@ +from datetime import timedelta + +import flask + +from pillar.tests import AbstractPillarTest + + +class LatestAssetsTest(AbstractPillarTest): + def setUp(self, **kwargs): + super().setUp(**kwargs) + + self.pid, _ = self.ensure_project_exists() + self.private_pid, _ = self.ensure_project_exists(project_overrides={ + '_id': '5672beecc0261b2005ed1a34', + 'is_private': True, + }) + self.file_id, _ = self.ensure_file_exists(file_overrides={ + 'variations': [ + {'format': 'mp4', + 'duration': 3661 # 01:01:01 + }, + ], + }) + self.uid = self.create_user() + + from pillar.api.utils import utcnow + self.fake_now = utcnow() + + def test_latest_assets_returns_12_newest_assets(self): + base_node = { + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + 'user': self.uid, + } + base_props = { + 'status': 'published', + 'file': self.file_id, + 'content_type': 'video', + 'order': 0 + } + + def create_asset(weeks): + return self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(weeks=weeks), + 'properties': base_props, + }) + + all_asset_ids = [str(create_asset(i)) for i in range(20)] + expected_ids = all_asset_ids[:12] # The 12 newest assets are expected + + + with self.app.app_context(): + url = flask.url_for('latest.latest_assets') + latest_assets = self.get(url).json['_items'] + + actual_ids = [asset['_id'] for asset in latest_assets] + self.assertListEqual( + expected_ids, actual_ids) + + def test_latest_assets_ignore(self): + base_node = { + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + 'user': self.uid, + } + base_props = { + 'status': 'published', + 'file': self.file_id, + 'content_type': 'video', + 'order': 0 + } + + ok_id = self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + }) + + # Private should be ignored + self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + 'project': self.private_pid, + }) + + # Deleted should be ignored + self.create_node({ + **base_node, + '_deleted': True, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + }) + + # Node type comment should be ignored + self.create_node({ + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'comment', + 'user': self.uid, + }) + + with self.app.app_context(): + url = flask.url_for('latest.latest_assets') + latest_assets = self.get(url).json['_items'] + + expected_ids = [str(ok_id)] + actual_ids = [asset['_id'] for asset in latest_assets] + self.assertListEqual( + expected_ids, actual_ids) + + def test_latest_assets_data(self): + base_node = { + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + 'user': self.uid, + } + base_props = { + 'status': 'published', + 'file': self.file_id, + 'content_type': 'video', + 'order': 0 + } + + ok_id = self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + }) + + with self.app.app_context(): + url = flask.url_for('latest.latest_assets') + latest_assets = self.get(url).json['_items'] + + asset = latest_assets[0] + self.assertEquals(str(ok_id), asset['_id']) + self.assertEquals('Just a node name', asset['name']) + + +class LatestCommentsTest(AbstractPillarTest): + def setUp(self, **kwargs): + super().setUp(**kwargs) + + self.pid, _ = self.ensure_project_exists() + self.private_pid, _ = self.ensure_project_exists(project_overrides={ + '_id': '5672beecc0261b2005ed1a34', + 'is_private': True, + }) + self.file_id, _ = self.ensure_file_exists(file_overrides={ + 'variations': [ + {'format': 'mp4', + 'duration': 3661 # 01:01:01 + }, + ], + }) + self.uid = self.create_user() + + from pillar.api.utils import utcnow + self.fake_now = utcnow() + + base_props = { + 'status': 'published', + 'file': self.file_id, + 'content_type': 'video', + 'order': 0 + } + + self.asset_node_id = self.create_node({ + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + 'user': self.uid, + '_created': self.fake_now - timedelta(weeks=52), + 'properties': base_props, + }) + + def test_latest_comments_returns_10_newest_comments(self): + base_node = { + 'name': 'Comment', + 'project': self.pid, + 'description': '', + 'node_type': 'comment', + 'user': self.uid, + 'parent': self.asset_node_id, + } + base_props = { + 'status': 'published', + 'content': 'एनिमेशन is animation in Hindi', + } + + def create_comment(weeks): + return self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(weeks=weeks), + 'properties': base_props, + }) + + all_comment_ids = [str(create_comment(i)) for i in range(20)] + expected_ids = all_comment_ids[:10] # The 10 newest comments are expected + + with self.app.app_context(): + url = flask.url_for('latest.latest_comments') + latest_assets = self.get(url).json['_items'] + + actual_ids = [asset['_id'] for asset in latest_assets] + self.assertListEqual( + expected_ids, actual_ids) + + def test_latest_comments_ignore(self): + base_node = { + 'name': 'Comment', + 'project': self.pid, + 'description': '', + 'node_type': 'comment', + 'user': self.uid, + 'parent': self.asset_node_id, + } + base_props = { + 'status': 'published', + 'content': 'एनिमेशन is animation in Hindi', + } + + ok_id = self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + }) + + # Private should be ignored + self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + 'project': self.private_pid, + }) + + # Deleted should be ignored + self.create_node({ + **base_node, + '_deleted': True, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + }) + + # Node type asset should be ignored + self.create_node({ + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + 'user': self.uid, + }) + + with self.app.app_context(): + url = flask.url_for('latest.latest_comments') + latest_comments = self.get(url).json['_items'] + + expected_ids = [str(ok_id)] + actual_ids = [comment['_id'] for comment in latest_comments] + self.assertListEqual( + expected_ids, actual_ids) + + def test_latest_comments_data(self): + base_node = { + 'name': 'Comment', + 'project': self.pid, + 'description': '', + 'node_type': 'comment', + 'user': self.uid, + 'parent': self.asset_node_id, + } + base_props = { + 'status': 'published', + 'content': 'एनिमेशन is animation in Hindi', + } + + ok_id = self.create_node({ + **base_node, + '_created': self.fake_now - timedelta(seconds=1), + 'properties': base_props, + }) + + with self.app.app_context(): + url = flask.url_for('latest.latest_comments') + latest_comments = self.get(url).json['_items'] + + comment = latest_comments[0] + self.assertEquals(str(ok_id), comment['_id']) + self.assertEquals('Comment', comment['name']) + self.assertEquals('एनिमेशन is animation in Hindi', comment['properties']['content']) diff --git a/tests/test_api/test_nodes.py b/tests/test_api/test_nodes.py index dbcf876a..d495b1f4 100644 --- a/tests/test_api/test_nodes.py +++ b/tests/test_api/test_nodes.py @@ -654,6 +654,7 @@ class TaggedNodesTest(AbstractPillarTest): base_props = {'status': 'published', 'file': self.file_id, 'content_type': 'video', + 'duration_seconds': 3661, # 01:01:01 'order': 0} self.create_node({ @@ -668,7 +669,7 @@ class TaggedNodesTest(AbstractPillarTest): mock_utcnow.return_value = self.fake_now url = flask.url_for('nodes_api.tagged', tag='एनिमेशन') resp = self.get(url).json[0] - self.assertEquals('01:01:01', resp['video_duration']) + self.assertEquals('01:01:01', resp['properties']['duration']) self.assertEquals('Unittest project', resp['project']['name']) self.assertEquals('default-project', resp['project']['url']) self.assertEquals('5m ago', resp['pretty_created']) diff --git a/tests/test_api/test_utils.py b/tests/test_api/test_utils.py index 46340830..7631435b 100644 --- a/tests/test_api/test_utils.py +++ b/tests/test_api/test_utils.py @@ -212,3 +212,17 @@ class TestRating(unittest.TestCase): sorted_by_hot = sorted(cases, key=lambda tup: tup[0]) for idx, t in enumerate(sorted_by_hot): self.assertEqual(cases[idx][0], t[0]) + + +class TestPrettyDuration(unittest.TestCase): + def test_formatting(self): + from pillar.api.utils import pretty_duration + pretty_duration(500) + self.assertEquals('00:00', pretty_duration(0)) + self.assertEquals('00:15', pretty_duration(15)) + self.assertEquals('01:05', pretty_duration(65)) + self.assertEquals('42:53', pretty_duration(2573)) + self.assertEquals('01:11:22', pretty_duration(4282)) + self.assertEquals('01:41', pretty_duration(100.85)) + self.assertEquals('25:00:00', pretty_duration(90000)) # More than a day + self.assertEquals('', pretty_duration(None)) diff --git a/tests/test_api/video-tiny.mp4 b/tests/test_api/video-tiny.mp4 new file mode 100644 index 00000000..3fcb3252 Binary files /dev/null and b/tests/test_api/video-tiny.mp4 differ diff --git a/tests/test_cli/test_maintenance.py b/tests/test_cli/test_maintenance.py index 70fd6ed3..ac5e7b67 100644 --- a/tests/test_cli/test_maintenance.py +++ b/tests/test_cli/test_maintenance.py @@ -1,5 +1,8 @@ +from unittest import mock + from bson import ObjectId +from pillar.api.utils import random_etag, utcnow from pillar.tests import AbstractPillarTest from pillar.tests import common_test_data as ctd @@ -250,3 +253,149 @@ class UpgradeAttachmentUsageTest(AbstractPillarTest): }, node['properties']['attachments'], 'The link should have been removed from the attachment') + + +class ReconcileNodeDurationTest(AbstractPillarTest): + def setUp(self, **kwargs): + super().setUp(**kwargs) + self.pid, _ = self.ensure_project_exists() + self.fake_now = utcnow() + + # Already correct. Should not be touched. + self.node_id0 = self._create_video_node(file_duration=123, node_duration=123) + # Out of sync, should be updated + self.node_id1 = self._create_video_node(file_duration=3661, node_duration=15) + self.node_id2 = self._create_video_node(file_duration=432, node_duration=5) + self.node_id3 = self._create_video_node(file_duration=222) + # No file duration. Should not be touched + self.node_id4 = self._create_video_node() + # No file. Should not be touched + self.node_id5 = self._create_video_node(include_file=False) + # Wrong node type. Should not be touched + self.image_node_id = self._create_image_node() + + def id_to_original_dict(*nids): + with self.app.app_context(): + nodes_coll = self.app.db('nodes') + return dict(((nid, nodes_coll.find_one({'_id': nid})) for nid in nids)) + + self.orig_nodes = id_to_original_dict( + self.node_id0, + self.node_id1, + self.node_id2, + self.node_id3, + self.node_id4, + self.node_id5, + self.image_node_id, + ) + + def test_reconcile_all(self): + from pillar.cli.maintenance import reconcile_node_video_duration + + with self.app.app_context(): + with mock.patch('pillar.api.utils.utcnow') as mock_utcnow: + mock_utcnow.return_value = self.fake_now + + reconcile_node_video_duration(all_nodes=True, go=False) # Dry run + self.assertAllUnchanged() + + reconcile_node_video_duration(all_nodes=True, go=True) + self.assertUnChanged( + self.node_id0, + self.node_id4, + self.image_node_id, + ) + self.assertUpdated(self.node_id1, duration_seconds=3661) + self.assertUpdated(self.node_id2, duration_seconds=432) + self.assertUpdated(self.node_id3, duration_seconds=222) + + def test_reconcile_some(self): + from pillar.cli.maintenance import reconcile_node_video_duration + + with self.app.app_context(): + with mock.patch('pillar.api.utils.utcnow') as mock_utcnow: + mock_utcnow.return_value = self.fake_now + + to_reconcile = [str(self.node_id0), str(self.node_id1), str(self.node_id2), str(self.node_id5)] + reconcile_node_video_duration(nodes_to_update=to_reconcile, go=False) # Dry run + self.assertAllUnchanged() + + reconcile_node_video_duration(nodes_to_update=to_reconcile, go=True) + self.assertUnChanged( + self.node_id0, + self.node_id3, + self.node_id4, + self.image_node_id, + ) + self.assertUpdated(self.node_id1, duration_seconds=3661) + self.assertUpdated(self.node_id2, duration_seconds=432) + + def assertUpdated(self, nid, duration_seconds): + nodes_coll = self.app.db('nodes') + new_node = nodes_coll.find_one({'_id': nid}) + orig_node = self.orig_nodes[nid] + self.assertNotEqual(orig_node['_etag'], new_node['_etag']) + self.assertEquals(self.fake_now, new_node['_updated']) + self.assertEquals(duration_seconds, new_node['properties']['duration_seconds']) + + def assertAllUnchanged(self): + self.assertUnChanged(*self.orig_nodes.keys()) + + def assertUnChanged(self, *node_ids): + nodes_coll = self.app.db('nodes') + for nid in node_ids: + new_node = nodes_coll.find_one({'_id': nid}) + orig_node = self.orig_nodes[nid] + self.assertEquals(orig_node, new_node) + + def _create_video_node(self, file_duration=None, node_duration=None, include_file=True): + file_id, _ = self.ensure_file_exists(file_overrides={ + '_id': ObjectId(), + 'content_type': 'video/mp4', + 'variations': [ + {'format': 'mp4', + 'duration': file_duration + }, + ], + }) + + node = { + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + '_etag': random_etag(), + } + props = {'status': 'published', + 'content_type': 'video', + 'order': 0} + if node_duration is not None: + props['duration_seconds'] = node_duration + if include_file: + props['file'] = file_id + return self.create_node({ + 'properties': props, + **node}) + + def _create_image_node(self): + file_id, _ = self.ensure_file_exists(file_overrides={ + '_id': ObjectId(), + 'variations': [ + {'format': 'jpeg'}, + ], + }) + + node = { + 'name': 'Just a node name', + 'project': self.pid, + 'description': '', + 'node_type': 'asset', + '_etag': random_etag(), + } + props = {'status': 'published', + 'file': file_id, + 'content_type': 'image', + 'order': 0} + return self.create_node({ + 'properties': props, + **node}) diff --git a/tests/test_web/test_jstree.py b/tests/test_web/test_jstree.py index c0ebb35a..6925bfa7 100644 --- a/tests/test_web/test_jstree.py +++ b/tests/test_web/test_jstree.py @@ -44,6 +44,38 @@ class JSTreeTest(AbstractPillarTest): 'custom_view': False, }) + def test_jstree_parse_video_node(self): + from pillar.web.utils.jstree import jstree_parse_node + + node_doc = {'_id': ObjectId('55f338f92beb3300c4ff99fe'), + '_created': parse('2015-09-11T22:26:33.000+0200'), + '_updated': parse('2015-10-30T22:44:27.000+0100'), + '_etag': '5248485b4ea7e55e858ff84b1bd4aae88917a37c', + 'picture': ObjectId('55f338f92beb3300c4ff99de'), + 'description': 'Play the full movie and see how it was cobbled together.', + 'parent': ObjectId('55f338f92beb3300c4ff99f9'), + 'project': self.project_id, + 'node_type': 'asset', + 'user': ObjectId('552b066b41acdf5dec4436f2'), + 'properties': {'status': 'published', + 'file': ObjectId('55f338f92beb3300c4ff99c2'), + 'content_type': 'video', + }, + 'name': 'Live Edit'} + + with self.app.test_request_context(): + parsed = jstree_parse_node(Node(node_doc)) + + self.assertEqual(parsed, { + 'id': 'n_55f338f92beb3300c4ff99fe', + 'a_attr': {'href': f"/p/{self.project['url']}/55f338f92beb3300c4ff99fe"}, + 'li_attr': {'data-node-type': 'asset'}, + 'text': Markup('Live <strong>Edit</strong>'), + 'type': 'video', + 'children': False, + 'custom_view': False, + }) + def test_jstree_parse_blog_node(self): from pillar.web.utils.jstree import jstree_parse_node