archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Video Duration: The duration of a video is now shown on thumbnails and bellow the video player

Browse Source 
Asset nodes now have a new field called "properties.duration_seconds". This holds a copy of the duration stored on the referenced video file and stays in sync using eve hooks.

To migrate existing duration times from files to nodes you need to run the following:
./manage.py maintenance reconcile_node_video_duration -ag

There are 2 more maintenance commands to be used to determine if there are any missing durations in either files or nodes:
find_video_files_without_duration
find_video_nodes_without_duration

FFProbe is now used to detect what duration a video file has.

Reviewed by Sybren.
This commit is contained in:
Tobias Johansson 2018-10-03 18:30:40 +02:00
parent a738cdcad8
commit 6ad12d0098
18 changed files with 789 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
pillar/api/file_storage/__init__.py
View File

@ -130,6 +130,67 @@ def _process_image(bucket: Bucket,
src_file['status'] = 'complete'
def _video_duration_seconds(filename: pathlib.Path) -> typing.Optional[int]:
"""Get the duration of a video file using ffprobe
https://superuser.com/questions/650291/how-to-get-video-duration-in-seconds
:param filename: file path to video
:return: video duration in seconds
"""
import subprocess
def run(cli_args):
if log.isEnabledFor(logging.INFO):
import shlex
cmd = ' '.join(shlex.quote(s) for s in cli_args)
log.info('Calling %s', cmd)
ffprobe = subprocess.run(
cli_args,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=10, # seconds
)
if ffprobe.returncode:
import shlex
cmd = ' '.join(shlex.quote(s) for s in cli_args)
log.error('Error running %s: stopped with return code %i',
cmd, ffprobe.returncode)
log.error('Output was: %s', ffprobe.stdout)
return None
try:
return int(float(ffprobe.stdout))
except ValueError as e:
log.exception('ffprobe produced invalid number: %s', ffprobe.stdout)
return None
ffprobe_from_container_args = [
current_app.config['BIN_FFPROBE'],
'-v', 'error',
'-show_entries', 'format=duration',
'-of', 'default=noprint_wrappers=1:nokey=1',
str(filename),
]
ffprobe_from_stream_args = [
current_app.config['BIN_FFPROBE'],
'-v', 'error',
'-hide_banner',
'-select_streams', 'v:0', # we only care about the first video stream
'-show_entries', 'stream=duration',
'-of', 'default=noprint_wrappers=1:nokey=1',
str(filename),
]
duration = run(ffprobe_from_stream_args) or\
run(ffprobe_from_container_args) or\
None
return duration
def _video_size_pixels(filename: pathlib.Path) -> typing.Tuple[int, int]:
"""Figures out the size (in pixels) of the video file.
@ -220,8 +281,10 @@ def _process_video(gcs,
# by determining the video size here we already have this information in the file
# document before Zencoder calls our notification URL. It also opens up possibilities
# for other encoding backends that don't support this functionality.
video_width, video_height = _video_size_pixels(pathlib.Path(local_file.name))
video_path = pathlib.Path(local_file.name)
video_width, video_height = _video_size_pixels(video_path)
capped_video_width, capped_video_height = _video_cap_at_1080(video_width, video_height)
video_duration = _video_duration_seconds(video_path)
# Create variations
root, _ = os.path.splitext(src_file['file_path'])
@ -234,12 +297,13 @@ def _process_video(gcs,
content_type='video/{}'.format(v),
file_path='{}-{}.{}'.format(root, v, v),
size='',
duration=0,
width=capped_video_width,
height=capped_video_height,
length=0,
md5='',
)
if video_duration:
file_variation['duration'] = video_duration
# Append file variation. Originally mp4 and webm were the available options,
# that's why we build a list.
src_file['variations'].append(file_variation)

1
pillar/api/latest.py
View File

@ -70,6 +70,7 @@ def latest_assets():
{'name': 1, 'node_type': 1,
'parent': 1, 'picture': 1, 'properties.status': 1,
'properties.content_type': 1,
'properties.duration_seconds': 1,
'permissions.world': 1},
12)

5
pillar/api/node_types/asset.py
View File

@ -24,6 +24,10 @@ node_type_asset = {
'content_type': {
'type': 'string'
},
# The duration of a video asset in seconds.
'duration_seconds': {
'type': 'integer'
},
# We point to the original file (and use it to extract any relevant
# variation useful for our scope).
'file': _file_embedded_schema,
@ -58,6 +62,7 @@ node_type_asset = {
},
'form_schema': {
'content_type': {'visible': False},
'duration_seconds': {'visible': False},
'order': {'visible': False},
'tags': {'visible': False},
'categories': {'visible': False},

20
pillar/api/nodes/__init__.py
View File

@ -66,8 +66,8 @@ def tagged(tag=''):
agg_list = _tagged(tag)
for node in agg_list:
if node.get('video_duration_seconds'):
node['video_duration'] = datetime.timedelta(seconds=node['video_duration_seconds'])
if node['properties'].get('duration_seconds'):
node['properties']['duration'] = datetime.timedelta(seconds=node['properties']['duration_seconds'])
if node.get('_created') is not None:
node['pretty_created'] = pretty_date(node['_created'])
@ -108,26 +108,16 @@ def _tagged(tag: str):
'foreignField': '_id',
'as': '_project',
}},
{'$lookup': {
'from': 'files',
'localField': 'properties.file',
'foreignField': '_id',
'as': '_file',
}},
{'$unwind': '$_file'},
{'$unwind': '$_project'},
{'$match': {'_project.is_private': False}},
{'$addFields': {
'project._id': '$_project._id',
'project.name': '$_project.name',
'project.url': '$_project.url',
'video_duration_seconds': {'$arrayElemAt': ['$_file.variations.duration', 0]},
}},
# Don't return the entire project/file for each node.
{'$project': {'_project': False,
'_file': False}
},
{'$project': {'_project': False}},
{'$sort': {'_created': -1}}
])
@ -224,13 +214,13 @@ def setup_app(app, url_prefix):
app.on_replace_nodes += eve_hooks.before_replacing_node
app.on_replace_nodes += eve_hooks.parse_markdown
app.on_replace_nodes += eve_hooks.texture_sort_files
app.on_replace_nodes += eve_hooks.deduct_content_type
app.on_replace_nodes += eve_hooks.deduct_content_type_and_duration
app.on_replace_nodes += eve_hooks.node_set_default_picture
app.on_replaced_nodes += eve_hooks.after_replacing_node
app.on_insert_nodes += eve_hooks.before_inserting_nodes
app.on_insert_nodes += eve_hooks.parse_markdowns
app.on_insert_nodes += eve_hooks.nodes_deduct_content_type
app.on_insert_nodes += eve_hooks.nodes_deduct_content_type_and_duration
app.on_insert_nodes += eve_hooks.nodes_set_default_picture
app.on_insert_nodes += eve_hooks.textures_sort_files
app.on_inserted_nodes += eve_hooks.after_inserting_nodes

16
pillar/api/nodes/eve_hooks.py
View File

@ -165,7 +165,7 @@ def after_inserting_nodes(items):
)
def deduct_content_type(node_doc, original=None):
def deduct_content_type_and_duration(node_doc, original=None):
"""Deduct the content type from the attached file, if any."""
if node_doc['node_type'] != 'asset':
@ -184,7 +184,8 @@ def deduct_content_type(node_doc, original=None):
files = current_app.data.driver.db['files']
file_doc = files.find_one({'_id': file_id},
{'content_type': 1})
{'content_type': 1,
'variations': 1})
if not file_doc:
log.warning('deduct_content_type: Node %s refers to non-existing file %s, rejecting.',
node_id, file_id)
@ -201,10 +202,17 @@ def deduct_content_type(node_doc, original=None):
node_doc['properties']['content_type'] = content_type
if content_type == 'video':
duration = file_doc['variations'][0].get('duration')
if duration:
node_doc['properties']['duration_seconds'] = duration
else:
log.warning('Video file %s has no duration', file_id)
def nodes_deduct_content_type(nodes):
def nodes_deduct_content_type_and_duration(nodes):
for node in nodes:
deduct_content_type(node)
deduct_content_type_and_duration(node)
def node_set_default_picture(node, original=None):

19
pillar/api/utils/__init__.py
View File

@ -57,6 +57,18 @@ def remove_private_keys(document):
return doc_copy
def pretty_duration(seconds):
if seconds is None:
return ''
seconds = round(seconds)
hours, seconds = divmod(seconds, 3600)
minutes, seconds = divmod(seconds, 60)
if hours > 0:
return f'{hours:02}:{minutes:02}:{seconds:02}'
else:
return f'{minutes:02}:{seconds:02}'
class PillarJSONEncoder(json.JSONEncoder):
"""JSON encoder with support for Pillar resources."""
@ -65,12 +77,7 @@ class PillarJSONEncoder(json.JSONEncoder):
return obj.strftime(RFC1123_DATE_FORMAT)
if isinstance(obj, datetime.timedelta):
hours, seconds = divmod(obj.seconds, 3600)
minutes, seconds = divmod(seconds, 60)
if hours > 0:
return f'{hours:02}:{minutes:02}:{seconds:02}'
else:
return f'{minutes:02}:{seconds:02}'
return pretty_duration(obj.total_seconds())
if isinstance(obj, bson.ObjectId):
return str(obj)

153
pillar/cli/maintenance.py
View File

@ -1022,3 +1022,156 @@ def delete_orphan_files():
log.warning('Soft-deletion modified %d of %d files', res.modified_count, file_count)
log.info('%d files have been soft-deleted', res.modified_count)
@manager_maintenance.command
def find_video_files_without_duration():
"""Finds video files without any duration
This is a heavy operation. Use with care.
"""
from pathlib import Path
output_fpath = Path(current_app.config['STORAGE_DIR']) / 'video_files_without_duration.txt'
if output_fpath.exists():
log.error('Output filename %s already exists, remove it first.', output_fpath)
return 1
start_timestamp = datetime.datetime.now()
files_coll = current_app.db('files')
starts_with_video = re.compile("^video", re.IGNORECASE)
aggr = files_coll.aggregate([
{'$match': {'content_type': starts_with_video,
'_deleted': {'$ne': True}}},
{'$unwind': '$variations'},
{'$match': {
'variations.duration': {'$not': {'$gt': 0}}
}},
{'$project': {'_id': 1}}
])
file_ids = [str(f['_id']) for f in aggr]
nbr_files = len(file_ids)
log.info('Total nbr video files without duration: %d', nbr_files)
end_timestamp = datetime.datetime.now()
duration = end_timestamp - start_timestamp
log.info('Finding files took %s', duration)
log.info('Writing Object IDs to %s', output_fpath)
with output_fpath.open('w', encoding='ascii') as outfile:
outfile.write('\n'.join(sorted(file_ids)))
@manager_maintenance.command
def find_video_nodes_without_duration():
"""Finds video nodes without any duration
This is a heavy operation. Use with care.
"""
from pathlib import Path
output_fpath = Path(current_app.config['STORAGE_DIR']) / 'video_nodes_without_duration.txt'
if output_fpath.exists():
log.error('Output filename %s already exists, remove it first.', output_fpath)
return 1
start_timestamp = datetime.datetime.now()
nodes_coll = current_app.db('nodes')
aggr = nodes_coll.aggregate([
{'$match': {'node_type': 'asset',
'properties.content_type': 'video',
'_deleted': {'$ne': True},
'properties.duration_seconds': {'$not': {'$gt': 0}}}},
{'$project': {'_id': 1}}
])
file_ids = [str(f['_id']) for f in aggr]
nbr_files = len(file_ids)
log.info('Total nbr video nodes without duration: %d', nbr_files)
end_timestamp = datetime.datetime.now()
duration = end_timestamp - start_timestamp
log.info('Finding nodes took %s', duration)
log.info('Writing Object IDs to %s', output_fpath)
with output_fpath.open('w', encoding='ascii') as outfile:
outfile.write('\n'.join(sorted(file_ids)))
@manager_maintenance.option('-n', '--nodes', dest='nodes_to_update', nargs='*',
help='List of nodes to update')
@manager_maintenance.option('-a', '--all', dest='all_nodes', action='store_true', default=False,
help='Update on all video nodes.')
@manager_maintenance.option('-g', '--go', dest='go', action='store_true', default=False,
help='Actually perform the changes (otherwise just show as dry-run).')
def reconcile_node_video_duration(nodes_to_update=None, all_nodes=False, go=False):
"""Copy video duration from file.variations.duration to node.properties.duraion_seconds
This is a heavy operation. Use with care.
"""
from pillar.api.utils import random_etag, utcnow
if bool(nodes_to_update) == all_nodes:
log.error('Use either --nodes or --all.')
return 1
start_timestamp = datetime.datetime.now()
nodes_coll = current_app.db('nodes')
node_subset = []
if nodes_to_update:
node_subset = [{'$match': {'_id': {'$in': [ObjectId(nid) for nid in nodes_to_update]}}}]
files = nodes_coll.aggregate(
[
*node_subset,
{'$match': {
'node_type': 'asset',
'properties.content_type': 'video',
'_deleted': {'$ne': True}}
},
{'$lookup': {
'from': 'files',
'localField': 'properties.file',
'foreignField': '_id',
'as': '_files',
}},
{'$unwind': '$_files'},
{'$unwind': '$_files.variations'},
{'$match': {'_files.variations.duration': {'$gt': 0}}},
{'$addFields': {
'need_update': {'$ne': ['$_files.variations.duration', '$properties.duration_seconds']}
}},
{'$match': {'need_update': True}},
{'$project': {
'_id': 1,
'duration': '$_files.variations.duration',
}}]
)
if not go:
log.info('Would try to update %d nodes', len(list(files)))
return 0
modified_count = 0
for f in files:
log.debug('Updating node %s with duration %d', f['_id'], f['duration'])
new_etag = random_etag()
now = utcnow()
resp = nodes_coll.update_one(
{'_id': f['_id']},
{'$set': {
'properties.duration_seconds': f['duration'],
'_etag': new_etag,
'_updated': now,
}}
)
if resp.modified_count == 0:
log.debug('Node %s was already up to date', f['_id'])
modified_count += resp.modified_count
log.info('Updated %d nodes', modified_count)
end_timestamp = datetime.datetime.now()
duration = end_timestamp - start_timestamp
log.info('Operation took %s', duration)
return 0

6
pillar/web/jinja.py
View File

@ -13,6 +13,7 @@ import werkzeug.exceptions as wz_exceptions
import pillarsdk
import pillar.api.utils
from pillar.api.utils import pretty_duration
from pillar.web.utils import pretty_date
from pillar.web.nodes.routes import url_for_node
import pillar.markdown
@ -28,6 +29,10 @@ def format_pretty_date_time(d):
return pretty_date(d, detail=True)
def format_pretty_duration(s):
return pretty_duration(s)
def format_undertitle(s):
"""Underscore-replacing title filter.
@ -203,6 +208,7 @@ def do_yesno(value, arg=None):
def setup_jinja_env(jinja_env, app_config: dict):
jinja_env.filters['pretty_date'] = format_pretty_date
jinja_env.filters['pretty_date_time'] = format_pretty_date_time
jinja_env.filters['pretty_duration'] = format_pretty_duration
jinja_env.filters['undertitle'] = format_undertitle
jinja_env.filters['hide_none'] = do_hide_none
jinja_env.filters['pluralize'] = do_pluralize

1
pillar/web/projects/routes.py
View File

@ -361,6 +361,7 @@ def render_project(project, api, extra_context=None, template_name=None):
# Construct query parameters outside the loop.
projection = {'name': 1, 'user': 1, 'node_type': 1, 'project': 1,
'properties.url': 1, 'properties.content_type': 1,
'properties.duration_seconds': 1,
'picture': 1}
params = {'projection': projection, 'embedded': {'user': 1}}

3
src/templates/_macros/_asset_list_item.pug
View File

@ -37,6 +37,9 @@ a.card.asset.card-image-fade.pr-0.mx-0.mb-2(
.card-label WATCHED
| {% endif %}
| {% endif %} {# endif progress #}
| {% if asset.properties.duration_seconds %}
.card-label.right {{ asset.properties.duration_seconds | pretty_duration }}
| {% endif %}
| {% endif %} {# endif video #}

4
src/templates/nodes/view_base.pug
View File

@ -62,6 +62,10 @@ section.node-details-meta.pl-4.pr-2.py-2.border-bottom
li.ml-auto
| {% if node.properties.duration_seconds %}
li.px-2(title="Duration")
| {{ node.properties.duration_seconds | pretty_duration }}
| {% endif %}
| {% if node.file %}
li.px-2(title="File size")
| {{ node.file.length | filesizeformat }}

20
tests/test_api/test_file_storage.py
View File

@ -280,3 +280,23 @@ class VideoSizeTest(AbstractPillarTest):
size = file_storage._video_cap_at_1080(2048, 2000)
self.assertIsInstance(size[0], int)
self.assertIsInstance(size[1], int)
class VideoDurationTest(AbstractPillarTest):
def test_video_duration_from_container(self):
from pillar.api import file_storage
from pathlib import Path
with self.app.test_request_context():
fname = Path(__file__).with_name('video-tiny.mkv')
self.assertEqual(1, file_storage._video_duration_seconds(fname))
def test_video_duration_from_stream(self):
from pillar.api import file_storage
from pathlib import Path
with self.app.test_request_context():
fname = Path(__file__).with_name('video-tiny.mp4')
self.assertEqual(2, file_storage._video_duration_seconds(fname))

303
tests/test_api/test_latest.py Normal file
View File

@ -0,0 +1,303 @@
from datetime import timedelta
import flask
from pillar.tests import AbstractPillarTest
class LatestAssetsTest(AbstractPillarTest):
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.private_pid, _ = self.ensure_project_exists(project_overrides={
'_id': '5672beecc0261b2005ed1a34',
'is_private': True,
})
self.file_id, _ = self.ensure_file_exists(file_overrides={
'variations': [
{'format': 'mp4',
'duration': 3661 # 01:01:01
},
],
})
self.uid = self.create_user()
from pillar.api.utils import utcnow
self.fake_now = utcnow()
def test_latest_assets_returns_12_newest_assets(self):
base_node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
}
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
def create_asset(weeks):
return self.create_node({
**base_node,
'_created': self.fake_now - timedelta(weeks=weeks),
'properties': base_props,
})
all_asset_ids = [str(create_asset(i)) for i in range(20)]
expected_ids = all_asset_ids[:12] # The 12 newest assets are expected
with self.app.app_context():
url = flask.url_for('latest.latest_assets')
latest_assets = self.get(url).json['_items']
actual_ids = [asset['_id'] for asset in latest_assets]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_assets_ignore(self):
base_node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
}
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Private should be ignored
self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'project': self.private_pid,
})
# Deleted should be ignored
self.create_node({
**base_node,
'_deleted': True,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Node type comment should be ignored
self.create_node({
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
})
with self.app.app_context():
url = flask.url_for('latest.latest_assets')
latest_assets = self.get(url).json['_items']
expected_ids = [str(ok_id)]
actual_ids = [asset['_id'] for asset in latest_assets]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_assets_data(self):
base_node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
}
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
with self.app.app_context():
url = flask.url_for('latest.latest_assets')
latest_assets = self.get(url).json['_items']
asset = latest_assets[0]
self.assertEquals(str(ok_id), asset['_id'])
self.assertEquals('Just a node name', asset['name'])
class LatestCommentsTest(AbstractPillarTest):
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.private_pid, _ = self.ensure_project_exists(project_overrides={
'_id': '5672beecc0261b2005ed1a34',
'is_private': True,
})
self.file_id, _ = self.ensure_file_exists(file_overrides={
'variations': [
{'format': 'mp4',
'duration': 3661 # 01:01:01
},
],
})
self.uid = self.create_user()
from pillar.api.utils import utcnow
self.fake_now = utcnow()
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
self.asset_node_id = self.create_node({
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
'_created': self.fake_now - timedelta(weeks=52),
'properties': base_props,
})
def test_latest_comments_returns_10_newest_comments(self):
base_node = {
'name': 'Comment',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
'parent': self.asset_node_id,
}
base_props = {
'status': 'published',
'content': 'एनिमेशन is animation in Hindi',
}
def create_comment(weeks):
return self.create_node({
**base_node,
'_created': self.fake_now - timedelta(weeks=weeks),
'properties': base_props,
})
all_comment_ids = [str(create_comment(i)) for i in range(20)]
expected_ids = all_comment_ids[:10] # The 10 newest comments are expected
with self.app.app_context():
url = flask.url_for('latest.latest_comments')
latest_assets = self.get(url).json['_items']
actual_ids = [asset['_id'] for asset in latest_assets]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_comments_ignore(self):
base_node = {
'name': 'Comment',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
'parent': self.asset_node_id,
}
base_props = {
'status': 'published',
'content': 'एनिमेशन is animation in Hindi',
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Private should be ignored
self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'project': self.private_pid,
})
# Deleted should be ignored
self.create_node({
**base_node,
'_deleted': True,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Node type asset should be ignored
self.create_node({
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
})
with self.app.app_context():
url = flask.url_for('latest.latest_comments')
latest_comments = self.get(url).json['_items']
expected_ids = [str(ok_id)]
actual_ids = [comment['_id'] for comment in latest_comments]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_comments_data(self):
base_node = {
'name': 'Comment',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
'parent': self.asset_node_id,
}
base_props = {
'status': 'published',
'content': 'एनिमेशन is animation in Hindi',
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
with self.app.app_context():
url = flask.url_for('latest.latest_comments')
latest_comments = self.get(url).json['_items']
comment = latest_comments[0]
self.assertEquals(str(ok_id), comment['_id'])
self.assertEquals('Comment', comment['name'])
self.assertEquals('एनिमेशन is animation in Hindi', comment['properties']['content'])

3
tests/test_api/test_nodes.py
View File

@ -654,6 +654,7 @@ class TaggedNodesTest(AbstractPillarTest):
base_props = {'status': 'published',
'file': self.file_id,
'content_type': 'video',
'duration_seconds': 3661, # 01:01:01
'order': 0}
self.create_node({
@ -668,7 +669,7 @@ class TaggedNodesTest(AbstractPillarTest):
mock_utcnow.return_value = self.fake_now
url = flask.url_for('nodes_api.tagged', tag='एनिमेशन')
resp = self.get(url).json[0]
self.assertEquals('01:01:01', resp['video_duration'])
self.assertEquals('01:01:01', resp['properties']['duration'])
self.assertEquals('Unittest project', resp['project']['name'])
self.assertEquals('default-project', resp['project']['url'])
self.assertEquals('5m ago', resp['pretty_created'])

14
tests/test_api/test_utils.py
View File

@ -212,3 +212,17 @@ class TestRating(unittest.TestCase):
sorted_by_hot = sorted(cases, key=lambda tup: tup[0])
for idx, t in enumerate(sorted_by_hot):
self.assertEqual(cases[idx][0], t[0])
class TestPrettyDuration(unittest.TestCase):
def test_formatting(self):
from pillar.api.utils import pretty_duration
pretty_duration(500)
self.assertEquals('00:00', pretty_duration(0))
self.assertEquals('00:15', pretty_duration(15))
self.assertEquals('01:05', pretty_duration(65))
self.assertEquals('42:53', pretty_duration(2573))
self.assertEquals('01:11:22', pretty_duration(4282))
self.assertEquals('01:41', pretty_duration(100.85))
self.assertEquals('25:00:00', pretty_duration(90000)) # More than a day
self.assertEquals('', pretty_duration(None))

BIN
tests/test_api/video-tiny.mp4 Normal file
View File

Binary file not shown.

149
tests/test_cli/test_maintenance.py
View File

@ -1,5 +1,8 @@
from unittest import mock
from bson import ObjectId
from pillar.api.utils import random_etag, utcnow
from pillar.tests import AbstractPillarTest
from pillar.tests import common_test_data as ctd
@ -250,3 +253,149 @@ class UpgradeAttachmentUsageTest(AbstractPillarTest):
},
node['properties']['attachments'],
'The link should have been removed from the attachment')
class ReconcileNodeDurationTest(AbstractPillarTest):
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.fake_now = utcnow()
# Already correct. Should not be touched.
self.node_id0 = self._create_video_node(file_duration=123, node_duration=123)
# Out of sync, should be updated
self.node_id1 = self._create_video_node(file_duration=3661, node_duration=15)
self.node_id2 = self._create_video_node(file_duration=432, node_duration=5)
self.node_id3 = self._create_video_node(file_duration=222)
# No file duration. Should not be touched
self.node_id4 = self._create_video_node()
# No file. Should not be touched
self.node_id5 = self._create_video_node(include_file=False)
# Wrong node type. Should not be touched
self.image_node_id = self._create_image_node()
def id_to_original_dict(*nids):
with self.app.app_context():
nodes_coll = self.app.db('nodes')
return dict(((nid, nodes_coll.find_one({'_id': nid})) for nid in nids))
self.orig_nodes = id_to_original_dict(
self.node_id0,
self.node_id1,
self.node_id2,
self.node_id3,
self.node_id4,
self.node_id5,
self.image_node_id,
)
def test_reconcile_all(self):
from pillar.cli.maintenance import reconcile_node_video_duration
with self.app.app_context():
with mock.patch('pillar.api.utils.utcnow') as mock_utcnow:
mock_utcnow.return_value = self.fake_now
reconcile_node_video_duration(all_nodes=True, go=False) # Dry run
self.assertAllUnchanged()
reconcile_node_video_duration(all_nodes=True, go=True)
self.assertUnChanged(
self.node_id0,
self.node_id4,
self.image_node_id,
)
self.assertUpdated(self.node_id1, duration_seconds=3661)
self.assertUpdated(self.node_id2, duration_seconds=432)
self.assertUpdated(self.node_id3, duration_seconds=222)
def test_reconcile_some(self):
from pillar.cli.maintenance import reconcile_node_video_duration
with self.app.app_context():
with mock.patch('pillar.api.utils.utcnow') as mock_utcnow:
mock_utcnow.return_value = self.fake_now
to_reconcile = [str(self.node_id0), str(self.node_id1), str(self.node_id2), str(self.node_id5)]
reconcile_node_video_duration(nodes_to_update=to_reconcile, go=False) # Dry run
self.assertAllUnchanged()
reconcile_node_video_duration(nodes_to_update=to_reconcile, go=True)
self.assertUnChanged(
self.node_id0,
self.node_id3,
self.node_id4,
self.image_node_id,
)
self.assertUpdated(self.node_id1, duration_seconds=3661)
self.assertUpdated(self.node_id2, duration_seconds=432)
def assertUpdated(self, nid, duration_seconds):
nodes_coll = self.app.db('nodes')
new_node = nodes_coll.find_one({'_id': nid})
orig_node = self.orig_nodes[nid]
self.assertNotEqual(orig_node['_etag'], new_node['_etag'])
self.assertEquals(self.fake_now, new_node['_updated'])
self.assertEquals(duration_seconds, new_node['properties']['duration_seconds'])
def assertAllUnchanged(self):
self.assertUnChanged(*self.orig_nodes.keys())
def assertUnChanged(self, *node_ids):
nodes_coll = self.app.db('nodes')
for nid in node_ids:
new_node = nodes_coll.find_one({'_id': nid})
orig_node = self.orig_nodes[nid]
self.assertEquals(orig_node, new_node)
def _create_video_node(self, file_duration=None, node_duration=None, include_file=True):
file_id, _ = self.ensure_file_exists(file_overrides={
'_id': ObjectId(),
'content_type': 'video/mp4',
'variations': [
{'format': 'mp4',
'duration': file_duration
},
],
})
node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'_etag': random_etag(),
}
props = {'status': 'published',
'content_type': 'video',
'order': 0}
if node_duration is not None:
props['duration_seconds'] = node_duration
if include_file:
props['file'] = file_id
return self.create_node({
'properties': props,
**node})
def _create_image_node(self):
file_id, _ = self.ensure_file_exists(file_overrides={
'_id': ObjectId(),
'variations': [
{'format': 'jpeg'},
],
})
node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'_etag': random_etag(),
}
props = {'status': 'published',
'file': file_id,
'content_type': 'image',
'order': 0}
return self.create_node({
'properties': props,
**node})

32
tests/test_web/test_jstree.py
View File

@ -44,6 +44,38 @@ class JSTreeTest(AbstractPillarTest):
'custom_view': False,
})
def test_jstree_parse_video_node(self):
from pillar.web.utils.jstree import jstree_parse_node
node_doc = {'_id': ObjectId('55f338f92beb3300c4ff99fe'),
'_created': parse('2015-09-11T22:26:33.000+0200'),
'_updated': parse('2015-10-30T22:44:27.000+0100'),
'_etag': '5248485b4ea7e55e858ff84b1bd4aae88917a37c',
'picture': ObjectId('55f338f92beb3300c4ff99de'),
'description': 'Play the full movie and see how it was cobbled together.',
'parent': ObjectId('55f338f92beb3300c4ff99f9'),
'project': self.project_id,
'node_type': 'asset',
'user': ObjectId('552b066b41acdf5dec4436f2'),
'properties': {'status': 'published',
'file': ObjectId('55f338f92beb3300c4ff99c2'),
'content_type': 'video',
},
'name': 'Live <strong>Edit</strong>'}
with self.app.test_request_context():
parsed = jstree_parse_node(Node(node_doc))
self.assertEqual(parsed, {
'id': 'n_55f338f92beb3300c4ff99fe',
'a_attr': {'href': f"/p/{self.project['url']}/55f338f92beb3300c4ff99fe"},
'li_attr': {'data-node-type': 'asset'},
'text': Markup('Live &lt;strong&gt;Edit&lt;/strong&gt;'),
'type': 'video',
'children': False,
'custom_view': False,
})
def test_jstree_parse_blog_node(self):
from pillar.web.utils.jstree import jstree_parse_node