archive/pillar
0
0
Fork 0
Code Issues 42 Pull Requests Projects Releases Wiki Activity

Video Duration: The duration of a video is now shown on thumbnails and bellow the video player

Browse Source 
Asset nodes now have a new field called "properties.duration_seconds". This holds a copy of the duration stored on the referenced video file and stays in sync using eve hooks.

To migrate existing duration times from files to nodes you need to run the following:
./manage.py maintenance reconcile_node_video_duration -ag

There are 2 more maintenance commands to be used to determine if there are any missing durations in either files or nodes:
find_video_files_without_duration
find_video_nodes_without_duration

FFProbe is now used to detect what duration a video file has.

Reviewed by Sybren.
This commit is contained in:
Tobias Johansson
2018-10-03 18:30:40 +02:00
parent a738cdcad8
commit 6ad12d0098
18 changed files with 789 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
tests/test_api/test_file_storage.py
View File

@@ -280,3 +280,23 @@ class VideoSizeTest(AbstractPillarTest):
size = file_storage._video_cap_at_1080(2048, 2000)
self.assertIsInstance(size[0], int)
self.assertIsInstance(size[1], int)
class VideoDurationTest(AbstractPillarTest):
def test_video_duration_from_container(self):
from pillar.api import file_storage
from pathlib import Path
with self.app.test_request_context():
fname = Path(__file__).with_name('video-tiny.mkv')
self.assertEqual(1, file_storage._video_duration_seconds(fname))
def test_video_duration_from_stream(self):
from pillar.api import file_storage
from pathlib import Path
with self.app.test_request_context():
fname = Path(__file__).with_name('video-tiny.mp4')
self.assertEqual(2, file_storage._video_duration_seconds(fname))

303
tests/test_api/test_latest.py Normal file
View File

@@ -0,0 +1,303 @@
from datetime import timedelta
import flask
from pillar.tests import AbstractPillarTest
class LatestAssetsTest(AbstractPillarTest):
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.private_pid, _ = self.ensure_project_exists(project_overrides={
'_id': '5672beecc0261b2005ed1a34',
'is_private': True,
})
self.file_id, _ = self.ensure_file_exists(file_overrides={
'variations': [
{'format': 'mp4',
'duration': 3661 # 01:01:01
},
],
})
self.uid = self.create_user()
from pillar.api.utils import utcnow
self.fake_now = utcnow()
def test_latest_assets_returns_12_newest_assets(self):
base_node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
}
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
def create_asset(weeks):
return self.create_node({
**base_node,
'_created': self.fake_now - timedelta(weeks=weeks),
'properties': base_props,
})
all_asset_ids = [str(create_asset(i)) for i in range(20)]
expected_ids = all_asset_ids[:12] # The 12 newest assets are expected
with self.app.app_context():
url = flask.url_for('latest.latest_assets')
latest_assets = self.get(url).json['_items']
actual_ids = [asset['_id'] for asset in latest_assets]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_assets_ignore(self):
base_node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
}
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Private should be ignored
self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'project': self.private_pid,
})
# Deleted should be ignored
self.create_node({
**base_node,
'_deleted': True,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Node type comment should be ignored
self.create_node({
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
})
with self.app.app_context():
url = flask.url_for('latest.latest_assets')
latest_assets = self.get(url).json['_items']
expected_ids = [str(ok_id)]
actual_ids = [asset['_id'] for asset in latest_assets]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_assets_data(self):
base_node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
}
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
with self.app.app_context():
url = flask.url_for('latest.latest_assets')
latest_assets = self.get(url).json['_items']
asset = latest_assets[0]
self.assertEquals(str(ok_id), asset['_id'])
self.assertEquals('Just a node name', asset['name'])
class LatestCommentsTest(AbstractPillarTest):
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.private_pid, _ = self.ensure_project_exists(project_overrides={
'_id': '5672beecc0261b2005ed1a34',
'is_private': True,
})
self.file_id, _ = self.ensure_file_exists(file_overrides={
'variations': [
{'format': 'mp4',
'duration': 3661 # 01:01:01
},
],
})
self.uid = self.create_user()
from pillar.api.utils import utcnow
self.fake_now = utcnow()
base_props = {
'status': 'published',
'file': self.file_id,
'content_type': 'video',
'order': 0
}
self.asset_node_id = self.create_node({
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
'_created': self.fake_now - timedelta(weeks=52),
'properties': base_props,
})
def test_latest_comments_returns_10_newest_comments(self):
base_node = {
'name': 'Comment',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
'parent': self.asset_node_id,
}
base_props = {
'status': 'published',
'content': 'एनिमेशन is animation in Hindi',
}
def create_comment(weeks):
return self.create_node({
**base_node,
'_created': self.fake_now - timedelta(weeks=weeks),
'properties': base_props,
})
all_comment_ids = [str(create_comment(i)) for i in range(20)]
expected_ids = all_comment_ids[:10] # The 10 newest comments are expected
with self.app.app_context():
url = flask.url_for('latest.latest_comments')
latest_assets = self.get(url).json['_items']
actual_ids = [asset['_id'] for asset in latest_assets]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_comments_ignore(self):
base_node = {
'name': 'Comment',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
'parent': self.asset_node_id,
}
base_props = {
'status': 'published',
'content': 'एनिमेशन is animation in Hindi',
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Private should be ignored
self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'project': self.private_pid,
})
# Deleted should be ignored
self.create_node({
**base_node,
'_deleted': True,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
# Node type asset should be ignored
self.create_node({
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'user': self.uid,
})
with self.app.app_context():
url = flask.url_for('latest.latest_comments')
latest_comments = self.get(url).json['_items']
expected_ids = [str(ok_id)]
actual_ids = [comment['_id'] for comment in latest_comments]
self.assertListEqual(
expected_ids, actual_ids)
def test_latest_comments_data(self):
base_node = {
'name': 'Comment',
'project': self.pid,
'description': '',
'node_type': 'comment',
'user': self.uid,
'parent': self.asset_node_id,
}
base_props = {
'status': 'published',
'content': 'एनिमेशन is animation in Hindi',
}
ok_id = self.create_node({
**base_node,
'_created': self.fake_now - timedelta(seconds=1),
'properties': base_props,
})
with self.app.app_context():
url = flask.url_for('latest.latest_comments')
latest_comments = self.get(url).json['_items']
comment = latest_comments[0]
self.assertEquals(str(ok_id), comment['_id'])
self.assertEquals('Comment', comment['name'])
self.assertEquals('एनिमेशन is animation in Hindi', comment['properties']['content'])

3
tests/test_api/test_nodes.py
View File

@@ -654,6 +654,7 @@ class TaggedNodesTest(AbstractPillarTest):
base_props = {'status': 'published',
'file': self.file_id,
'content_type': 'video',
'duration_seconds': 3661, # 01:01:01
'order': 0}
self.create_node({
@@ -668,7 +669,7 @@ class TaggedNodesTest(AbstractPillarTest):
mock_utcnow.return_value = self.fake_now
url = flask.url_for('nodes_api.tagged', tag='एनिमेशन')
resp = self.get(url).json[0]
self.assertEquals('01:01:01', resp['video_duration'])
self.assertEquals('01:01:01', resp['properties']['duration'])
self.assertEquals('Unittest project', resp['project']['name'])
self.assertEquals('default-project', resp['project']['url'])
self.assertEquals('5m ago', resp['pretty_created'])

14
tests/test_api/test_utils.py
View File

@@ -212,3 +212,17 @@ class TestRating(unittest.TestCase):
sorted_by_hot = sorted(cases, key=lambda tup: tup[0])
for idx, t in enumerate(sorted_by_hot):
self.assertEqual(cases[idx][0], t[0])
class TestPrettyDuration(unittest.TestCase):
def test_formatting(self):
from pillar.api.utils import pretty_duration
pretty_duration(500)
self.assertEquals('00:00', pretty_duration(0))
self.assertEquals('00:15', pretty_duration(15))
self.assertEquals('01:05', pretty_duration(65))
self.assertEquals('42:53', pretty_duration(2573))
self.assertEquals('01:11:22', pretty_duration(4282))
self.assertEquals('01:41', pretty_duration(100.85))
self.assertEquals('25:00:00', pretty_duration(90000)) # More than a day
self.assertEquals('', pretty_duration(None))

BIN
tests/test_api/video-tiny.mp4 Normal file
View File

Binary file not shown.

149
tests/test_cli/test_maintenance.py
View File

@@ -1,5 +1,8 @@
from unittest import mock
from bson import ObjectId
from pillar.api.utils import random_etag, utcnow
from pillar.tests import AbstractPillarTest
from pillar.tests import common_test_data as ctd
@@ -250,3 +253,149 @@ class UpgradeAttachmentUsageTest(AbstractPillarTest):
},
node['properties']['attachments'],
'The link should have been removed from the attachment')
class ReconcileNodeDurationTest(AbstractPillarTest):
def setUp(self, **kwargs):
super().setUp(**kwargs)
self.pid, _ = self.ensure_project_exists()
self.fake_now = utcnow()
# Already correct. Should not be touched.
self.node_id0 = self._create_video_node(file_duration=123, node_duration=123)
# Out of sync, should be updated
self.node_id1 = self._create_video_node(file_duration=3661, node_duration=15)
self.node_id2 = self._create_video_node(file_duration=432, node_duration=5)
self.node_id3 = self._create_video_node(file_duration=222)
# No file duration. Should not be touched
self.node_id4 = self._create_video_node()
# No file. Should not be touched
self.node_id5 = self._create_video_node(include_file=False)
# Wrong node type. Should not be touched
self.image_node_id = self._create_image_node()
def id_to_original_dict(*nids):
with self.app.app_context():
nodes_coll = self.app.db('nodes')
return dict(((nid, nodes_coll.find_one({'_id': nid})) for nid in nids))
self.orig_nodes = id_to_original_dict(
self.node_id0,
self.node_id1,
self.node_id2,
self.node_id3,
self.node_id4,
self.node_id5,
self.image_node_id,
)
def test_reconcile_all(self):
from pillar.cli.maintenance import reconcile_node_video_duration
with self.app.app_context():
with mock.patch('pillar.api.utils.utcnow') as mock_utcnow:
mock_utcnow.return_value = self.fake_now
reconcile_node_video_duration(all_nodes=True, go=False) # Dry run
self.assertAllUnchanged()
reconcile_node_video_duration(all_nodes=True, go=True)
self.assertUnChanged(
self.node_id0,
self.node_id4,
self.image_node_id,
)
self.assertUpdated(self.node_id1, duration_seconds=3661)
self.assertUpdated(self.node_id2, duration_seconds=432)
self.assertUpdated(self.node_id3, duration_seconds=222)
def test_reconcile_some(self):
from pillar.cli.maintenance import reconcile_node_video_duration
with self.app.app_context():
with mock.patch('pillar.api.utils.utcnow') as mock_utcnow:
mock_utcnow.return_value = self.fake_now
to_reconcile = [str(self.node_id0), str(self.node_id1), str(self.node_id2), str(self.node_id5)]
reconcile_node_video_duration(nodes_to_update=to_reconcile, go=False) # Dry run
self.assertAllUnchanged()
reconcile_node_video_duration(nodes_to_update=to_reconcile, go=True)
self.assertUnChanged(
self.node_id0,
self.node_id3,
self.node_id4,
self.image_node_id,
)
self.assertUpdated(self.node_id1, duration_seconds=3661)
self.assertUpdated(self.node_id2, duration_seconds=432)
def assertUpdated(self, nid, duration_seconds):
nodes_coll = self.app.db('nodes')
new_node = nodes_coll.find_one({'_id': nid})
orig_node = self.orig_nodes[nid]
self.assertNotEqual(orig_node['_etag'], new_node['_etag'])
self.assertEquals(self.fake_now, new_node['_updated'])
self.assertEquals(duration_seconds, new_node['properties']['duration_seconds'])
def assertAllUnchanged(self):
self.assertUnChanged(*self.orig_nodes.keys())
def assertUnChanged(self, *node_ids):
nodes_coll = self.app.db('nodes')
for nid in node_ids:
new_node = nodes_coll.find_one({'_id': nid})
orig_node = self.orig_nodes[nid]
self.assertEquals(orig_node, new_node)
def _create_video_node(self, file_duration=None, node_duration=None, include_file=True):
file_id, _ = self.ensure_file_exists(file_overrides={
'_id': ObjectId(),
'content_type': 'video/mp4',
'variations': [
{'format': 'mp4',
'duration': file_duration
},
],
})
node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'_etag': random_etag(),
}
props = {'status': 'published',
'content_type': 'video',
'order': 0}
if node_duration is not None:
props['duration_seconds'] = node_duration
if include_file:
props['file'] = file_id
return self.create_node({
'properties': props,
**node})
def _create_image_node(self):
file_id, _ = self.ensure_file_exists(file_overrides={
'_id': ObjectId(),
'variations': [
{'format': 'jpeg'},
],
})
node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'_etag': random_etag(),
}
props = {'status': 'published',
'file': file_id,
'content_type': 'image',
'order': 0}
return self.create_node({
'properties': props,
**node})

32
tests/test_web/test_jstree.py
View File

@@ -44,6 +44,38 @@ class JSTreeTest(AbstractPillarTest):
'custom_view': False,
})
def test_jstree_parse_video_node(self):
from pillar.web.utils.jstree import jstree_parse_node
node_doc = {'_id': ObjectId('55f338f92beb3300c4ff99fe'),
'_created': parse('2015-09-11T22:26:33.000+0200'),
'_updated': parse('2015-10-30T22:44:27.000+0100'),
'_etag': '5248485b4ea7e55e858ff84b1bd4aae88917a37c',
'picture': ObjectId('55f338f92beb3300c4ff99de'),
'description': 'Play the full movie and see how it was cobbled together.',
'parent': ObjectId('55f338f92beb3300c4ff99f9'),
'project': self.project_id,
'node_type': 'asset',
'user': ObjectId('552b066b41acdf5dec4436f2'),
'properties': {'status': 'published',
'file': ObjectId('55f338f92beb3300c4ff99c2'),
'content_type': 'video',
},
'name': 'Live <strong>Edit</strong>'}
with self.app.test_request_context():
parsed = jstree_parse_node(Node(node_doc))
self.assertEqual(parsed, {
'id': 'n_55f338f92beb3300c4ff99fe',
'a_attr': {'href': f"/p/{self.project['url']}/55f338f92beb3300c4ff99fe"},
'li_attr': {'data-node-type': 'asset'},
'text': Markup('Live &lt;strong&gt;Edit&lt;/strong&gt;'),
'type': 'video',
'children': False,
'custom_view': False,
})
def test_jstree_parse_blog_node(self):
from pillar.web.utils.jstree import jstree_parse_node