0
0
forked from blender/blender

VSE: Add sound strip retiming support #2

Closed
Richard Antalik wants to merge 1 commits from retiming-sound into retiming-tool

When changing the target branch, be careful to rebase the branch in your fork to match. See documentation.
22 changed files with 238 additions and 28 deletions

View File

@ -165,6 +165,12 @@ AUD_API void AUD_SequenceEntry_move(AUD_SequenceEntry* entry, double begin, doub
(*entry)->move(begin, end, skip);
}
AUD_API void AUD_SequenceEntry_setAnimationData_constant_range(AUD_SequenceEntry* entry, AUD_AnimateablePropertyType type, int frame_start, int frame_end, float* data)
{
AnimateableProperty* prop = (*entry)->getAnimProperty(static_cast<AnimateablePropertyType>(type));
prop->write_range(data, frame_start, frame_end);
}
AUD_API void AUD_SequenceEntry_setAnimationData(AUD_SequenceEntry* entry, AUD_AnimateablePropertyType type, int frame, float* data, char animated)
{
AnimateableProperty* prop = (*entry)->getAnimProperty(static_cast<AnimateablePropertyType>(type));

View File

@ -68,6 +68,16 @@ extern AUD_API void AUD_Sequence_remove(AUD_Sound* sequence, AUD_SequenceEntry*
* Writes animation data to a sequence.
* \param sequence The sound scene.
* \param type The type of animation data.
* \param frame_start Start of the frame range.
* \param frame_end End of the frame range.
* \param data The data to write.
*/
AUD_API void AUD_SequenceEntry_setAnimationData_constant_range(AUD_SequenceEntry* entry, AUD_AnimateablePropertyType type, int frame_start, int frame_end, float* data);
/**
* Writes animation data to a sequenced entry.
* \param entry The sequenced entry.
* \param type The type of animation data.
* \param frame The frame this data is for.
* \param data The data to write.
* \param animated Whether the attribute is animated.

View File

@ -112,6 +112,14 @@ public:
*/
void write(const float* data, int position, int count);
/**
* Fills the properties frame range with constant value and marks it animated.
* \param data The new value.
* \param position_start The start position in the animation in frames.
* \param position_end The end position in the animation in frames.
*/
void write_range(const float* data, int position_start, int position_end);
/**
* Reads the properties value.
* \param position The position in the animation in frames.

View File

@ -63,6 +63,9 @@ private:
/// How many seconds are skipped at the beginning.
double m_skip;
/// The FPS of the scene.
float m_fps;
/// Whether the entry is muted.
bool m_muted;
@ -124,7 +127,7 @@ public:
* \param skip How much seconds should be skipped at the beginning.
* \param id The ID of the entry.
*/
SequenceEntry(std::shared_ptr<ISound> sound, double begin, double end, double skip, int id);
SequenceEntry(std::shared_ptr<ISound> sound, double begin, double end, double skip, float fps, int id);
virtual ~SequenceEntry();
/**

View File

@ -65,6 +65,19 @@ void AnimateableProperty::write(const float* data)
std::memcpy(getBuffer(), data, m_count * sizeof(float));
}
void AnimateableProperty::write_range(const float* data, int position_start, int position_end)
{
assureSize(position_end * m_count * sizeof(float), true);
float* buf = getBuffer();
for(int i = position_start; i < position_end; i++)
{
std::memcpy(buf + i * m_count, data, m_count * sizeof(float));
}
m_isAnimated = true;
}
void AnimateableProperty::write(const float* data, int position, int count)
{
std::lock_guard<std::recursive_mutex> lock(m_mutex);

View File

@ -153,7 +153,7 @@ std::shared_ptr<SequenceEntry> SequenceData::add(std::shared_ptr<ISound> sound,
{
std::lock_guard<std::recursive_mutex> lock(m_mutex);
std::shared_ptr<SequenceEntry> entry = std::shared_ptr<SequenceEntry>(new SequenceEntry(sound, begin, end, skip, m_id++));
std::shared_ptr<SequenceEntry> entry = std::shared_ptr<SequenceEntry>(new SequenceEntry(sound, begin, end, skip, m_fps, m_id++));
m_entries.push_back(entry);
m_entry_status++;

View File

@ -22,7 +22,7 @@
AUD_NAMESPACE_BEGIN
SequenceEntry::SequenceEntry(std::shared_ptr<ISound> sound, double begin, double end, double skip, int id) :
SequenceEntry::SequenceEntry(std::shared_ptr<ISound> sound, double begin, double end, double skip, float fps, int id) :
m_status(0),
m_pos_status(1),
m_sound_status(0),
@ -31,6 +31,7 @@ SequenceEntry::SequenceEntry(std::shared_ptr<ISound> sound, double begin, double
m_begin(begin),
m_end(end),
m_skip(skip),
m_fps(fps),
m_muted(false),
m_relative(true),
m_volume_max(1.0f),

View File

@ -241,10 +241,30 @@ bool SequenceHandle::seek(double position)
return false;
std::lock_guard<ILockable> lock(*m_entry);
double seekpos = position - m_entry->m_begin;
if(seekpos < 0)
seekpos = 0;
seekpos += m_entry->m_skip;
float seek_frame = (position - m_entry->m_begin) * m_entry->m_fps;
if(seek_frame < 0)
seek_frame = 0;
seek_frame += m_entry->m_skip * m_entry->m_fps;
AnimateableProperty* pitch_property = m_entry->getAnimProperty(AP_PITCH);
float target_frame = 0;
if(pitch_property != nullptr)
{
for(int i = 0; i < seek_frame; i++)
{
float pitch;
pitch_property->read(i, &pitch);
target_frame += pitch;
}
}
else
{
target_frame = seek_frame;
}
double seekpos = target_frame / m_entry->m_fps;
m_handle->setPitch(1.0f);
m_handle->seek(seekpos);

View File

@ -154,6 +154,11 @@ void BKE_sound_set_scene_sound_volume(void *handle, float volume, char animated)
void BKE_sound_set_scene_sound_pitch(void *handle, float pitch, char animated);
void BKE_sound_set_scene_sound_pitch_constant_range(void *handle,
int frame_start,
int frame_end,
float pitch);
void BKE_sound_set_scene_sound_pan(void *handle, float pan, char animated);
void BKE_sound_update_sequencer(struct Main *main, struct bSound *sound);

View File

@ -819,6 +819,15 @@ void BKE_sound_set_scene_sound_pitch(void *handle, float pitch, char animated)
AUD_SequenceEntry_setAnimationData(handle, AUD_AP_PITCH, sound_cfra, &pitch, animated);
}
void BKE_sound_set_scene_sound_pitch_constant_range(void *handle,
int frame_start,
int frame_end,
float pitch)
{
AUD_SequenceEntry_setAnimationData_constant_range(
handle, AUD_AP_PITCH, frame_start, frame_end, &pitch);
}
void BKE_sound_set_scene_sound_pan(void *handle, float pan, char animated)
{
AUD_SequenceEntry_setAnimationData(handle, AUD_AP_PANNING, sound_cfra, &pan, animated);

View File

@ -413,7 +413,6 @@ static void draw_seq_waveform_overlay(
const float frames_per_pixel = BLI_rctf_size_x(&region->v2d.cur) / region->winx;
const float samples_per_frame = SOUND_WAVE_SAMPLES_PER_SECOND / FPS;
float samples_per_pixel = samples_per_frame * frames_per_pixel;
/* Align strip start with nearest pixel to prevent waveform flickering. */
const float x1_aligned = align_frame_with_pixel(x1, frames_per_pixel);
@ -439,15 +438,17 @@ static void draw_seq_waveform_overlay(
size_t wave_data_len = 0;
/* Offset must be also aligned, otherwise waveform flickers when moving left handle. */
const float strip_offset = align_frame_with_pixel(seq->startofs + seq->anim_startofs,
frames_per_pixel);
float start_sample = strip_offset * samples_per_frame;
start_sample += seq->sound->offset_time * SOUND_WAVE_SAMPLES_PER_SECOND;
float start_frame = SEQ_time_left_handle_frame_get(scene, seq);
/* Add off-screen part of strip to offset. */
start_sample += (frame_start - x1_aligned) * samples_per_frame;
start_frame += (frame_start - x1_aligned);
start_frame += seq->sound->offset_time / FPS;
for (int i = 0; i < pixels_to_draw; i++) {
float sample = start_sample + i * samples_per_pixel;
float timeline_frame = start_frame + i * frames_per_pixel;
/* TODO: Use linear interpolation between frames to avoid bad drawing quality. */
float frame_index = SEQ_give_frame_index(scene, seq, timeline_frame);
float sample = frame_index * samples_per_frame;
int sample_index = round_fl_to_int(sample);
if (sample_index < 0) {
@ -468,6 +469,8 @@ static void draw_seq_waveform_overlay(
value_min = (1.0f - f) * value_min + f * waveform->data[sample_index * 3 + 3];
value_max = (1.0f - f) * value_max + f * waveform->data[sample_index * 3 + 4];
rms = (1.0f - f) * rms + f * waveform->data[sample_index * 3 + 5];
float samples_per_pixel = samples_per_frame * frames_per_pixel;
if (samples_per_pixel > 1.0f) {
/* We need to sum up the values we skip over until the next step. */
float next_pos = sample + samples_per_pixel;

View File

@ -38,6 +38,9 @@ float SEQ_retiming_handle_speed_get(const struct Scene *scene,
const struct SeqRetimingHandle *handle);
int SEQ_retiming_handle_index_get(const struct Sequence *seq,
const struct SeqRetimingHandle *handle);
int SEQ_retiming_handle_index_get(const struct Sequence *seq,
const struct SeqRetimingHandle *handle);
void SEQ_retiming_sound_animation_data_set(const struct Scene *scene, const struct Sequence *seq);
#ifdef __cplusplus
}
#endif

View File

@ -73,6 +73,8 @@ int SEQ_time_find_next_prev_edit(struct Scene *scene,
bool SEQ_time_strip_intersects_frame(const struct Scene *scene,
const struct Sequence *seq,
int timeline_frame);
/* Convert timeline frame so strip frame index. */
float SEQ_give_frame_index(const struct Scene *scene, struct Sequence *seq, float timeline_frame);
/**
* Returns true if strip has frames without content to render.
*/

View File

@ -2620,7 +2620,7 @@ float seq_speed_effect_target_frame_get(Scene *scene,
}
SEQ_effect_handle_get(seq_speed); /* Ensure, that data are initialized. */
int frame_index = seq_give_frame_index(scene, seq_speed, timeline_frame);
int frame_index = SEQ_give_frame_index(scene, seq_speed, timeline_frame);
SpeedControlVars *s = (SpeedControlVars *)seq_speed->effectdata;
const Sequence *source = seq_speed->seq1;

View File

@ -142,7 +142,7 @@ static float seq_cache_timeline_frame_to_frame_index(Scene *scene,
* images or extended frame range of movies will only generate one cache entry. No special
* treatment in converting frame index to timeline_frame is needed. */
if (ELEM(type, SEQ_CACHE_STORE_RAW, SEQ_CACHE_STORE_THUMBNAIL)) {
return seq_give_frame_index(scene, seq, timeline_frame);
return SEQ_give_frame_index(scene, seq, timeline_frame);
}
return timeline_frame - SEQ_time_start_frame_get(seq);

View File

@ -209,7 +209,7 @@ ImBuf *seq_proxy_fetch(const SeqRenderData *context, Sequence *seq, int timeline
}
if (proxy->storage & SEQ_STORAGE_PROXY_CUSTOM_FILE) {
int frameno = (int)seq_give_frame_index(context->scene, seq, timeline_frame) +
int frameno = (int)SEQ_give_frame_index(context->scene, seq, timeline_frame) +
seq->anim_startofs;
if (proxy->anim == NULL) {
if (seq_proxy_get_fname(

View File

@ -238,7 +238,7 @@ StripElem *SEQ_render_give_stripelem(const Scene *scene, Sequence *seq, int time
* all other strips don't use this...
*/
int frame_index = (int)seq_give_frame_index(scene, seq, timeline_frame);
int frame_index = (int)SEQ_give_frame_index(scene, seq, timeline_frame);
if (frame_index == -1 || se == NULL) {
return NULL;
@ -1042,7 +1042,7 @@ static ImBuf *seq_render_movie_strip_custom_file_proxy(const SeqRenderData *cont
}
}
int frameno = (int)seq_give_frame_index(context->scene, seq, timeline_frame) +
int frameno = (int)SEQ_give_frame_index(context->scene, seq, timeline_frame) +
seq->anim_startofs;
return IMB_anim_absolute(proxy->anim, frameno, IMB_TC_NONE, IMB_PROXY_NONE);
}
@ -1658,7 +1658,7 @@ static ImBuf *do_render_strip_uncached(const SeqRenderData *context,
bool *r_is_proxy_image)
{
ImBuf *ibuf = NULL;
float frame_index = seq_give_frame_index(context->scene, seq, timeline_frame);
float frame_index = SEQ_give_frame_index(context->scene, seq, timeline_frame);
int type = (seq->type & SEQ_TYPE_EFFECT) ? SEQ_TYPE_EFFECT : seq->type;
switch (type) {
case SEQ_TYPE_META: {

View File

@ -990,9 +990,7 @@ static bool seq_update_seq_cb(Sequence *seq, void *user_data)
}
BKE_sound_set_scene_sound_volume(
seq->scene_sound, seq->volume, (seq->flag & SEQ_AUDIO_VOLUME_ANIMATED) != 0);
BKE_sound_set_scene_sound_pitch(seq->scene_sound,
SEQ_sound_pitch_get(scene, seq),
(seq->flag & SEQ_AUDIO_PITCH_ANIMATED) != 0);
SEQ_retiming_sound_animation_data_set(scene, seq);
BKE_sound_set_scene_sound_pan(
seq->scene_sound, seq->pan, (seq->flag & SEQ_AUDIO_PAN_ANIMATED) != 0);
}

View File

@ -433,6 +433,7 @@ Sequence *SEQ_add_movie_strip(Main *bmain, Scene *scene, ListBase *seqbase, SeqL
if (load_data->flags & SEQ_LOAD_MOVIE_SYNC_FPS) {
scene->r.frs_sec = fps_denom;
scene->r.frs_sec_base = fps_num;
DEG_id_tag_update(&scene->id, ID_RECALC_AUDIO_FPS | ID_RECALC_SEQUENCER_STRIPS);
}
load_data->r_video_stream_start = IMD_anim_get_offset(anim_arr[0]);

View File

@ -118,6 +118,7 @@ bool SEQ_retiming_is_active(const Sequence *seq)
bool SEQ_retiming_is_allowed(const Sequence *seq)
{
return ELEM(seq->type,
SEQ_TYPE_SOUND_RAM,
SEQ_TYPE_IMAGE,
SEQ_TYPE_META,
SEQ_TYPE_SCENE,
@ -126,7 +127,7 @@ bool SEQ_retiming_is_allowed(const Sequence *seq)
SEQ_TYPE_MASK);
}
float seq_retiming_evaluate(const Sequence *seq, const int frame_index)
float seq_retiming_evaluate(const Sequence *seq, const float frame_index)
{
const SeqRetimingHandle *previous_handle = retiming_find_segment_start_handle(seq, frame_index);
const SeqRetimingHandle *next_handle = previous_handle + 1;
@ -140,7 +141,7 @@ float seq_retiming_evaluate(const Sequence *seq, const int frame_index)
}
const int segment_length = next_handle->strip_frame_index - previous_handle->strip_frame_index;
const int segment_frame_index = frame_index - previous_handle->strip_frame_index;
const float segment_frame_index = frame_index - previous_handle->strip_frame_index;
const float segment_fac = segment_frame_index / (float)segment_length;
const float target_diff = next_handle->retiming_factor - previous_handle->retiming_factor;
@ -245,3 +246,131 @@ float SEQ_retiming_handle_speed_get(const Scene *scene,
const float speed = (float)fragment_length_retimed / (float)fragment_length_original;
return speed;
}
using std::vector;
class RetimingRange {
public:
int start, end;
float speed;
enum eIntersectType {
FULL,
PARTIAL_START,
PARTIAL_END,
INSIDE,
NONE,
};
RetimingRange(int start_frame, int end_frame, float speed)
: start(start_frame), end(end_frame), speed(speed)
{
}
const eIntersectType intersect_type(const RetimingRange &other) const
{
if (other.start <= start && other.end >= end) {
return FULL;
}
if (other.start > start && other.start < end && other.end > start && other.end < end) {
return INSIDE;
}
if (other.start > start && other.start < end) {
return PARTIAL_END;
}
if (other.end > start && other.end < end) {
return PARTIAL_START;
}
return NONE;
}
};
class RetimingRangeData {
public:
vector<RetimingRange> ranges;
RetimingRangeData(const Scene *scene, const Sequence *seq)
{
MutableSpan handles = SEQ_retiming_handles_get(seq);
for (const SeqRetimingHandle &handle : handles) {
if (handle.strip_frame_index == 0) {
continue;
}
const SeqRetimingHandle *handle_prev = &handle - 1;
float speed = SEQ_retiming_handle_speed_get(scene, seq, &handle);
int frame_start = SEQ_time_start_frame_get(seq) + handle_prev->strip_frame_index;
int frame_end = SEQ_time_start_frame_get(seq) + handle.strip_frame_index;
RetimingRange range = RetimingRange(frame_start, frame_end, speed);
ranges.push_back(range);
}
}
RetimingRangeData &operator*=(const RetimingRangeData rhs)
{
if (ranges.size() == 0) {
for (const RetimingRange &rhs_range : rhs.ranges) {
RetimingRange range = RetimingRange(rhs_range.start, rhs_range.end, rhs_range.speed);
ranges.push_back(range);
}
}
else {
for (int i = 0; i < ranges.size(); i++) {
RetimingRange &range = ranges[i];
for (const RetimingRange &rhs_range : rhs.ranges) {
if (range.intersect_type(rhs_range) == range.NONE) {
continue;
}
else if (range.intersect_type(rhs_range) == range.FULL) {
range.speed *= rhs_range.speed;
}
else if (range.intersect_type(rhs_range) == range.PARTIAL_START) {
RetimingRange range_left = RetimingRange(
range.start, rhs_range.end, range.speed * rhs_range.speed);
range.start = rhs_range.end + 1;
ranges.insert(ranges.begin() + i, range_left);
}
else if (range.intersect_type(rhs_range) == range.PARTIAL_END) {
RetimingRange range_left = RetimingRange(
range.start, rhs_range.start - 1, range.speed);
range.start = rhs_range.start;
ranges.insert(ranges.begin() + i, range_left);
}
else if (range.intersect_type(rhs_range) == range.INSIDE) {
RetimingRange range_left = RetimingRange(
range.start, rhs_range.start - 1, range.speed);
RetimingRange range_mid = RetimingRange(
rhs_range.start, rhs_range.start, rhs_range.speed * range.speed);
range.start = rhs_range.end + 1;
ranges.insert(ranges.begin() + i, range_left);
ranges.insert(ranges.begin() + i, range_mid);
break;
}
}
}
}
return *this;
}
};
static RetimingRangeData seq_retiming_range_data_get(const Scene *scene, const Sequence *seq)
{
RetimingRangeData strip_retiming_data = RetimingRangeData(scene, seq);
const Sequence *meta_parent = seq_sequence_lookup_meta_by_seq(scene, seq);
if (meta_parent == nullptr) {
return strip_retiming_data;
}
RetimingRangeData meta_retiming_data = RetimingRangeData(scene, meta_parent);
strip_retiming_data *= meta_retiming_data;
return strip_retiming_data;
}
void SEQ_retiming_sound_animation_data_set(const Scene *scene, const Sequence *seq)
{
RetimingRangeData retiming_data = seq_retiming_range_data_get(scene, seq);
for (const RetimingRange &range : retiming_data.ranges) {
BKE_sound_set_scene_sound_pitch_constant_range(
seq->scene_sound, range.start, range.end, range.speed);
}
}

View File

@ -62,7 +62,7 @@ int seq_time_strip_original_content_length_get(const Scene *scene, const Sequenc
return seq->len / seq_time_media_playback_rate_factor_get(scene, seq);
}
float seq_give_frame_index(const Scene *scene, Sequence *seq, float timeline_frame)
float SEQ_give_frame_index(const Scene *scene, Sequence *seq, float timeline_frame)
{
float frame_index;
float sta = SEQ_time_start_frame_get(seq);

View File

@ -16,7 +16,6 @@ struct Scene;
struct Sequence;
struct SeqCollection;
float seq_give_frame_index(const struct Scene *scene, struct Sequence *seq, float timeline_frame);
void seq_update_sound_bounds_recursive(const struct Scene *scene, struct Sequence *metaseq);
/* Describes gap between strips in timeline. */
@ -45,7 +44,7 @@ float seq_time_media_playback_rate_factor_get(const struct Scene *scene,
const struct Sequence *seq);
int seq_time_strip_original_content_length_get(const struct Scene *scene,
const struct Sequence *seq);
float seq_retiming_evaluate(const struct Sequence *seq, const int frame_index);
float seq_retiming_evaluate(const struct Sequence *seq, const float frame_index);
#ifdef __cplusplus
}