Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/timing/timing.h"
#include <algorithm>
#include "api/units/time_delta.h"
#include "modules/video_coding/timing/decode_time_percentile_filter.h"
#include "modules/video_coding/timing/timestamp_extrapolator.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
namespace {
// Default pacing that is used for the low-latency renderer path.
constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8);
constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold =
TimeDelta::Millis(500);
void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) {
if (min_delay > max_delay) {
RTC_LOG(LS_ERROR)
<< "Playout delays set incorrectly: min playout delay (" << min_delay
<< ") > max playout delay (" << max_delay
<< "). This is undefined behaviour. Application writers should "
"ensure that the min delay is always less than or equals max "
"delay. If trying to use the playout delay header extensions "
"described in "
"native-code/rtp-hdrext/playout-delay/, be careful that a playout "
"delay hint or A/V sync settings may have caused this conflict.";
}
}
} // namespace
VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials)
: clock_(clock),
ts_extrapolator_(
std::make_unique<TimestampExtrapolator>(clock_->CurrentTime())),
decode_time_filter_(std::make_unique<DecodeTimePercentileFilter>()),
render_delay_(kDefaultRenderDelay),
min_playout_delay_(TimeDelta::Zero()),
max_playout_delay_(TimeDelta::Seconds(10)),
jitter_delay_(TimeDelta::Zero()),
current_delay_(TimeDelta::Zero()),
prev_frame_timestamp_(0),
num_decoded_frames_(0),
zero_playout_delay_min_pacing_("min_pacing",
kZeroPlayoutDelayDefaultMinPacing),
last_decode_scheduled_(Timestamp::Zero()) {
ParseFieldTrial({&zero_playout_delay_min_pacing_},
field_trials.Lookup("WebRTC-ZeroPlayoutDelay"));
}
void VCMTiming::Reset() {
MutexLock lock(&mutex_);
ts_extrapolator_->Reset(clock_->CurrentTime());
decode_time_filter_ = std::make_unique<DecodeTimePercentileFilter>();
render_delay_ = kDefaultRenderDelay;
min_playout_delay_ = TimeDelta::Zero();
jitter_delay_ = TimeDelta::Zero();
current_delay_ = TimeDelta::Zero();
prev_frame_timestamp_ = 0;
}
void VCMTiming::set_render_delay(TimeDelta render_delay) {
MutexLock lock(&mutex_);
render_delay_ = render_delay;
}
TimeDelta VCMTiming::min_playout_delay() const {
MutexLock lock(&mutex_);
return min_playout_delay_;
}
void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) {
MutexLock lock(&mutex_);
if (min_playout_delay_ != min_playout_delay) {
CheckDelaysValid(min_playout_delay, max_playout_delay_);
min_playout_delay_ = min_playout_delay;
}
}
void VCMTiming::set_max_playout_delay(TimeDelta max_playout_delay) {
MutexLock lock(&mutex_);
if (max_playout_delay_ != max_playout_delay) {
CheckDelaysValid(min_playout_delay_, max_playout_delay);
max_playout_delay_ = max_playout_delay;
}
}
void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) {
MutexLock lock(&mutex_);
if (jitter_delay != jitter_delay_) {
jitter_delay_ = jitter_delay;
// When in initial state, set current delay to minimum delay.
if (current_delay_.IsZero()) {
current_delay_ = jitter_delay_;
}
}
}
void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) {
MutexLock lock(&mutex_);
TimeDelta target_delay = TargetDelayInternal();
if (current_delay_.IsZero()) {
// Not initialized, set current delay to target.
current_delay_ = target_delay;
} else if (target_delay != current_delay_) {
TimeDelta delay_diff = target_delay - current_delay_;
// Never change the delay with more than 100 ms every second. If we're
// changing the delay in too large steps we will get noticeable freezes. By
// limiting the change we can increase the delay in smaller steps, which
// will be experienced as the video is played in slow motion. When lowering
// the delay the video will be played at a faster pace.
TimeDelta max_change = TimeDelta::Zero();
if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) {
// wrap
max_change =
TimeDelta::Millis(kDelayMaxChangeMsPerS *
(frame_timestamp + (static_cast<int64_t>(1) << 32) -
prev_frame_timestamp_) /
90000);
} else {
max_change =
TimeDelta::Millis(kDelayMaxChangeMsPerS *
(frame_timestamp - prev_frame_timestamp_) / 90000);
}
if (max_change <= TimeDelta::Zero()) {
// Any changes less than 1 ms are truncated and will be postponed.
// Negative change will be due to reordering and should be ignored.
return;
}
delay_diff = std::max(delay_diff, -max_change);
delay_diff = std::min(delay_diff, max_change);
current_delay_ = current_delay_ + delay_diff;
}
prev_frame_timestamp_ = frame_timestamp;
}
void VCMTiming::UpdateCurrentDelay(Timestamp render_time,
Timestamp actual_decode_time) {
MutexLock lock(&mutex_);
TimeDelta target_delay = TargetDelayInternal();
TimeDelta delayed = (actual_decode_time - render_time) +
EstimatedMaxDecodeTime() + render_delay_;
// Only consider `delayed` as negative by more than a few microseconds.
if (delayed.ms() < 0) {
return;
}
if (current_delay_ + delayed <= target_delay) {
current_delay_ += delayed;
} else {
current_delay_ = target_delay;
}
}
void VCMTiming::StopDecodeTimer(TimeDelta decode_time, Timestamp now) {
MutexLock lock(&mutex_);
decode_time_filter_->AddTiming(decode_time.ms(), now.ms());
RTC_DCHECK_GE(decode_time, TimeDelta::Zero());
++num_decoded_frames_;
}
void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) {
MutexLock lock(&mutex_);
ts_extrapolator_->Update(now, rtp_timestamp);
}
Timestamp VCMTiming::RenderTime(uint32_t frame_timestamp, Timestamp now) const {
MutexLock lock(&mutex_);
return RenderTimeInternal(frame_timestamp, now);
}
void VCMTiming::SetLastDecodeScheduledTimestamp(
Timestamp last_decode_scheduled) {
MutexLock lock(&mutex_);
last_decode_scheduled_ = last_decode_scheduled;
}
Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp,
Timestamp now) const {
if (UseLowLatencyRendering()) {
// Render as soon as possible or with low-latency renderer algorithm.
return Timestamp::Zero();
}
// Note that TimestampExtrapolator::ExtrapolateLocalTime is not a const
// method; it mutates the object's wraparound state.
Timestamp estimated_complete_time =
ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp).value_or(now);
// Make sure the actual delay stays in the range of `min_playout_delay_`
// and `max_playout_delay_`.
TimeDelta actual_delay =
current_delay_.Clamped(min_playout_delay_, max_playout_delay_);
return estimated_complete_time + actual_delay;
}
TimeDelta VCMTiming::EstimatedMaxDecodeTime() const {
const int decode_time_ms = decode_time_filter_->RequiredDecodeTimeMs();
RTC_DCHECK_GE(decode_time_ms, 0);
return TimeDelta::Millis(decode_time_ms);
}
TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time,
Timestamp now,
bool too_many_frames_queued) const {
MutexLock lock(&mutex_);
if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 &&
min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) {
// `render_time` == 0 indicates that the frame should be decoded and
// rendered as soon as possible. However, the decoder can be choked if too
// many frames are sent at once. Therefore, limit the interframe delay to
// |zero_playout_delay_min_pacing_| unless too many frames are queued in
// which case the frames are sent to the decoder at once.
if (too_many_frames_queued) {
return TimeDelta::Zero();
}
Timestamp earliest_next_decode_start_time =
last_decode_scheduled_ + zero_playout_delay_min_pacing_;
TimeDelta max_wait_time = now >= earliest_next_decode_start_time
? TimeDelta::Zero()
: earliest_next_decode_start_time - now;
return max_wait_time;
}
return render_time - now - EstimatedMaxDecodeTime() - render_delay_;
}
TimeDelta VCMTiming::TargetVideoDelay() const {
MutexLock lock(&mutex_);
return TargetDelayInternal();
}
TimeDelta VCMTiming::TargetDelayInternal() const {
return std::max(min_playout_delay_,
jitter_delay_ + EstimatedMaxDecodeTime() + render_delay_);
}
// TODO(crbug.com/webrtc/15197): Centralize delay arithmetic.
TimeDelta VCMTiming::StatsTargetDelayInternal() const {
TimeDelta stats_target_delay =
TargetDelayInternal() - (EstimatedMaxDecodeTime() + render_delay_);
return std::max(TimeDelta::Zero(), stats_target_delay);
}
VideoFrame::RenderParameters VCMTiming::RenderParameters() const {
MutexLock lock(&mutex_);
return {.use_low_latency_rendering = UseLowLatencyRendering(),
.max_composition_delay_in_frames = max_composition_delay_in_frames_};
}
bool VCMTiming::UseLowLatencyRendering() const {
// min_playout_delay_==0,
// max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates
// that the low-latency path should be used, which means that frames should be
// decoded and rendered as soon as possible.
return min_playout_delay_.IsZero() &&
max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold;
}
VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const {
MutexLock lock(&mutex_);
return VideoDelayTimings{
.num_decoded_frames = num_decoded_frames_,
.minimum_delay = jitter_delay_,
.estimated_max_decode_time = EstimatedMaxDecodeTime(),
.render_delay = render_delay_,
.min_playout_delay = min_playout_delay_,
.max_playout_delay = max_playout_delay_,
.target_delay = StatsTargetDelayInternal(),
.current_delay = current_delay_};
}
void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) {
MutexLock lock(&mutex_);
timing_frame_info_.emplace(info);
}
std::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() {
MutexLock lock(&mutex_);
return timing_frame_info_;
}
void VCMTiming::SetMaxCompositionDelayInFrames(
std::optional<int> max_composition_delay_in_frames) {
MutexLock lock(&mutex_);
max_composition_delay_in_frames_ = max_composition_delay_in_frames;
}
} // namespace webrtc