Source code
Revision control
Copy as Markdown
Other Tools
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/utility/quality_scaler.h"
#include <memory>
#include <utility>
#include "api/field_trials_view.h"
#include "api/units/time_delta.h"
#include "api/video/video_adaptation_reason.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/weak_ptr.h"
namespace webrtc {
namespace {
// Threshold constant used until first downscale (to permit fast rampup).
static const int kMeasureMs = 2000;
static const float kSamplePeriodScaleFactor = 2.5;
static const int kFramedropPercentThreshold = 60;
static const size_t kMinFramesNeededToScale = 2 * 30;
} // namespace
class QualityScaler::QpSmoother {
public:
explicit QpSmoother(float alpha)
: alpha_(alpha),
// The initial value of last_sample_ms doesn't matter since the smoother
// will ignore the time delta for the first update.
last_sample_ms_(0),
smoother_(alpha) {}
std::optional<int> GetAvg() const {
float value = smoother_.filtered();
if (value == rtc::ExpFilter::kValueUndefined) {
return std::nullopt;
}
return static_cast<int>(value);
}
void Add(float sample, int64_t time_sent_us) {
int64_t now_ms = time_sent_us / 1000;
smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
last_sample_ms_ = now_ms;
}
void Reset() { smoother_.Reset(alpha_); }
private:
const float alpha_;
int64_t last_sample_ms_;
rtc::ExpFilter smoother_;
};
// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
// task will either run to completion and trigger a new task being queued, or it
// will be destroyed because the QualityScaler is destroyed.
//
// When high or low QP is reported, the task will be pending until a callback is
// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
// asynchronously and prevents checking for QP until the stream has potentially
// been reconfigured.
class QualityScaler::CheckQpTask {
public:
// The result of one CheckQpTask may influence the delay of the next
// CheckQpTask.
struct Result {
bool observed_enough_frames = false;
bool qp_usage_reported = false;
};
CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
: quality_scaler_(quality_scaler),
state_(State::kNotStarted),
previous_task_result_(previous_task_result),
weak_ptr_factory_(this) {}
void StartDelayedTask() {
RTC_DCHECK_EQ(state_, State::kNotStarted);
state_ = State::kCheckingQp;
TaskQueueBase::Current()->PostDelayedTask(
[this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
if (!this_weak_ptr) {
// The task has been cancelled through destruction.
return;
}
RTC_DCHECK_EQ(state_, State::kCheckingQp);
RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
switch (quality_scaler_->CheckQp()) {
case QualityScaler::CheckQpResult::kInsufficientSamples: {
result_.observed_enough_frames = false;
// After this line, `this` may be deleted.
break;
}
case QualityScaler::CheckQpResult::kNormalQp: {
result_.observed_enough_frames = true;
break;
}
case QualityScaler::CheckQpResult::kHighQp: {
result_.observed_enough_frames = true;
result_.qp_usage_reported = true;
quality_scaler_->fast_rampup_ = false;
quality_scaler_->handler_->OnReportQpUsageHigh();
quality_scaler_->ClearSamples();
break;
}
case QualityScaler::CheckQpResult::kLowQp: {
result_.observed_enough_frames = true;
result_.qp_usage_reported = true;
quality_scaler_->handler_->OnReportQpUsageLow();
quality_scaler_->ClearSamples();
break;
}
}
state_ = State::kCompleted;
// Starting the next task deletes the pending task. After this line,
// `this` has been deleted.
quality_scaler_->StartNextCheckQpTask();
},
TimeDelta::Millis(GetCheckingQpDelayMs()));
}
bool HasCompletedTask() const { return state_ == State::kCompleted; }
Result result() const {
RTC_DCHECK(HasCompletedTask());
return result_;
}
private:
enum class State {
kNotStarted,
kCheckingQp,
kCompleted,
};
// Determines the sampling period of CheckQpTasks.
int64_t GetCheckingQpDelayMs() const {
RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
if (quality_scaler_->fast_rampup_) {
return quality_scaler_->sampling_period_ms_;
}
if (quality_scaler_->experiment_enabled_ &&
!previous_task_result_.observed_enough_frames) {
// Use half the interval while waiting for enough frames.
return quality_scaler_->sampling_period_ms_ / 2;
}
if (quality_scaler_->scale_factor_ &&
!previous_task_result_.qp_usage_reported) {
// Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
return quality_scaler_->sampling_period_ms_ *
quality_scaler_->scale_factor_.value();
}
return quality_scaler_->sampling_period_ms_ *
quality_scaler_->initial_scale_factor_;
}
QualityScaler* const quality_scaler_;
State state_;
const Result previous_task_result_;
Result result_;
rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
};
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds,
const FieldTrialsView& field_trials)
: QualityScaler(handler, thresholds, field_trials, kMeasureMs) {}
// Protected ctor, should not be called directly.
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds,
const FieldTrialsView& field_trials,
int64_t default_sampling_period_ms)
: handler_(handler),
thresholds_(thresholds),
sampling_period_ms_(QualityScalerSettings(field_trials)
.SamplingPeriodMs()
.value_or(default_sampling_period_ms)),
fast_rampup_(true),
// Arbitrarily choose size based on 30 fps for 5 seconds.
average_qp_(QualityScalerSettings(field_trials)
.AverageQpWindow()
.value_or(5 * 30)),
framedrop_percent_media_opt_(5 * 30),
framedrop_percent_all_(5 * 30),
experiment_enabled_(QualityScalingExperiment::Enabled(field_trials)),
min_frames_needed_(QualityScalerSettings(field_trials)
.MinFrames()
.value_or(kMinFramesNeededToScale)),
initial_scale_factor_(QualityScalerSettings(field_trials)
.InitialScaleFactor()
.value_or(kSamplePeriodScaleFactor)),
scale_factor_(QualityScalerSettings(field_trials).ScaleFactor()) {
RTC_DCHECK_RUN_ON(&task_checker_);
if (experiment_enabled_) {
config_ = QualityScalingExperiment::GetConfig(field_trials);
qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
}
RTC_DCHECK(handler_ != nullptr);
StartNextCheckQpTask();
RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
<< ", high: " << thresholds_.high;
}
QualityScaler::~QualityScaler() {
RTC_DCHECK_RUN_ON(&task_checker_);
}
void QualityScaler::StartNextCheckQpTask() {
RTC_DCHECK_RUN_ON(&task_checker_);
RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
<< "A previous CheckQpTask has not completed yet!";
CheckQpTask::Result previous_task_result;
if (pending_qp_task_) {
previous_task_result = pending_qp_task_->result();
}
pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
pending_qp_task_->StartDelayedTask();
}
void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
RTC_DCHECK_RUN_ON(&task_checker_);
thresholds_ = thresholds;
}
void QualityScaler::ReportDroppedFrameByMediaOpt() {
RTC_DCHECK_RUN_ON(&task_checker_);
framedrop_percent_media_opt_.AddSample(100);
framedrop_percent_all_.AddSample(100);
}
void QualityScaler::ReportDroppedFrameByEncoder() {
RTC_DCHECK_RUN_ON(&task_checker_);
framedrop_percent_all_.AddSample(100);
}
void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
RTC_DCHECK_RUN_ON(&task_checker_);
framedrop_percent_media_opt_.AddSample(0);
framedrop_percent_all_.AddSample(0);
average_qp_.AddSample(qp);
if (qp_smoother_high_)
qp_smoother_high_->Add(qp, time_sent_us);
if (qp_smoother_low_)
qp_smoother_low_->Add(qp, time_sent_us);
}
QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
RTC_DCHECK_RUN_ON(&task_checker_);
// Should be set through InitEncode -> Should be set by now.
RTC_DCHECK_GE(thresholds_.low, 0);
// If we have not observed at least this many frames we can't make a good
// scaling decision.
const size_t frames = config_.use_all_drop_reasons
? framedrop_percent_all_.Size()
: framedrop_percent_media_opt_.Size();
if (frames < min_frames_needed_) {
return CheckQpResult::kInsufficientSamples;
}
// Check if we should scale down due to high frame drop.
const std::optional<int> drop_rate =
config_.use_all_drop_reasons
? framedrop_percent_all_.GetAverageRoundedDown()
: framedrop_percent_media_opt_.GetAverageRoundedDown();
if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
return CheckQpResult::kHighQp;
}
// Check if we should scale up or down based on QP.
const std::optional<int> avg_qp_high =
qp_smoother_high_ ? qp_smoother_high_->GetAvg()
: average_qp_.GetAverageRoundedDown();
const std::optional<int> avg_qp_low =
qp_smoother_low_ ? qp_smoother_low_->GetAvg()
: average_qp_.GetAverageRoundedDown();
if (avg_qp_high && avg_qp_low) {
RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
<< *avg_qp_low << ").";
if (*avg_qp_high > thresholds_.high) {
return CheckQpResult::kHighQp;
}
if (*avg_qp_low <= thresholds_.low) {
// QP has been low. We want to try a higher resolution.
return CheckQpResult::kLowQp;
}
}
return CheckQpResult::kNormalQp;
}
void QualityScaler::ClearSamples() {
RTC_DCHECK_RUN_ON(&task_checker_);
framedrop_percent_media_opt_.Reset();
framedrop_percent_all_.Reset();
average_qp_.Reset();
if (qp_smoother_high_)
qp_smoother_high_->Reset();
if (qp_smoother_low_)
qp_smoother_low_->Reset();
}
QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
} // namespace webrtc