498 lines
20 KiB
C++
498 lines
20 KiB
C++
/*
|
|
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "video/config/simulcast.h"
|
|
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
|
|
#include <algorithm>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "absl/strings/match.h"
|
|
#include "absl/types/optional.h"
|
|
#include "api/video/video_codec_constants.h"
|
|
#include "media/base/media_constants.h"
|
|
#include "modules/video_coding/utility/simulcast_rate_allocator.h"
|
|
#include "rtc_base/checks.h"
|
|
#include "rtc_base/experiments/field_trial_parser.h"
|
|
#include "rtc_base/experiments/min_video_bitrate_experiment.h"
|
|
#include "rtc_base/experiments/normalize_simulcast_size_experiment.h"
|
|
#include "rtc_base/experiments/rate_control_settings.h"
|
|
#include "rtc_base/logging.h"
|
|
|
|
namespace cricket {
|
|
|
|
namespace {
|
|
|
|
constexpr char kUseLegacySimulcastLayerLimitFieldTrial[] =
|
|
"WebRTC-LegacySimulcastLayerLimit";
|
|
|
|
constexpr double kDefaultMaxRoundupRate = 0.1;
|
|
|
|
// Limits for legacy conference screensharing mode. Currently used for the
|
|
// lower of the two simulcast streams.
|
|
constexpr webrtc::DataRate kScreenshareDefaultTl0Bitrate =
|
|
webrtc::DataRate::KilobitsPerSec(200);
|
|
constexpr webrtc::DataRate kScreenshareDefaultTl1Bitrate =
|
|
webrtc::DataRate::KilobitsPerSec(1000);
|
|
|
|
// Min/max bitrate for the higher one of the two simulcast stream used for
|
|
// screen content.
|
|
constexpr webrtc::DataRate kScreenshareHighStreamMinBitrate =
|
|
webrtc::DataRate::KilobitsPerSec(600);
|
|
constexpr webrtc::DataRate kScreenshareHighStreamMaxBitrate =
|
|
webrtc::DataRate::KilobitsPerSec(1250);
|
|
|
|
constexpr int kDefaultNumTemporalLayers = 3;
|
|
constexpr int kScreenshareMaxSimulcastLayers = 2;
|
|
constexpr int kScreenshareTemporalLayers = 2;
|
|
|
|
struct SimulcastFormat {
|
|
int width;
|
|
int height;
|
|
// The maximum number of simulcast layers can be used for
|
|
// resolutions at `widthxheight` for legacy applications.
|
|
size_t max_layers;
|
|
// The maximum bitrate for encoding stream at `widthxheight`, when we are
|
|
// not sending the next higher spatial stream.
|
|
webrtc::DataRate max_bitrate;
|
|
// The target bitrate for encoding stream at `widthxheight`, when this layer
|
|
// is not the highest layer (i.e., when we are sending another higher spatial
|
|
// stream).
|
|
webrtc::DataRate target_bitrate;
|
|
// The minimum bitrate needed for encoding stream at `widthxheight`.
|
|
webrtc::DataRate min_bitrate;
|
|
};
|
|
|
|
// These tables describe from which resolution we can use how many
|
|
// simulcast layers at what bitrates (maximum, target, and minimum).
|
|
// Important!! Keep this table from high resolution to low resolution.
|
|
constexpr const SimulcastFormat kSimulcastFormats[] = {
|
|
{1920, 1080, 3, webrtc::DataRate::KilobitsPerSec(5000),
|
|
webrtc::DataRate::KilobitsPerSec(4000),
|
|
webrtc::DataRate::KilobitsPerSec(800)},
|
|
{1280, 720, 3, webrtc::DataRate::KilobitsPerSec(2500),
|
|
webrtc::DataRate::KilobitsPerSec(2500),
|
|
webrtc::DataRate::KilobitsPerSec(600)},
|
|
{960, 540, 3, webrtc::DataRate::KilobitsPerSec(1200),
|
|
webrtc::DataRate::KilobitsPerSec(1200),
|
|
webrtc::DataRate::KilobitsPerSec(350)},
|
|
{640, 360, 2, webrtc::DataRate::KilobitsPerSec(700),
|
|
webrtc::DataRate::KilobitsPerSec(500),
|
|
webrtc::DataRate::KilobitsPerSec(150)},
|
|
{480, 270, 2, webrtc::DataRate::KilobitsPerSec(450),
|
|
webrtc::DataRate::KilobitsPerSec(350),
|
|
webrtc::DataRate::KilobitsPerSec(150)},
|
|
{320, 180, 1, webrtc::DataRate::KilobitsPerSec(200),
|
|
webrtc::DataRate::KilobitsPerSec(150),
|
|
webrtc::DataRate::KilobitsPerSec(30)},
|
|
// As the resolution goes down, interpolate the target and max bitrates down
|
|
// towards zero. The min bitrate is still limited at 30 kbps and the target
|
|
// and the max will be capped from below accordingly.
|
|
{0, 0, 1, webrtc::DataRate::KilobitsPerSec(0),
|
|
webrtc::DataRate::KilobitsPerSec(0),
|
|
webrtc::DataRate::KilobitsPerSec(30)}};
|
|
|
|
constexpr webrtc::DataRate Interpolate(const webrtc::DataRate& a,
|
|
const webrtc::DataRate& b,
|
|
float rate) {
|
|
return a * (1.0 - rate) + b * rate;
|
|
}
|
|
|
|
// TODO(webrtc:12415): Flip this to a kill switch when this feature launches.
|
|
bool EnableLowresBitrateInterpolation(const webrtc::FieldTrialsView& trials) {
|
|
return absl::StartsWith(
|
|
trials.Lookup("WebRTC-LowresSimulcastBitrateInterpolation"), "Enabled");
|
|
}
|
|
|
|
std::vector<SimulcastFormat> GetSimulcastFormats(
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
std::vector<SimulcastFormat> formats;
|
|
formats.insert(formats.begin(), std::begin(kSimulcastFormats),
|
|
std::end(kSimulcastFormats));
|
|
if (!enable_lowres_bitrate_interpolation) {
|
|
RTC_CHECK_GE(formats.size(), 2u);
|
|
SimulcastFormat& format0x0 = formats[formats.size() - 1];
|
|
const SimulcastFormat& format_prev = formats[formats.size() - 2];
|
|
format0x0.max_bitrate = format_prev.max_bitrate;
|
|
format0x0.target_bitrate = format_prev.target_bitrate;
|
|
format0x0.min_bitrate = format_prev.min_bitrate;
|
|
}
|
|
return formats;
|
|
}
|
|
|
|
// Multiway: Number of temporal layers for each simulcast stream.
|
|
int DefaultNumberOfTemporalLayers(const webrtc::FieldTrialsView& trials) {
|
|
const std::string group_name =
|
|
trials.Lookup("WebRTC-VP8ConferenceTemporalLayers");
|
|
if (group_name.empty())
|
|
return kDefaultNumTemporalLayers;
|
|
|
|
int num_temporal_layers = kDefaultNumTemporalLayers;
|
|
if (sscanf(group_name.c_str(), "%d", &num_temporal_layers) == 1 &&
|
|
num_temporal_layers > 0 &&
|
|
num_temporal_layers <= webrtc::kMaxTemporalStreams) {
|
|
return num_temporal_layers;
|
|
}
|
|
|
|
RTC_LOG(LS_WARNING) << "Attempt to set number of temporal layers to "
|
|
"incorrect value: "
|
|
<< group_name;
|
|
|
|
return kDefaultNumTemporalLayers;
|
|
}
|
|
|
|
int FindSimulcastFormatIndex(int width,
|
|
int height,
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
RTC_DCHECK_GE(width, 0);
|
|
RTC_DCHECK_GE(height, 0);
|
|
const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation);
|
|
for (uint32_t i = 0; i < formats.size(); ++i) {
|
|
if (width * height >= formats[i].width * formats[i].height) {
|
|
return i;
|
|
}
|
|
}
|
|
RTC_DCHECK_NOTREACHED();
|
|
return -1;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// Round size to nearest simulcast-friendly size.
|
|
// Simulcast stream width and height must both be dividable by
|
|
// |2 ^ (simulcast_layers - 1)|.
|
|
int NormalizeSimulcastSize(int size, size_t simulcast_layers) {
|
|
int base2_exponent = static_cast<int>(simulcast_layers) - 1;
|
|
const absl::optional<int> experimental_base2_exponent =
|
|
webrtc::NormalizeSimulcastSizeExperiment::GetBase2Exponent();
|
|
if (experimental_base2_exponent &&
|
|
(size > (1 << *experimental_base2_exponent))) {
|
|
base2_exponent = *experimental_base2_exponent;
|
|
}
|
|
return ((size >> base2_exponent) << base2_exponent);
|
|
}
|
|
|
|
SimulcastFormat InterpolateSimulcastFormat(
|
|
int width,
|
|
int height,
|
|
absl::optional<double> max_roundup_rate,
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation);
|
|
const int index = FindSimulcastFormatIndex(
|
|
width, height, enable_lowres_bitrate_interpolation);
|
|
if (index == 0)
|
|
return formats[index];
|
|
const int total_pixels_up =
|
|
formats[index - 1].width * formats[index - 1].height;
|
|
const int total_pixels_down = formats[index].width * formats[index].height;
|
|
const int total_pixels = width * height;
|
|
const float rate = (total_pixels_up - total_pixels) /
|
|
static_cast<float>(total_pixels_up - total_pixels_down);
|
|
|
|
// Use upper resolution if `rate` is below the configured threshold.
|
|
size_t max_layers = (rate < max_roundup_rate.value_or(kDefaultMaxRoundupRate))
|
|
? formats[index - 1].max_layers
|
|
: formats[index].max_layers;
|
|
webrtc::DataRate max_bitrate = Interpolate(formats[index - 1].max_bitrate,
|
|
formats[index].max_bitrate, rate);
|
|
webrtc::DataRate target_bitrate = Interpolate(
|
|
formats[index - 1].target_bitrate, formats[index].target_bitrate, rate);
|
|
webrtc::DataRate min_bitrate = Interpolate(formats[index - 1].min_bitrate,
|
|
formats[index].min_bitrate, rate);
|
|
|
|
return {width, height, max_layers, max_bitrate, target_bitrate, min_bitrate};
|
|
}
|
|
|
|
SimulcastFormat InterpolateSimulcastFormat(
|
|
int width,
|
|
int height,
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
return InterpolateSimulcastFormat(width, height, absl::nullopt,
|
|
enable_lowres_bitrate_interpolation);
|
|
}
|
|
|
|
webrtc::DataRate FindSimulcastMaxBitrate(
|
|
int width,
|
|
int height,
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
return InterpolateSimulcastFormat(width, height,
|
|
enable_lowres_bitrate_interpolation)
|
|
.max_bitrate;
|
|
}
|
|
|
|
webrtc::DataRate FindSimulcastTargetBitrate(
|
|
int width,
|
|
int height,
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
return InterpolateSimulcastFormat(width, height,
|
|
enable_lowres_bitrate_interpolation)
|
|
.target_bitrate;
|
|
}
|
|
|
|
webrtc::DataRate FindSimulcastMinBitrate(
|
|
int width,
|
|
int height,
|
|
bool enable_lowres_bitrate_interpolation) {
|
|
return InterpolateSimulcastFormat(width, height,
|
|
enable_lowres_bitrate_interpolation)
|
|
.min_bitrate;
|
|
}
|
|
|
|
void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate,
|
|
std::vector<webrtc::VideoStream>* layers) {
|
|
if (layers->empty())
|
|
return;
|
|
|
|
const webrtc::DataRate total_bitrate = GetTotalMaxBitrate(*layers);
|
|
|
|
// We're still not using all available bits.
|
|
if (total_bitrate < max_bitrate) {
|
|
// Spend additional bits to boost the max layer.
|
|
const webrtc::DataRate bitrate_left = max_bitrate - total_bitrate;
|
|
layers->back().max_bitrate_bps += bitrate_left.bps();
|
|
}
|
|
}
|
|
|
|
webrtc::DataRate GetTotalMaxBitrate(
|
|
const std::vector<webrtc::VideoStream>& layers) {
|
|
if (layers.empty())
|
|
return webrtc::DataRate::Zero();
|
|
|
|
int total_max_bitrate_bps = 0;
|
|
for (size_t s = 0; s < layers.size() - 1; ++s) {
|
|
total_max_bitrate_bps += layers[s].target_bitrate_bps;
|
|
}
|
|
total_max_bitrate_bps += layers.back().max_bitrate_bps;
|
|
return webrtc::DataRate::BitsPerSec(total_max_bitrate_bps);
|
|
}
|
|
|
|
size_t LimitSimulcastLayerCount(int width,
|
|
int height,
|
|
size_t need_layers,
|
|
size_t layer_count,
|
|
const webrtc::FieldTrialsView& trials) {
|
|
if (!absl::StartsWith(trials.Lookup(kUseLegacySimulcastLayerLimitFieldTrial),
|
|
"Disabled")) {
|
|
// Max layers from one higher resolution in kSimulcastFormats will be used
|
|
// if the ratio (pixels_up - pixels) / (pixels_up - pixels_down) is less
|
|
// than configured `max_ratio`. pixels_down is the selected index in
|
|
// kSimulcastFormats based on pixels.
|
|
webrtc::FieldTrialOptional<double> max_ratio("max_ratio");
|
|
webrtc::ParseFieldTrial({&max_ratio},
|
|
trials.Lookup("WebRTC-SimulcastLayerLimitRoundUp"));
|
|
|
|
const bool enable_lowres_bitrate_interpolation =
|
|
EnableLowresBitrateInterpolation(trials);
|
|
size_t adaptive_layer_count = std::max(
|
|
need_layers,
|
|
InterpolateSimulcastFormat(width, height, max_ratio.GetOptional(),
|
|
enable_lowres_bitrate_interpolation)
|
|
.max_layers);
|
|
if (layer_count > adaptive_layer_count) {
|
|
RTC_LOG(LS_WARNING) << "Reducing simulcast layer count from "
|
|
<< layer_count << " to " << adaptive_layer_count;
|
|
layer_count = adaptive_layer_count;
|
|
}
|
|
}
|
|
return layer_count;
|
|
}
|
|
|
|
std::vector<webrtc::VideoStream> GetSimulcastConfig(
|
|
size_t min_layers,
|
|
size_t max_layers,
|
|
int width,
|
|
int height,
|
|
double bitrate_priority,
|
|
int max_qp,
|
|
bool is_screenshare_with_conference_mode,
|
|
bool temporal_layers_supported,
|
|
const webrtc::FieldTrialsView& trials) {
|
|
RTC_DCHECK_LE(min_layers, max_layers);
|
|
RTC_DCHECK(max_layers > 1 || is_screenshare_with_conference_mode);
|
|
|
|
const bool base_heavy_tl3_rate_alloc =
|
|
webrtc::RateControlSettings::ParseFromKeyValueConfig(&trials)
|
|
.Vp8BaseHeavyTl3RateAllocation();
|
|
if (is_screenshare_with_conference_mode) {
|
|
return GetScreenshareLayers(max_layers, width, height, bitrate_priority,
|
|
max_qp, temporal_layers_supported,
|
|
base_heavy_tl3_rate_alloc, trials);
|
|
} else {
|
|
// Some applications rely on the old behavior limiting the simulcast layer
|
|
// count based on the resolution automatically, which they can get through
|
|
// the WebRTC-LegacySimulcastLayerLimit field trial until they update.
|
|
max_layers =
|
|
LimitSimulcastLayerCount(width, height, min_layers, max_layers, trials);
|
|
|
|
return GetNormalSimulcastLayers(max_layers, width, height, bitrate_priority,
|
|
max_qp, temporal_layers_supported,
|
|
base_heavy_tl3_rate_alloc, trials);
|
|
}
|
|
}
|
|
|
|
std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
|
|
size_t layer_count,
|
|
int width,
|
|
int height,
|
|
double bitrate_priority,
|
|
int max_qp,
|
|
bool temporal_layers_supported,
|
|
bool base_heavy_tl3_rate_alloc,
|
|
const webrtc::FieldTrialsView& trials) {
|
|
std::vector<webrtc::VideoStream> layers(layer_count);
|
|
|
|
const bool enable_lowres_bitrate_interpolation =
|
|
EnableLowresBitrateInterpolation(trials);
|
|
|
|
// Format width and height has to be divisible by |2 ^ num_simulcast_layers -
|
|
// 1|.
|
|
width = NormalizeSimulcastSize(width, layer_count);
|
|
height = NormalizeSimulcastSize(height, layer_count);
|
|
// Add simulcast streams, from highest resolution (`s` = num_simulcast_layers
|
|
// -1) to lowest resolution at `s` = 0.
|
|
for (size_t s = layer_count - 1;; --s) {
|
|
layers[s].width = width;
|
|
layers[s].height = height;
|
|
// TODO(pbos): Fill actual temporal-layer bitrate thresholds.
|
|
layers[s].max_qp = max_qp;
|
|
layers[s].num_temporal_layers =
|
|
temporal_layers_supported ? DefaultNumberOfTemporalLayers(trials) : 1;
|
|
layers[s].max_bitrate_bps =
|
|
FindSimulcastMaxBitrate(width, height,
|
|
enable_lowres_bitrate_interpolation)
|
|
.bps();
|
|
layers[s].target_bitrate_bps =
|
|
FindSimulcastTargetBitrate(width, height,
|
|
enable_lowres_bitrate_interpolation)
|
|
.bps();
|
|
int num_temporal_layers = DefaultNumberOfTemporalLayers(trials);
|
|
if (s == 0) {
|
|
// If alternative temporal rate allocation is selected, adjust the
|
|
// bitrate of the lowest simulcast stream so that absolute bitrate for
|
|
// the base temporal layer matches the bitrate for the base temporal
|
|
// layer with the default 3 simulcast streams. Otherwise we risk a
|
|
// higher threshold for receiving a feed at all.
|
|
float rate_factor = 1.0;
|
|
if (num_temporal_layers == 3) {
|
|
if (base_heavy_tl3_rate_alloc) {
|
|
// Base heavy allocation increases TL0 bitrate from 40% to 60%.
|
|
rate_factor = 0.4 / 0.6;
|
|
}
|
|
} else {
|
|
rate_factor =
|
|
webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
|
|
3, 0, /*base_heavy_tl3_rate_alloc=*/false) /
|
|
webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
|
|
num_temporal_layers, 0, /*base_heavy_tl3_rate_alloc=*/false);
|
|
}
|
|
|
|
layers[s].max_bitrate_bps =
|
|
static_cast<int>(layers[s].max_bitrate_bps * rate_factor);
|
|
layers[s].target_bitrate_bps =
|
|
static_cast<int>(layers[s].target_bitrate_bps * rate_factor);
|
|
}
|
|
layers[s].min_bitrate_bps =
|
|
FindSimulcastMinBitrate(width, height,
|
|
enable_lowres_bitrate_interpolation)
|
|
.bps();
|
|
|
|
// Ensure consistency.
|
|
layers[s].max_bitrate_bps =
|
|
std::max(layers[s].min_bitrate_bps, layers[s].max_bitrate_bps);
|
|
layers[s].target_bitrate_bps =
|
|
std::max(layers[s].min_bitrate_bps, layers[s].target_bitrate_bps);
|
|
|
|
layers[s].max_framerate = kDefaultVideoMaxFramerate;
|
|
|
|
width /= 2;
|
|
height /= 2;
|
|
|
|
if (s == 0) {
|
|
break;
|
|
}
|
|
}
|
|
// Currently the relative bitrate priority of the sender is controlled by
|
|
// the value of the lowest VideoStream.
|
|
// TODO(bugs.webrtc.org/8630): The web specification describes being able to
|
|
// control relative bitrate for each individual simulcast layer, but this
|
|
// is currently just implemented per rtp sender.
|
|
layers[0].bitrate_priority = bitrate_priority;
|
|
return layers;
|
|
}
|
|
|
|
std::vector<webrtc::VideoStream> GetScreenshareLayers(
|
|
size_t max_layers,
|
|
int width,
|
|
int height,
|
|
double bitrate_priority,
|
|
int max_qp,
|
|
bool temporal_layers_supported,
|
|
bool base_heavy_tl3_rate_alloc,
|
|
const webrtc::FieldTrialsView& trials) {
|
|
size_t num_simulcast_layers =
|
|
std::min<int>(max_layers, kScreenshareMaxSimulcastLayers);
|
|
|
|
std::vector<webrtc::VideoStream> layers(num_simulcast_layers);
|
|
// For legacy screenshare in conference mode, tl0 and tl1 bitrates are
|
|
// piggybacked on the VideoCodec struct as target and max bitrates,
|
|
// respectively. See eg. webrtc::LibvpxVp8Encoder::SetRates().
|
|
layers[0].width = width;
|
|
layers[0].height = height;
|
|
layers[0].max_qp = max_qp;
|
|
layers[0].max_framerate = 5;
|
|
layers[0].min_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps;
|
|
layers[0].target_bitrate_bps = kScreenshareDefaultTl0Bitrate.bps();
|
|
layers[0].max_bitrate_bps = kScreenshareDefaultTl1Bitrate.bps();
|
|
layers[0].num_temporal_layers = temporal_layers_supported ? 2 : 1;
|
|
|
|
// With simulcast enabled, add another spatial layer. This one will have a
|
|
// more normal layout, with the regular 3 temporal layer pattern and no fps
|
|
// restrictions. The base simulcast layer will still use legacy setup.
|
|
if (num_simulcast_layers == kScreenshareMaxSimulcastLayers) {
|
|
// Add optional upper simulcast layer.
|
|
int max_bitrate_bps;
|
|
bool using_boosted_bitrate = false;
|
|
if (!temporal_layers_supported) {
|
|
// Set the max bitrate to where the base layer would have been if temporal
|
|
// layers were enabled.
|
|
max_bitrate_bps = static_cast<int>(
|
|
kScreenshareHighStreamMaxBitrate.bps() *
|
|
webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
|
|
kScreenshareTemporalLayers, 0, base_heavy_tl3_rate_alloc));
|
|
} else {
|
|
// Experimental temporal layer mode used, use increased max bitrate.
|
|
max_bitrate_bps = kScreenshareHighStreamMaxBitrate.bps();
|
|
using_boosted_bitrate = true;
|
|
}
|
|
|
|
layers[1].width = width;
|
|
layers[1].height = height;
|
|
layers[1].max_qp = max_qp;
|
|
layers[1].max_framerate = kDefaultVideoMaxFramerate;
|
|
layers[1].num_temporal_layers =
|
|
temporal_layers_supported ? kScreenshareTemporalLayers : 1;
|
|
layers[1].min_bitrate_bps = using_boosted_bitrate
|
|
? kScreenshareHighStreamMinBitrate.bps()
|
|
: layers[0].target_bitrate_bps * 2;
|
|
layers[1].target_bitrate_bps = max_bitrate_bps;
|
|
layers[1].max_bitrate_bps = max_bitrate_bps;
|
|
}
|
|
|
|
// The bitrate priority currently implemented on a per-sender level, so we
|
|
// just set it for the first simulcast layer.
|
|
layers[0].bitrate_priority = bitrate_priority;
|
|
return layers;
|
|
}
|
|
|
|
} // namespace cricket
|