273 lines
12 KiB
C
273 lines
12 KiB
C
|
/*
|
||
|
|
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
|
||
|
|
*
|
||
|
|
* Use of this source code is governed by a BSD-style license
|
||
|
|
* that can be found in the LICENSE file in the root of the source
|
||
|
|
* tree. An additional intellectual property rights grant can be found
|
||
|
|
* in the file PATENTS. All contributing project authors may
|
||
|
|
* be found in the AUTHORS file in the root of the source tree.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
|
||
|
|
#define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
|
||
|
|
|
||
|
|
#include <memory>
|
||
|
|
#include <utility>
|
||
|
|
#include <vector>
|
||
|
|
|
||
|
|
#include "absl/types/optional.h"
|
||
|
|
#include "absl/types/variant.h"
|
||
|
|
#include "api/adaptation/resource.h"
|
||
|
|
#include "api/rtp_parameters.h"
|
||
|
|
#include "api/video/video_adaptation_counters.h"
|
||
|
|
#include "call/adaptation/adaptation_constraint.h"
|
||
|
|
#include "call/adaptation/degradation_preference_provider.h"
|
||
|
|
#include "call/adaptation/video_source_restrictions.h"
|
||
|
|
#include "call/adaptation/video_stream_input_state.h"
|
||
|
|
#include "call/adaptation/video_stream_input_state_provider.h"
|
||
|
|
#include "modules/video_coding/utility/quality_scaler.h"
|
||
|
|
#include "rtc_base/experiments/balanced_degradation_settings.h"
|
||
|
|
#include "rtc_base/thread_annotations.h"
|
||
|
|
|
||
|
|
namespace webrtc {
|
||
|
|
|
||
|
|
// The listener is responsible for carrying out the reconfiguration of the video
|
||
|
|
// source such that the VideoSourceRestrictions are fulfilled.
|
||
|
|
class VideoSourceRestrictionsListener {
|
||
|
|
public:
|
||
|
|
virtual ~VideoSourceRestrictionsListener();
|
||
|
|
|
||
|
|
// The |restrictions| are filtered by degradation preference but not the
|
||
|
|
// |adaptation_counters|, which are currently only reported for legacy stats
|
||
|
|
// calculation purposes.
|
||
|
|
virtual void OnVideoSourceRestrictionsUpdated(
|
||
|
|
VideoSourceRestrictions restrictions,
|
||
|
|
const VideoAdaptationCounters& adaptation_counters,
|
||
|
|
rtc::scoped_refptr<Resource> reason,
|
||
|
|
const VideoSourceRestrictions& unfiltered_restrictions) = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
class VideoStreamAdapter;
|
||
|
|
|
||
|
|
extern const int kMinFrameRateFps;
|
||
|
|
|
||
|
|
VideoSourceRestrictions FilterRestrictionsByDegradationPreference(
|
||
|
|
VideoSourceRestrictions source_restrictions,
|
||
|
|
DegradationPreference degradation_preference);
|
||
|
|
|
||
|
|
int GetHigherResolutionThan(int pixel_count);
|
||
|
|
|
||
|
|
// Either represents the next VideoSourceRestrictions the VideoStreamAdapter
|
||
|
|
// will take, or provides a Status code indicating the reason for not adapting
|
||
|
|
// if the adaptation is not valid.
|
||
|
|
class Adaptation final {
|
||
|
|
public:
|
||
|
|
enum class Status {
|
||
|
|
// Applying this adaptation will have an effect. All other Status codes
|
||
|
|
// indicate that adaptation is not possible and why.
|
||
|
|
kValid,
|
||
|
|
// Cannot adapt. The minimum or maximum adaptation has already been reached.
|
||
|
|
// There are no more steps to take.
|
||
|
|
kLimitReached,
|
||
|
|
// Cannot adapt. The resolution or frame rate requested by a recent
|
||
|
|
// adaptation has not yet been reflected in the input resolution or frame
|
||
|
|
// rate; adaptation is refused to avoid "double-adapting".
|
||
|
|
kAwaitingPreviousAdaptation,
|
||
|
|
// Not enough input.
|
||
|
|
kInsufficientInput,
|
||
|
|
// Adaptation disabled via degradation preference.
|
||
|
|
kAdaptationDisabled,
|
||
|
|
// Adaptation up was rejected by a VideoAdaptationConstraint.
|
||
|
|
kRejectedByConstraint,
|
||
|
|
};
|
||
|
|
|
||
|
|
static const char* StatusToString(Status status);
|
||
|
|
|
||
|
|
Status status() const;
|
||
|
|
const VideoStreamInputState& input_state() const;
|
||
|
|
const VideoSourceRestrictions& restrictions() const;
|
||
|
|
const VideoAdaptationCounters& counters() const;
|
||
|
|
// Used for stats reporting.
|
||
|
|
bool min_pixel_limit_reached() const;
|
||
|
|
|
||
|
|
private:
|
||
|
|
friend class VideoStreamAdapter;
|
||
|
|
|
||
|
|
// Constructs with a valid adaptation. Status is kValid.
|
||
|
|
Adaptation(int validation_id,
|
||
|
|
VideoSourceRestrictions restrictions,
|
||
|
|
VideoAdaptationCounters counters,
|
||
|
|
VideoStreamInputState input_state,
|
||
|
|
bool min_pixel_limit_reached);
|
||
|
|
// Constructor when adaptation is not valid. Status MUST NOT be kValid.
|
||
|
|
Adaptation(int validation_id,
|
||
|
|
Status invalid_status,
|
||
|
|
VideoStreamInputState input_state,
|
||
|
|
bool min_pixel_limit_reached);
|
||
|
|
|
||
|
|
// An Adaptation can become invalidated if the state of VideoStreamAdapter is
|
||
|
|
// modified before the Adaptation is applied. To guard against this, this ID
|
||
|
|
// has to match VideoStreamAdapter::adaptation_validation_id_ when applied.
|
||
|
|
// TODO(https://crbug.com/webrtc/11700): Remove the validation_id_.
|
||
|
|
const int validation_id_;
|
||
|
|
const Status status_;
|
||
|
|
const bool min_pixel_limit_reached_;
|
||
|
|
// Input state when adaptation was made.
|
||
|
|
const VideoStreamInputState input_state_;
|
||
|
|
const VideoSourceRestrictions restrictions_;
|
||
|
|
const VideoAdaptationCounters counters_;
|
||
|
|
};
|
||
|
|
|
||
|
|
// Owns the VideoSourceRestriction for a single stream and is responsible for
|
||
|
|
// adapting it up or down when told to do so. This class serves the following
|
||
|
|
// purposes:
|
||
|
|
// 1. Keep track of a stream's restrictions.
|
||
|
|
// 2. Provide valid ways to adapt up or down the stream's restrictions.
|
||
|
|
// 3. Modify the stream's restrictions in one of the valid ways.
|
||
|
|
class VideoStreamAdapter {
|
||
|
|
public:
|
||
|
|
explicit VideoStreamAdapter(
|
||
|
|
VideoStreamInputStateProvider* input_state_provider);
|
||
|
|
~VideoStreamAdapter();
|
||
|
|
|
||
|
|
VideoSourceRestrictions source_restrictions() const;
|
||
|
|
const VideoAdaptationCounters& adaptation_counters() const;
|
||
|
|
void ClearRestrictions();
|
||
|
|
|
||
|
|
void AddRestrictionsListener(
|
||
|
|
VideoSourceRestrictionsListener* restrictions_listener);
|
||
|
|
void RemoveRestrictionsListener(
|
||
|
|
VideoSourceRestrictionsListener* restrictions_listener);
|
||
|
|
void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint);
|
||
|
|
void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint);
|
||
|
|
|
||
|
|
// TODO(hbos): Setting the degradation preference should not clear
|
||
|
|
// restrictions! This is not defined in the spec and is unexpected, there is a
|
||
|
|
// tiny risk that people would discover and rely on this behavior.
|
||
|
|
void SetDegradationPreference(DegradationPreference degradation_preference);
|
||
|
|
|
||
|
|
// Returns an adaptation that we are guaranteed to be able to apply, or a
|
||
|
|
// status code indicating the reason why we cannot adapt.
|
||
|
|
// TODO(https://crbug.com/webrtc/11771) |resource| is needed by the
|
||
|
|
// AdaptationConstraint resources. Remove this parameter when it's removed.
|
||
|
|
Adaptation GetAdaptationUp(rtc::scoped_refptr<Resource> resource);
|
||
|
|
Adaptation GetAdaptationDown();
|
||
|
|
Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters,
|
||
|
|
const VideoSourceRestrictions& restrictions);
|
||
|
|
// Tries to adapt the resolution one step. This is used for initial frame
|
||
|
|
// dropping. Does nothing if the degradation preference is not BALANCED or
|
||
|
|
// MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce
|
||
|
|
// the resolution. If it fails twice it gives up.
|
||
|
|
Adaptation GetAdaptDownResolution();
|
||
|
|
|
||
|
|
// Updates source_restrictions() the Adaptation.
|
||
|
|
void ApplyAdaptation(const Adaptation& adaptation,
|
||
|
|
rtc::scoped_refptr<Resource> resource);
|
||
|
|
|
||
|
|
struct RestrictionsWithCounters {
|
||
|
|
VideoSourceRestrictions restrictions;
|
||
|
|
VideoAdaptationCounters counters;
|
||
|
|
};
|
||
|
|
|
||
|
|
private:
|
||
|
|
void BroadcastVideoRestrictionsUpdate(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const rtc::scoped_refptr<Resource>& resource);
|
||
|
|
|
||
|
|
bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state)
|
||
|
|
const RTC_RUN_ON(&sequence_checker_);
|
||
|
|
|
||
|
|
using RestrictionsOrState =
|
||
|
|
absl::variant<RestrictionsWithCounters, Adaptation::Status>;
|
||
|
|
RestrictionsOrState GetAdaptationUpStep(
|
||
|
|
const VideoStreamInputState& input_state) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
RestrictionsOrState GetAdaptationDownStep(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const RestrictionsWithCounters& current_restrictions) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
RestrictionsOrState GetAdaptDownResolutionStepForBalanced(
|
||
|
|
const VideoStreamInputState& input_state) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
RestrictionsOrState AdaptIfFpsDiffInsufficient(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const RestrictionsWithCounters& restrictions) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
|
||
|
|
// TODO(https://crbug.com/webrtc/11771) |resource| is needed by the
|
||
|
|
// AdaptationConstraint resources. Remove this parameter when it's removed.
|
||
|
|
Adaptation GetAdaptationUp(const VideoStreamInputState& input_state,
|
||
|
|
rtc::scoped_refptr<Resource> resource) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
|
||
|
|
static RestrictionsOrState DecreaseResolution(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const RestrictionsWithCounters& current_restrictions);
|
||
|
|
static RestrictionsOrState IncreaseResolution(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const RestrictionsWithCounters& current_restrictions);
|
||
|
|
// Framerate methods are member functions because they need internal state
|
||
|
|
// if the degradation preference is BALANCED.
|
||
|
|
RestrictionsOrState DecreaseFramerate(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const RestrictionsWithCounters& current_restrictions) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
RestrictionsOrState IncreaseFramerate(
|
||
|
|
const VideoStreamInputState& input_state,
|
||
|
|
const RestrictionsWithCounters& current_restrictions) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
|
||
|
|
struct RestrictionsOrStateVisitor;
|
||
|
|
Adaptation RestrictionsOrStateToAdaptation(
|
||
|
|
RestrictionsOrState step_or_state,
|
||
|
|
const VideoStreamInputState& input_state) const
|
||
|
|
RTC_RUN_ON(&sequence_checker_);
|
||
|
|
|
||
|
|
SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
// Gets the input state which is the basis of all adaptations.
|
||
|
|
// Thread safe.
|
||
|
|
VideoStreamInputStateProvider* input_state_provider_;
|
||
|
|
// Decides the next adaptation target in DegradationPreference::BALANCED.
|
||
|
|
const BalancedDegradationSettings balanced_settings_;
|
||
|
|
// To guard against applying adaptations that have become invalidated, an
|
||
|
|
// Adaptation that is applied has to have a matching validation ID.
|
||
|
|
int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
// When deciding the next target up or down, different strategies are used
|
||
|
|
// depending on the DegradationPreference.
|
||
|
|
// https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
|
||
|
|
DegradationPreference degradation_preference_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
// Used to avoid adapting twice. Stores the resolution at the time of the last
|
||
|
|
// adaptation.
|
||
|
|
// TODO(hbos): Can we implement a more general "cooldown" mechanism of
|
||
|
|
// resources intead? If we already have adapted it seems like we should wait
|
||
|
|
// a while before adapting again, so that we are not acting on usage
|
||
|
|
// measurements that are made obsolete/unreliable by an "ongoing" adaptation.
|
||
|
|
struct AwaitingFrameSizeChange {
|
||
|
|
AwaitingFrameSizeChange(bool pixels_increased, int frame_size);
|
||
|
|
const bool pixels_increased;
|
||
|
|
const int frame_size_pixels;
|
||
|
|
};
|
||
|
|
absl::optional<AwaitingFrameSizeChange> awaiting_frame_size_change_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
// The previous restrictions value. Starts as unrestricted.
|
||
|
|
VideoSourceRestrictions last_video_source_restrictions_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
VideoSourceRestrictions last_filtered_restrictions_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
|
||
|
|
std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
std::vector<AdaptationConstraint*> adaptation_constraints_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
|
||
|
|
RestrictionsWithCounters current_restrictions_
|
||
|
|
RTC_GUARDED_BY(&sequence_checker_);
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace webrtc
|
||
|
|
|
||
|
|
#endif // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
|