/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef API_NETEQ_NETEQ_H_ #define API_NETEQ_NETEQ_H_ #include // Provide access to size_t. #include #include #include #include "absl/types/optional.h" #include "api/audio_codecs/audio_codec_pair_id.h" #include "api/audio_codecs/audio_decoder.h" #include "api/audio_codecs/audio_format.h" #include "api/rtp_headers.h" #include "api/scoped_refptr.h" namespace webrtc { // Forward declarations. class AudioFrame; class AudioDecoderFactory; class Clock; struct NetEqNetworkStatistics { uint16_t current_buffer_size_ms; // Current jitter buffer size in ms. uint16_t preferred_buffer_size_ms; // Target buffer size in ms. uint16_t jitter_peaks_found; // 1 if adding extra delay due to peaky // jitter; 0 otherwise. uint16_t expand_rate; // Fraction (of original stream) of synthesized // audio inserted through expansion (in Q14). uint16_t speech_expand_rate; // Fraction (of original stream) of synthesized // speech inserted through expansion (in Q14). uint16_t preemptive_rate; // Fraction of data inserted through pre-emptive // expansion (in Q14). uint16_t accelerate_rate; // Fraction of data removed through acceleration // (in Q14). uint16_t secondary_decoded_rate; // Fraction of data coming from FEC/RED // decoding (in Q14). uint16_t secondary_discarded_rate; // Fraction of discarded FEC/RED data (in // Q14). // Statistics for packet waiting times, i.e., the time between a packet // arrives until it is decoded. int mean_waiting_time_ms; int median_waiting_time_ms; int min_waiting_time_ms; int max_waiting_time_ms; }; // NetEq statistics that persist over the lifetime of the class. // These metrics are never reset. struct NetEqLifetimeStatistics { // Stats below correspond to similarly-named fields in the WebRTC stats spec. // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats uint64_t total_samples_received = 0; uint64_t concealed_samples = 0; uint64_t concealment_events = 0; uint64_t jitter_buffer_delay_ms = 0; uint64_t jitter_buffer_emitted_count = 0; uint64_t jitter_buffer_target_delay_ms = 0; uint64_t inserted_samples_for_deceleration = 0; uint64_t removed_samples_for_acceleration = 0; uint64_t silent_concealed_samples = 0; uint64_t fec_packets_received = 0; uint64_t fec_packets_discarded = 0; // Below stats are not part of the spec. uint64_t delayed_packet_outage_samples = 0; // This is sum of relative packet arrival delays of received packets so far. // Since end-to-end delay of a packet is difficult to measure and is not // necessarily useful for measuring jitter buffer performance, we report a // relative packet arrival delay. The relative packet arrival delay of a // packet is defined as the arrival delay compared to the first packet // received, given that it had zero delay. To avoid clock drift, the "first" // packet can be made dynamic. uint64_t relative_packet_arrival_delay_ms = 0; uint64_t jitter_buffer_packets_received = 0; // An interruption is a loss-concealment event lasting at least 150 ms. The // two stats below count the number os such events and the total duration of // these events. int32_t interruption_count = 0; int32_t total_interruption_duration_ms = 0; }; // Metrics that describe the operations performed in NetEq, and the internal // state. struct NetEqOperationsAndState { // These sample counters are cumulative, and don't reset. As a reference, the // total number of output samples can be found in // NetEqLifetimeStatistics::total_samples_received. uint64_t preemptive_samples = 0; uint64_t accelerate_samples = 0; // Count of the number of buffer flushes. uint64_t packet_buffer_flushes = 0; // The number of primary packets that were discarded. uint64_t discarded_primary_packets = 0; // The statistics below are not cumulative. // The waiting time of the last decoded packet. uint64_t last_waiting_time_ms = 0; // The sum of the packet and jitter buffer size in ms. uint64_t current_buffer_size_ms = 0; // The current frame size in ms. uint64_t current_frame_size_ms = 0; // Flag to indicate that the next packet is available. bool next_packet_available = false; }; // This is the interface class for NetEq. class NetEq { public: struct Config { Config(); Config(const Config&); Config(Config&&); ~Config(); Config& operator=(const Config&); Config& operator=(Config&&); std::string ToString() const; int sample_rate_hz = 16000; // Initial value. Will change with input data. bool enable_post_decode_vad = false; size_t max_packets_in_buffer = 200; int max_delay_ms = 0; int min_delay_ms = 0; bool enable_fast_accelerate = false; bool enable_muted_state = false; bool enable_rtx_handling = false; absl::optional codec_pair_id; bool for_test_no_time_stretching = false; // Use only for testing. }; enum ReturnCodes { kOK = 0, kFail = -1 }; enum class Operation { kNormal, kMerge, kExpand, kAccelerate, kFastAccelerate, kPreemptiveExpand, kRfc3389Cng, kRfc3389CngNoPacket, kCodecInternalCng, kDtmf, kUndefined, }; enum class Mode { kNormal, kExpand, kMerge, kAccelerateSuccess, kAccelerateLowEnergy, kAccelerateFail, kPreemptiveExpandSuccess, kPreemptiveExpandLowEnergy, kPreemptiveExpandFail, kRfc3389Cng, kCodecInternalCng, kCodecPlc, kDtmf, kError, kUndefined, }; // Return type for GetDecoderFormat. struct DecoderFormat { int sample_rate_hz; int num_channels; SdpAudioFormat sdp_format; }; virtual ~NetEq() {} // Inserts a new packet into NetEq. // Returns 0 on success, -1 on failure. virtual int InsertPacket(const RTPHeader& rtp_header, rtc::ArrayView payload) = 0; // Lets NetEq know that a packet arrived with an empty payload. This typically // happens when empty packets are used for probing the network channel, and // these packets use RTP sequence numbers from the same series as the actual // audio packets. virtual void InsertEmptyPacket(const RTPHeader& rtp_header) = 0; // Instructs NetEq to deliver 10 ms of audio data. The data is written to // `audio_frame`. All data in `audio_frame` is wiped; `data_`, `speech_type_`, // `num_channels_`, `sample_rate_hz_`, `samples_per_channel_`, and // `vad_activity_` are updated upon success. If an error is returned, some // fields may not have been updated, or may contain inconsistent values. // If muted state is enabled (through Config::enable_muted_state), `muted` // may be set to true after a prolonged expand period. When this happens, the // `data_` in `audio_frame` is not written, but should be interpreted as being // all zeros. For testing purposes, an override can be supplied in the // `action_override` argument, which will cause NetEq to take this action // next, instead of the action it would normally choose. An optional output // argument for fetching the current sample rate can be provided, which // will return the same value as last_output_sample_rate_hz() but will avoid // additional synchronization. // Returns kOK on success, or kFail in case of an error. virtual int GetAudio( AudioFrame* audio_frame, bool* muted, int* current_sample_rate_hz = nullptr, absl::optional action_override = absl::nullopt) = 0; // Replaces the current set of decoders with the given one. virtual void SetCodecs(const std::map& codecs) = 0; // Associates `rtp_payload_type` with the given codec, which NetEq will // instantiate when it needs it. Returns true iff successful. virtual bool RegisterPayloadType(int rtp_payload_type, const SdpAudioFormat& audio_format) = 0; // Removes `rtp_payload_type` from the codec database. Returns 0 on success, // -1 on failure. Removing a payload type that is not registered is ok and // will not result in an error. virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0; // Removes all payload types from the codec database. virtual void RemoveAllPayloadTypes() = 0; // Sets a minimum delay in millisecond for packet buffer. The minimum is // maintained unless a higher latency is dictated by channel condition. // Returns true if the minimum is successfully applied, otherwise false is // returned. virtual bool SetMinimumDelay(int delay_ms) = 0; // Sets a maximum delay in milliseconds for packet buffer. The latency will // not exceed the given value, even required delay (given the channel // conditions) is higher. Calling this method has the same effect as setting // the `max_delay_ms` value in the NetEq::Config struct. virtual bool SetMaximumDelay(int delay_ms) = 0; // Sets a base minimum delay in milliseconds for packet buffer. The minimum // delay which is set via `SetMinimumDelay` can't be lower than base minimum // delay. Calling this method is similar to setting the `min_delay_ms` value // in the NetEq::Config struct. Returns true if the base minimum is // successfully applied, otherwise false is returned. virtual bool SetBaseMinimumDelayMs(int delay_ms) = 0; // Returns current value of base minimum delay in milliseconds. virtual int GetBaseMinimumDelayMs() const = 0; // Returns the current target delay in ms. This includes any extra delay // requested through SetMinimumDelay. virtual int TargetDelayMs() const = 0; // Returns the current total delay (packet buffer and sync buffer) in ms, // with smoothing applied to even out short-time fluctuations due to jitter. // The packet buffer part of the delay is not updated during DTX/CNG periods. virtual int FilteredCurrentDelayMs() const = 0; // Writes the current network statistics to `stats`. The statistics are reset // after the call. virtual int NetworkStatistics(NetEqNetworkStatistics* stats) = 0; // Current values only, not resetting any state. virtual NetEqNetworkStatistics CurrentNetworkStatistics() const = 0; // Returns a copy of this class's lifetime statistics. These statistics are // never reset. virtual NetEqLifetimeStatistics GetLifetimeStatistics() const = 0; // Returns statistics about the performed operations and internal state. These // statistics are never reset. virtual NetEqOperationsAndState GetOperationsAndState() const = 0; // Enables post-decode VAD. When enabled, GetAudio() will return // kOutputVADPassive when the signal contains no speech. virtual void EnableVad() = 0; // Disables post-decode VAD. virtual void DisableVad() = 0; // Returns the RTP timestamp for the last sample delivered by GetAudio(). // The return value will be empty if no valid timestamp is available. virtual absl::optional GetPlayoutTimestamp() const = 0; // Returns the sample rate in Hz of the audio produced in the last GetAudio // call. If GetAudio has not been called yet, the configured sample rate // (Config::sample_rate_hz) is returned. virtual int last_output_sample_rate_hz() const = 0; // Returns the decoder info for the given payload type. Returns empty if no // such payload type was registered. virtual absl::optional GetDecoderFormat( int payload_type) const = 0; // Flushes both the packet buffer and the sync buffer. virtual void FlushBuffers() = 0; // Enables NACK and sets the maximum size of the NACK list, which should be // positive and no larger than Nack::kNackListSizeLimit. If NACK is already // enabled then the maximum NACK list size is modified accordingly. virtual void EnableNack(size_t max_nack_list_size) = 0; virtual void DisableNack() = 0; // Returns a list of RTP sequence numbers corresponding to packets to be // retransmitted, given an estimate of the round-trip time in milliseconds. virtual std::vector GetNackList( int64_t round_trip_time_ms) const = 0; // Returns a vector containing the timestamps of the packets that were decoded // in the last GetAudio call. If no packets were decoded in the last call, the // vector is empty. // Mainly intended for testing. virtual std::vector LastDecodedTimestamps() const = 0; // Returns the length of the audio yet to play in the sync buffer. // Mainly intended for testing. virtual int SyncBufferSizeMs() const = 0; }; } // namespace webrtc #endif // API_NETEQ_NETEQ_H_