Add powerEfficientDecoder and powerEfficientEncoder stats

The spec for these are at https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-powerefficientdecoder and https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-powerefficientdecoder

These stats are based on the is_hardware_accelerated boolean in both the
DecoderInfo and EncoderInfo structs.

Bug: webrtc:14483
Change-Id: I4610da3c6ae977f5853a3b3424d91d864fe72592
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/274409
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Evan Shrubsole <eshr@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38441}
This commit is contained in:
Evan Shrubsole 2022-10-14 14:38:31 +00:00 committed by WebRTC LUCI CQ
parent 6253a4ff9a
commit 09da10e24f
27 changed files with 111 additions and 51 deletions

View File

@ -29,7 +29,7 @@ class MockVideoStreamEncoderObserver : public VideoStreamEncoderObserver {
(override));
MOCK_METHOD(void,
OnEncoderImplementationChanged,
(const std::string&),
(EncoderImplementation),
(override));
MOCK_METHOD(void, OnFrameDropped, (DropReason), (override));
MOCK_METHOD(void,

View File

@ -87,6 +87,7 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface {
// Decoder stats.
std::string decoder_implementation_name = "unknown";
absl::optional<bool> power_efficient_decoder;
FrameCounts frame_counts;
int decode_ms = 0;
int max_decode_ms = 0;

View File

@ -140,6 +140,7 @@ class VideoSendStream {
webrtc::VideoContentType::UNSPECIFIED;
uint32_t frames_sent = 0;
uint32_t huge_frames_sent = 0;
absl::optional<bool> power_efficient_encoder;
};
struct Config {

View File

@ -604,6 +604,7 @@ struct VideoSenderInfo : public MediaSenderInfo {
uint32_t huge_frames_sent = 0;
uint32_t aggregated_huge_frames_sent = 0;
absl::optional<std::string> rid;
absl::optional<bool> power_efficient_encoder;
};
struct VideoReceiverInfo : public MediaReceiverInfo {
@ -611,6 +612,7 @@ struct VideoReceiverInfo : public MediaReceiverInfo {
~VideoReceiverInfo();
std::vector<SsrcGroup> ssrc_groups;
std::string decoder_implementation_name;
absl::optional<bool> power_efficient_decoder;
int packets_concealed = 0;
int firs_sent = 0;
int plis_sent = 0;

View File

@ -2625,6 +2625,7 @@ WebRtcVideoChannel::WebRtcVideoSendStream::GetPerLayerVideoSenderInfos(
common_info.content_type = stats.content_type;
common_info.aggregated_framerate_sent = stats.encode_frame_rate;
common_info.aggregated_huge_frames_sent = stats.huge_frames_sent;
common_info.power_efficient_encoder = stats.power_efficient_encoder;
// If we don't have any substreams, get the remaining metrics from `stats`.
// Otherwise, these values are obtained from `sub_stream` below.
@ -3213,6 +3214,7 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo(
info.add_ssrc(config_.rtp.remote_ssrc);
webrtc::VideoReceiveStreamInterface::Stats stats = stream_->GetStats();
info.decoder_implementation_name = stats.decoder_implementation_name;
info.power_efficient_decoder = stats.power_efficient_decoder;
if (stats.current_payload_type != -1) {
info.codec_payload_type = stats.current_payload_type;
auto decoder_it = absl::c_find_if(config_.decoders, [&](const auto& d) {

View File

@ -5405,6 +5405,17 @@ TEST_F(WebRtcVideoChannelTest, GetStatsReportsEncoderImplementationName) {
info.senders[0].encoder_implementation_name);
}
TEST_F(WebRtcVideoChannelTest, GetStatsReportsPowerEfficientEncoder) {
FakeVideoSendStream* stream = AddSendStream();
webrtc::VideoSendStream::Stats stats;
stats.power_efficient_encoder = true;
stream->SetStats(stats);
cricket::VideoMediaInfo info;
ASSERT_TRUE(channel_->GetStats(&info));
EXPECT_TRUE(info.senders[0].power_efficient_encoder);
}
TEST_F(WebRtcVideoChannelTest, GetStatsReportsCpuOveruseMetrics) {
FakeVideoSendStream* stream = AddSendStream();
webrtc::VideoSendStream::Stats stats;
@ -6150,6 +6161,7 @@ TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesDecodeStatsCorrectly) {
stats.total_decode_time = webrtc::TimeDelta::Millis(16);
stats.total_assembly_time = webrtc::TimeDelta::Millis(4);
stats.frames_assembled_from_multiple_packets = 2;
stats.power_efficient_decoder = true;
stream->SetStats(stats);
cricket::VideoMediaInfo info;
@ -6181,6 +6193,7 @@ TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesDecodeStatsCorrectly) {
EXPECT_EQ(stats.total_assembly_time, info.receivers[0].total_assembly_time);
EXPECT_EQ(stats.frames_assembled_from_multiple_packets,
info.receivers[0].frames_assembled_from_multiple_packets);
EXPECT_TRUE(info.receivers[0].power_efficient_decoder);
}
TEST_F(WebRtcVideoChannelTest,

View File

@ -20,6 +20,7 @@
#include "absl/algorithm/container.h"
#include "absl/types/optional.h"
#include "api/video/video_timing.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/include/module_common_types_public.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/checks.h"
@ -202,9 +203,9 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
frame_info->content_type);
}
void VCMDecodedFrameCallback::OnDecoderImplementationName(
const char* implementation_name) {
_receiveCallback->OnDecoderImplementationName(implementation_name);
void VCMDecodedFrameCallback::OnDecoderInfoChanged(
const VideoDecoder::DecoderInfo& decoder_info) {
_receiveCallback->OnDecoderInfoChanged(decoder_info);
}
void VCMDecodedFrameCallback::Map(FrameInfo frameInfo) {
@ -254,8 +255,7 @@ bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
decoder_info_ = decoder_->GetDecoderInfo();
RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString();
if (_callback) {
_callback->OnDecoderImplementationName(
decoder_info_.implementation_name.c_str());
_callback->OnDecoderInfoChanged(decoder_info_);
}
return ok;
}
@ -293,10 +293,10 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
RTC_LOG(LS_INFO) << "Changed decoder implementation to: "
<< decoder_info.ToString();
decoder_info_ = decoder_info;
_callback->OnDecoderImplementationName(
decoder_info.implementation_name.empty()
? "unknown"
: decoder_info.implementation_name.c_str());
if (decoder_info.implementation_name.empty()) {
decoder_info.implementation_name = "unknown";
}
_callback->OnDecoderInfoChanged(std::move(decoder_info));
}
if (ret < WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp "
@ -314,8 +314,7 @@ int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
_callback = callback;
int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
if (callback && !decoder_info_.implementation_name.empty()) {
callback->OnDecoderImplementationName(
decoder_info_.implementation_name.c_str());
callback->OnDecoderInfoChanged(decoder_info_);
}
return ret;
}

View File

@ -63,7 +63,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
absl::optional<int32_t> decode_time_ms,
absl::optional<uint8_t> qp) override;
void OnDecoderImplementationName(const char* implementation_name);
void OnDecoderInfoChanged(const VideoDecoder::DecoderInfo& decoder_info);
void Map(FrameInfo frameInfo);
void ClearTimestampMap();

View File

@ -18,6 +18,7 @@
#include "api/video/video_content_type.h"
#include "api/video/video_frame.h"
#include "api/video/video_timing.h"
#include "api/video_codecs/video_decoder.h"
namespace webrtc {
@ -58,7 +59,8 @@ class VCMReceiveCallback {
// Called when the current receive codec changes.
virtual void OnIncomingPayloadType(int payload_type);
virtual void OnDecoderImplementationName(const char* implementation_name);
virtual void OnDecoderInfoChanged(
const VideoDecoder::DecoderInfo& decoder_info);
protected:
virtual ~VCMReceiveCallback() {}

View File

@ -14,7 +14,7 @@ namespace webrtc {
void VCMReceiveCallback::OnDroppedFrames(uint32_t frames_dropped) {}
void VCMReceiveCallback::OnIncomingPayloadType(int payload_type) {}
void VCMReceiveCallback::OnDecoderImplementationName(
const char* implementation_name) {}
void VCMReceiveCallback::OnDecoderInfoChanged(
const VideoDecoder::DecoderInfo&) {}
} // namespace webrtc

View File

@ -41,7 +41,10 @@ class MockVCMReceiveCallback : public VCMReceiveCallback {
(VideoFrame&, absl::optional<uint8_t>, TimeDelta, VideoContentType),
(override));
MOCK_METHOD(void, OnIncomingPayloadType, (int), (override));
MOCK_METHOD(void, OnDecoderImplementationName, (const char*), (override));
MOCK_METHOD(void,
OnDecoderInfoChanged,
(const VideoDecoder::DecoderInfo&),
(override));
};
class TestEncodedFrame : public EncodedFrame {
@ -126,7 +129,7 @@ TEST_F(VideoReceiver2Test, RegisterReceiveCodecs) {
EXPECT_TRUE(receiver_.IsExternalDecoderRegistered(kPayloadType));
EXPECT_CALL(receive_callback_, OnIncomingPayloadType(kPayloadType));
EXPECT_CALL(receive_callback_, OnDecoderImplementationName);
EXPECT_CALL(receive_callback_, OnDecoderInfoChanged);
// Call `Decode`. This triggers the above call expectations.
EXPECT_EQ(receiver_.Decode(&frame), VCM_OK);

View File

@ -44,7 +44,10 @@ class MockVCMReceiveCallback : public VCMReceiveCallback {
(VideoFrame&, absl::optional<uint8_t>, TimeDelta, VideoContentType),
(override));
MOCK_METHOD(void, OnIncomingPayloadType, (int), (override));
MOCK_METHOD(void, OnDecoderImplementationName, (const char*), (override));
MOCK_METHOD(void,
OnDecoderInfoChanged,
(const VideoDecoder::DecoderInfo&),
(override));
};
class TestVideoReceiver : public ::testing::Test {
@ -74,8 +77,7 @@ class TestVideoReceiver : public ::testing::Test {
// Since we call Decode, we need to provide a valid receive callback.
// However, for the purposes of these tests, we ignore the callbacks.
EXPECT_CALL(receive_callback_, OnIncomingPayloadType(_)).Times(AnyNumber());
EXPECT_CALL(receive_callback_, OnDecoderImplementationName(_))
.Times(AnyNumber());
EXPECT_CALL(receive_callback_, OnDecoderInfoChanged).Times(AnyNumber());
receiver_.RegisterReceiveCallback(&receive_callback_);
}

View File

@ -95,7 +95,7 @@ const char* FakeDecoder::kImplementationName = "fake_decoder";
VideoDecoder::DecoderInfo FakeDecoder::GetDecoderInfo() const {
DecoderInfo info;
info.implementation_name = kImplementationName;
info.is_hardware_accelerated = false;
info.is_hardware_accelerated = true;
return info;
}
const char* FakeDecoder::ImplementationName() const {

View File

@ -275,6 +275,7 @@ const char* FakeEncoder::kImplementationName = "fake_encoder";
VideoEncoder::EncoderInfo FakeEncoder::GetEncoderInfo() const {
EncoderInfo info;
info.implementation_name = kImplementationName;
info.is_hardware_accelerated = true;
MutexLock lock(&mutex_);
for (int sid = 0; sid < config_.numberOfSimulcastStreams; ++sid) {
int number_of_temporal_layers =

View File

@ -809,6 +809,7 @@ if (rtc_include_tests) {
":video_stream_buffer_controller",
":video_stream_decoder_impl",
":video_stream_encoder_impl",
":video_stream_encoder_interface",
"../api:create_frame_generator",
"../api:fake_frame_decryptor",
"../api:fake_frame_encryptor",

View File

@ -101,6 +101,8 @@ TEST_F(StatsEndToEndTest, GetStats) {
send_stats_filled_["DecoderImplementationName"] |=
stats.decoder_implementation_name ==
test::FakeDecoder::kImplementationName;
receive_stats_filled_["PowerEfficientDecoder"] =
stats.power_efficient_decoder.has_value();
receive_stats_filled_["RenderDelayAsHighAsExpected"] |=
stats.render_delay_ms >= kExpectedRenderDelayMs;
@ -160,6 +162,9 @@ TEST_F(StatsEndToEndTest, GetStats) {
stats.encoder_implementation_name ==
test::FakeEncoder::kImplementationName;
send_stats_filled_["PowerEfficientEncoder"] |=
stats.power_efficient_encoder == true;
for (const auto& kv : stats.substreams) {
if (expected_send_ssrcs_.find(kv.first) == expected_send_ssrcs_.end())
continue; // Probably RTX.

View File

@ -647,13 +647,16 @@ void ReceiveStatisticsProxy::OnIncomingPayloadType(int payload_type) {
}));
}
void ReceiveStatisticsProxy::OnDecoderImplementationName(
const char* implementation_name) {
void ReceiveStatisticsProxy::OnDecoderInfo(
const VideoDecoder::DecoderInfo& decoder_info) {
RTC_DCHECK_RUN_ON(&decode_queue_);
worker_thread_->PostTask(SafeTask(
task_safety_.flag(), [name = std::string(implementation_name), this]() {
task_safety_.flag(),
[this, name = decoder_info.implementation_name,
is_hardware_accelerated = decoder_info.is_hardware_accelerated]() {
RTC_DCHECK_RUN_ON(&main_thread_);
stats_.decoder_implementation_name = name;
stats_.power_efficient_decoder = is_hardware_accelerated;
}));
}

View File

@ -21,6 +21,7 @@
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/timestamp.h"
#include "api/video_codecs/video_decoder.h"
#include "call/video_receive_stream.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_coding_defines.h"
@ -75,7 +76,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
double estimated_freq_khz);
void OnRenderedFrame(const VideoFrameMetaData& frame_meta);
void OnIncomingPayloadType(int payload_type);
void OnDecoderImplementationName(const char* implementation_name);
void OnDecoderInfo(const VideoDecoder::DecoderInfo& decoder_info);
void OnPreDecode(VideoCodecType codec_type, int qp);

View File

@ -46,9 +46,9 @@ class ReceiveStatisticsProxy2Test : public ::testing::Test {
public:
ReceiveStatisticsProxy2Test() : time_controller_(Timestamp::Millis(1234)) {
metrics::Reset();
statistics_proxy_.reset(
new ReceiveStatisticsProxy(kRemoteSsrc, time_controller_.GetClock(),
time_controller_.GetMainThread()));
statistics_proxy_ = std::make_unique<ReceiveStatisticsProxy>(
kRemoteSsrc, time_controller_.GetClock(),
time_controller_.GetMainThread());
}
~ReceiveStatisticsProxy2Test() override { statistics_proxy_.reset(); }
@ -578,12 +578,19 @@ TEST_F(ReceiveStatisticsProxy2Test, GetStatsReportsIncomingPayloadType) {
EXPECT_EQ(kPayloadType, statistics_proxy_->GetStats().current_payload_type);
}
TEST_F(ReceiveStatisticsProxy2Test, GetStatsReportsDecoderImplementationName) {
const char* kName = "decoderName";
statistics_proxy_->OnDecoderImplementationName(kName);
TEST_F(ReceiveStatisticsProxy2Test, GetStatsReportsDecoderInfo) {
auto init_stats = statistics_proxy_->GetStats();
EXPECT_EQ(init_stats.decoder_implementation_name, "unknown");
EXPECT_EQ(init_stats.power_efficient_decoder, absl::nullopt);
const VideoDecoder::DecoderInfo decoder_info{
.implementation_name = "decoderName", .is_hardware_accelerated = true};
statistics_proxy_->OnDecoderInfo(decoder_info);
time_controller_.AdvanceTime(TimeDelta::Zero());
EXPECT_STREQ(
kName, statistics_proxy_->GetStats().decoder_implementation_name.c_str());
auto stats = statistics_proxy_->GetStats();
EXPECT_EQ(decoder_info.implementation_name,
stats.decoder_implementation_name);
EXPECT_TRUE(stats.power_efficient_decoder);
}
TEST_F(ReceiveStatisticsProxy2Test, GetStatsReportsOnCompleteFrame) {

View File

@ -1053,11 +1053,12 @@ void SendStatisticsProxy::OnSendEncodedImage(
}
void SendStatisticsProxy::OnEncoderImplementationChanged(
const std::string& implementation_name) {
EncoderImplementation implementation) {
MutexLock lock(&mutex_);
encoder_changed_ = EncoderChangeEvent{stats_.encoder_implementation_name,
implementation_name};
stats_.encoder_implementation_name = implementation_name;
implementation.name};
stats_.encoder_implementation_name = implementation.name;
stats_.power_efficient_encoder = implementation.is_hardware_accelerated;
}
int SendStatisticsProxy::GetInputFrameRate() const {

View File

@ -62,7 +62,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver,
const CodecSpecificInfo* codec_info) override;
void OnEncoderImplementationChanged(
const std::string& implementation_name) override;
EncoderImplementation implementation) override;
// Used to update incoming frame rate.
void OnIncomingFrame(int width, int height) override;

View File

@ -24,9 +24,11 @@
#include "api/video_codecs/video_codec.h"
#include "rtc_base/fake_clock.h"
#include "system_wrappers/include/metrics.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/scoped_key_value_config.h"
#include "video/config/video_encoder_config.h"
#include "video/video_stream_encoder_observer.h"
namespace webrtc {
namespace {
@ -2819,8 +2821,13 @@ TEST_F(SendStatisticsProxyTest, FecBitrateNotReportedWhenNotEnabled) {
TEST_F(SendStatisticsProxyTest, GetStatsReportsEncoderImplementationName) {
const std::string kName = "encoderName";
statistics_proxy_->OnEncoderImplementationChanged(kName);
statistics_proxy_->OnEncoderImplementationChanged(EncoderImplementation{
.name = kName,
.is_hardware_accelerated = true,
});
EXPECT_EQ(kName, statistics_proxy_->GetStats().encoder_implementation_name);
EXPECT_THAT(statistics_proxy_->GetStats().power_efficient_encoder,
::testing::IsTrue());
}
TEST_F(SendStatisticsProxyTest, Vp9SvcLowSpatialLayerDoesNotUpdateResolution) {
@ -2875,7 +2882,8 @@ class ForcedFallbackTest : public SendStatisticsProxyTest {
protected:
void InsertEncodedFrames(int num_frames, int interval_ms) {
statistics_proxy_->OnEncoderImplementationChanged(codec_name_);
statistics_proxy_->OnEncoderImplementationChanged(
{.name = codec_name_, .is_hardware_accelerated = false});
// First frame is not updating stats, insert initial frame.
if (statistics_proxy_->GetStats().frames_encoded == 0) {

View File

@ -42,7 +42,6 @@
#include "modules/pacing/packet_router.h"
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "modules/video_coding/encoded_frame.h"
#include "rtc_base/event.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/clock.h"
#include "test/fake_decoder.h"

View File

@ -10,6 +10,7 @@
#include "video/video_stream_decoder2.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/video_coding/video_receiver2.h"
#include "rtc_base/checks.h"
#include "video/receive_statistics_proxy2.h"
@ -60,9 +61,9 @@ void VideoStreamDecoder::OnIncomingPayloadType(int payload_type) {
receive_stats_callback_->OnIncomingPayloadType(payload_type);
}
void VideoStreamDecoder::OnDecoderImplementationName(
const char* implementation_name) {
receive_stats_callback_->OnDecoderImplementationName(implementation_name);
void VideoStreamDecoder::OnDecoderInfoChanged(
const VideoDecoder::DecoderInfo& decoder_info) {
receive_stats_callback_->OnDecoderInfo(decoder_info);
}
} // namespace internal

View File

@ -18,6 +18,7 @@
#include "api/scoped_refptr.h"
#include "api/video/video_sink_interface.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "rtc_base/platform_thread.h"
@ -45,7 +46,8 @@ class VideoStreamDecoder : public VCMReceiveCallback {
VideoContentType content_type) override;
void OnDroppedFrames(uint32_t frames_dropped) override;
void OnIncomingPayloadType(int payload_type) override;
void OnDecoderImplementationName(const char* implementation_name) override;
void OnDecoderInfoChanged(
const VideoDecoder::DecoderInfo& decoder_info) override;
private:
VideoReceiver2* const video_receiver_;

View File

@ -1838,9 +1838,12 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
// Encoder metadata needs to be updated before encode complete callback.
VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
if (info.implementation_name != encoder_info_.implementation_name) {
encoder_stats_observer_->OnEncoderImplementationChanged(
info.implementation_name);
if (info.implementation_name != encoder_info_.implementation_name ||
info.is_hardware_accelerated != encoder_info_.is_hardware_accelerated) {
encoder_stats_observer_->OnEncoderImplementationChanged({
.name = info.implementation_name,
.is_hardware_accelerated = info.is_hardware_accelerated,
});
if (bitrate_adjuster_) {
// Encoder implementation changed, reset overshoot detector states.
bitrate_adjuster_->Reset();

View File

@ -14,11 +14,9 @@
#include <string>
#include <vector>
#include "absl/types/optional.h"
#include "api/video/video_adaptation_counters.h"
#include "api/video/video_adaptation_reason.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/video_encoder.h"
#include "video/config/video_encoder_config.h"
@ -29,6 +27,11 @@ namespace webrtc {
// encoded data. So use some other type to represent that.
class EncodedImage;
struct EncoderImplementation {
const std::string& name;
bool is_hardware_accelerated;
};
// Broken out into a base class, with public inheritance below, only to ease
// unit testing of the internal class OveruseFrameDetector.
class CpuOveruseMetricsObserver {
@ -71,7 +74,7 @@ class VideoStreamEncoderObserver : public CpuOveruseMetricsObserver {
const CodecSpecificInfo* codec_info) = 0;
virtual void OnEncoderImplementationChanged(
const std::string& implementation_name) = 0;
EncoderImplementation implementation) = 0;
virtual void OnFrameDropped(DropReason reason) = 0;