mollyim
/
webrtc
mirror of
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

RtpEncodingParameters::request_resolution patch 2 

This cl/ implements configuring of encode resolution
in the video_stream_encoder (webrtc_video_engine) in
a way that is independent of frame resolution (i.e
not using scale_resolution_down_by).

The cl/ reuses the VideoAdapter as is, and hence
the output resolution will be the same as it is today.

Anticipated further patches
3) Hook up resource adaptation
4) Let VideoSource do adaption if possible

Bug: webrtc:14451
Change-Id: I881b031c5b23be26cacfe138730154f1cb1b66a8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276742
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Jonas Oreland <jonaso@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38245}
This commit is contained in:
Jonas Oreland 2022-09-29 15:01:09 +02:00 committed by WebRTC LUCI CQ
parent 96c1a9b9e2
commit 80c87d7151
18 changed files with 352 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/video/resolution.h
View File

@ -11,12 +11,18 @@
#ifndef API_VIDEO_RESOLUTION_H_
#define API_VIDEO_RESOLUTION_H_
#include <utility>
namespace webrtc {
// A struct representing a video resolution in pixels.
struct Resolution {
int width = 0;
int height = 0;
// Helper methods.
int PixelCount() const { return width * height; }
std::pair<int, int> ToPair() const { return std::make_pair(width, height); }
};
inline bool operator==(const Resolution& lhs, const Resolution& rhs) {

8
call/call_perf_tests.cc
View File

@ -789,11 +789,11 @@ TEST_F(CallPerfTest, MAYBE_KeepsHighBitrateWhenReconfiguringSender) {
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
const VideoEncoderConfig& encoder_config) override {
int frame_width,
int frame_height,
const webrtc::VideoEncoderConfig& encoder_config) override {
std::vector<VideoStream> streams =
test::CreateVideoStreams(width, height, encoder_config);
test::CreateVideoStreams(frame_width, frame_height, encoder_config);
streams[0].min_bitrate_bps = 50000;
streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
return streams;

6
call/rampup_tests.cc
View File

@ -146,11 +146,11 @@ class RampUpTester::VideoStreamFactory
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const VideoEncoderConfig& encoder_config) override {
std::vector<VideoStream> streams =
test::CreateVideoStreams(width, height, encoder_config);
test::CreateVideoStreams(frame_width, frame_height, encoder_config);
if (encoder_config.number_of_streams == 1) {
streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
}

6
media/engine/fake_webrtc_call.cc
View File

@ -248,13 +248,14 @@ void FakeVideoSendStream::OnFrame(const webrtc::VideoFrame& frame) {
encoder_config_.video_stream_factory->CreateEncoderStreams(
frame.width(), frame.height(), encoder_config_);
} else {
webrtc::VideoEncoder::EncoderInfo encoder_info;
rtc::scoped_refptr<
webrtc::VideoEncoderConfig::VideoStreamFactoryInterface>
factory = rtc::make_ref_counted<cricket::EncoderStreamFactory>(
encoder_config_.video_format.name, encoder_config_.max_qp,
encoder_config_.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen,
encoder_config_.legacy_conference_mode);
encoder_config_.legacy_conference_mode, encoder_info);
video_streams_ = factory->CreateEncoderStreams(
frame.width(), frame.height(), encoder_config_);
@ -286,12 +287,13 @@ void FakeVideoSendStream::ReconfigureVideoEncoder(
video_streams_ = config.video_stream_factory->CreateEncoderStreams(
width, height, config);
} else {
webrtc::VideoEncoder::EncoderInfo encoder_info;
rtc::scoped_refptr<webrtc::VideoEncoderConfig::VideoStreamFactoryInterface>
factory = rtc::make_ref_counted<cricket::EncoderStreamFactory>(
config.video_format.name, config.max_qp,
config.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen,
config.legacy_conference_mode);
config.legacy_conference_mode, encoder_info);
video_streams_ = factory->CreateEncoderStreams(width, height, config);
}

207
media/engine/webrtc_video_engine_unittest.cc
View File

@ -239,6 +239,18 @@ class MockVideoSource : public rtc::VideoSourceInterface<webrtc::VideoFrame> {
(override));
};
std::vector<webrtc::Resolution> GetStreamResolutions(
const std::vector<webrtc::VideoStream>& streams) {
std::vector<webrtc::Resolution> res;
for (const auto& s : streams) {
if (s.active) {
res.push_back(
{rtc::checked_cast<int>(s.width), rtc::checked_cast<int>(s.height)});
}
}
return res;
}
} // namespace
#define EXPECT_FRAME_WAIT(c, w, h, t) \
@ -8995,4 +9007,199 @@ TEST_F(WebRtcVideoChannelBaseTest, EncoderSelectorSwitchCodec) {
channel_->SetEncoderSelector(kSsrc, nullptr);
}
TEST_F(WebRtcVideoChannelTest, RequestedResolutionSinglecast) {
cricket::VideoSendParameters parameters;
parameters.codecs.push_back(GetEngineCodec("VP8"));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = AddSendStream();
webrtc::test::FrameForwarder frame_forwarder;
cricket::FakeFrameSource frame_source(1280, 720,
rtc::kNumMicrosecsPerSec / 30);
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder));
{ // TEST requested_resolution < frame size
webrtc::RtpParameters rtp_parameters =
channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(1UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 640,
.height = 360};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
auto streams = stream->GetVideoStreams();
ASSERT_EQ(streams.size(), 1u);
EXPECT_EQ(rtc::checked_cast<size_t>(640), streams[0].width);
EXPECT_EQ(rtc::checked_cast<size_t>(360), streams[0].height);
}
{ // TEST requested_resolution == frame size
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(1UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 1280,
.height = 720};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
auto streams = stream->GetVideoStreams();
ASSERT_EQ(streams.size(), 1u);
EXPECT_EQ(rtc::checked_cast<size_t>(1280), streams[0].width);
EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].height);
}
{ // TEST requested_resolution > frame size
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(1UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 2 * 1280,
.height = 2 * 720};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
auto streams = stream->GetVideoStreams();
ASSERT_EQ(streams.size(), 1u);
EXPECT_EQ(rtc::checked_cast<size_t>(1280), streams[0].width);
EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].height);
}
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
TEST_F(WebRtcVideoChannelTest, RequestedResolutionSinglecastCropping) {
cricket::VideoSendParameters parameters;
parameters.codecs.push_back(GetEngineCodec("VP8"));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = AddSendStream();
webrtc::test::FrameForwarder frame_forwarder;
cricket::FakeFrameSource frame_source(1280, 720,
rtc::kNumMicrosecsPerSec / 30);
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder));
{
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(1UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 720,
.height = 720};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
auto streams = stream->GetVideoStreams();
ASSERT_EQ(streams.size(), 1u);
EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].width);
EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].height);
}
{
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(1UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 1280,
.height = 1280};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
auto streams = stream->GetVideoStreams();
ASSERT_EQ(streams.size(), 1u);
EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].width);
EXPECT_EQ(rtc::checked_cast<size_t>(720), streams[0].height);
}
{
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(1UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 650,
.height = 650};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
auto streams = stream->GetVideoStreams();
ASSERT_EQ(streams.size(), 1u);
EXPECT_EQ(rtc::checked_cast<size_t>(480), streams[0].width);
EXPECT_EQ(rtc::checked_cast<size_t>(480), streams[0].height);
}
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
TEST_F(WebRtcVideoChannelTest, RequestedResolutionSimulcast) {
cricket::VideoSendParameters parameters;
parameters.codecs.push_back(GetEngineCodec("VP8"));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = SetUpSimulcast(true, false);
webrtc::test::FrameForwarder frame_forwarder;
cricket::FakeFrameSource frame_source(1280, 720,
rtc::kNumMicrosecsPerSec / 30);
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder));
{
webrtc::RtpParameters rtp_parameters =
channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(3UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 320,
.height = 180};
rtp_parameters.encodings[1].requested_resolution = {.width = 640,
.height = 360};
rtp_parameters.encodings[2].requested_resolution = {.width = 1280,
.height = 720};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
EXPECT_EQ(GetStreamResolutions(stream->GetVideoStreams()),
(std::vector<webrtc::Resolution>{
{.width = 320, .height = 180},
{.width = 640, .height = 360},
{.width = 1280, .height = 720},
}));
}
{
webrtc::RtpParameters rtp_parameters =
channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(3UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 320,
.height = 180};
rtp_parameters.encodings[1].active = false;
rtp_parameters.encodings[2].requested_resolution = {.width = 1280,
.height = 720};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
EXPECT_EQ(GetStreamResolutions(stream->GetVideoStreams()),
(std::vector<webrtc::Resolution>{
{.width = 320, .height = 180},
{.width = 1280, .height = 720},
}));
}
{
webrtc::RtpParameters rtp_parameters =
channel_->GetRtpSendParameters(last_ssrc_);
EXPECT_EQ(3UL, rtp_parameters.encodings.size());
rtp_parameters.encodings[0].requested_resolution = {.width = 320,
.height = 180};
rtp_parameters.encodings[1].active = true;
rtp_parameters.encodings[1].requested_resolution = {.width = 640,
.height = 360};
rtp_parameters.encodings[2].requested_resolution = {.width = 960,
.height = 540};
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
EXPECT_EQ(GetStreamResolutions(stream->GetVideoStreams()),
(std::vector<webrtc::Resolution>{
{.width = 320, .height = 180},
{.width = 640, .height = 360},
{.width = 960, .height = 540},
}));
}
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
} // namespace cricket

3
rtc_tools/rtp_generator/rtp_generator.cc
View File

@ -175,6 +175,7 @@ RtpGenerator::RtpGenerator(const RtpGeneratorOptions& options)
constexpr int kMaxBitrateBps = 2500000; // 2.5 Mbps
int stream_count = 0;
webrtc::VideoEncoder::EncoderInfo encoder_info;
for (const auto& send_config : options.video_streams) {
webrtc::VideoSendStream::Config video_config(this);
video_config.encoder_settings.encoder_factory =
@ -225,7 +226,7 @@ RtpGenerator::RtpGenerator(const RtpGeneratorOptions& options)
encoder_config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
video_config.rtp.payload_name, /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
// Setup the fake video stream for this.
std::unique_ptr<test::FrameGeneratorCapturer> frame_generator =

6
test/encoder_settings.cc
View File

@ -106,10 +106,10 @@ std::vector<VideoStream> CreateVideoStreams(
DefaultVideoStreamFactory::DefaultVideoStreamFactory() {}
std::vector<VideoStream> DefaultVideoStreamFactory::CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const webrtc::VideoEncoderConfig& encoder_config) {
return CreateVideoStreams(width, height, encoder_config);
return CreateVideoStreams(frame_width, frame_height, encoder_config);
}
void FillEncoderConfiguration(VideoCodecType codec_type,

6
test/encoder_settings.h
View File

@ -35,9 +35,9 @@ class DefaultVideoStreamFactory
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
const VideoEncoderConfig& encoder_config) override;
int frame_width,
int frame_height,
const webrtc::VideoEncoderConfig& encoder_config) override;
};
// Creates `encoder_config.number_of_streams` VideoStreams where index

4
test/scenario/video_stream.cc
View File

@ -224,6 +224,7 @@ CreateEncoderSpecificSettings(VideoStreamConfig config) {
}
VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) {
webrtc::VideoEncoder::EncoderInfo encoder_info;
VideoEncoderConfig encoder_config;
encoder_config.codec_type = config.encoder.codec;
encoder_config.content_type = ConvertContentType(config.encoder.content_type);
@ -241,7 +242,8 @@ VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) {
VideoStreamConfig::Encoder::ContentType::kScreen;
encoder_config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
cricket_codec, kDefaultMaxQp, screenshare, screenshare);
cricket_codec, kDefaultMaxQp, screenshare, screenshare,
encoder_info);
} else {
encoder_config.video_stream_factory =
rtc::make_ref_counted<DefaultVideoStreamFactory>();

62
video/config/encoder_stream_factory.cc
View File

@ -18,9 +18,11 @@
#include "absl/strings/match.h"
#include "api/video/video_codec_constants.h"
#include "media/base/media_constants.h"
#include "media/base/video_adapter.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include "rtc_base/experiments/min_video_bitrate_experiment.h"
#include "rtc_base/experiments/normalize_simulcast_size_experiment.h"
#include "rtc_base/logging.h"
#include "video/config/simulcast.h"
namespace cricket {
@ -96,22 +98,35 @@ static int GetMaxDefaultVideoBitrateKbps(int width,
// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of
// EncoderStreamFactory and instead set this value individually for each stream
// in the VideoEncoderConfig.simulcast_layers.
EncoderStreamFactory::EncoderStreamFactory(std::string codec_name,
int max_qp,
bool is_screenshare,
bool conference_mode)
: codec_name_(codec_name),
max_qp_(max_qp),
is_screenshare_(is_screenshare),
conference_mode_(conference_mode),
trials_(fallback_trials_),
encoder_info_requested_resolution_alignment_(1) {}
EncoderStreamFactory::EncoderStreamFactory(
std::string codec_name,
int max_qp,
bool is_screenshare,
bool conference_mode,
const webrtc::VideoEncoder::EncoderInfo& encoder_info,
const webrtc::FieldTrialsView* trials)
: codec_name_(codec_name),
max_qp_(max_qp),
is_screenshare_(is_screenshare),
conference_mode_(conference_mode),
trials_(trials ? *trials : fallback_trials_) {}
trials_(trials ? *trials : fallback_trials_),
encoder_info_requested_resolution_alignment_(
encoder_info.requested_resolution_alignment) {}
std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const webrtc::VideoEncoderConfig& encoder_config) {
RTC_DCHECK_GT(encoder_config.number_of_streams, 0);
RTC_DCHECK_GE(encoder_config.simulcast_layers.size(),
@ -125,10 +140,10 @@ std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
absl::EqualsIgnoreCase(codec_name_, kH264CodecName)) &&
is_screenshare_ && conference_mode_)) {
return CreateSimulcastOrConferenceModeScreenshareStreams(
width, height, encoder_config, experimental_min_bitrate);
frame_width, frame_height, encoder_config, experimental_min_bitrate);
}
return CreateDefaultVideoStreams(width, height, encoder_config,
return CreateDefaultVideoStreams(frame_width, frame_height, encoder_config,
experimental_min_bitrate);
}
@ -172,7 +187,13 @@ EncoderStreamFactory::CreateDefaultVideoStreams(
layer.active = absl::c_any_of(encoder_config.simulcast_layers,
[](const auto& layer) { return layer.active; });
if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) {
if (encoder_config.simulcast_layers[0].requested_resolution) {
auto res = GetLayerResolutionFromRequestedResolution(
width, height,
*encoder_config.simulcast_layers[0].requested_resolution);
layer.width = res.width;
layer.height = res.height;
} else if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) {
layer.width = ScaleDownResolution(
layer.width,
encoder_config.simulcast_layers[0].scale_resolution_down_by,
@ -307,7 +328,13 @@ EncoderStreamFactory::CreateSimulcastOrConferenceModeScreenshareStreams(
layers[i].max_framerate =
encoder_config.simulcast_layers[i].max_framerate;
}
if (has_scale_resolution_down_by) {
if (encoder_config.simulcast_layers[i].requested_resolution.has_value()) {
auto res = GetLayerResolutionFromRequestedResolution(
normalized_width, normalized_height,
*encoder_config.simulcast_layers[i].requested_resolution);
layers[i].width = res.width;
layers[i].height = res.height;
} else if (has_scale_resolution_down_by) {
const double scale_resolution_down_by = std::max(
encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0);
layers[i].width = ScaleDownResolution(
@ -398,4 +425,23 @@ EncoderStreamFactory::CreateSimulcastOrConferenceModeScreenshareStreams(
return layers;
}
webrtc::Resolution
EncoderStreamFactory::GetLayerResolutionFromRequestedResolution(
int frame_width,
int frame_height,
webrtc::Resolution requested_resolution) const {
VideoAdapter adapter(encoder_info_requested_resolution_alignment_);
adapter.OnOutputFormatRequest(requested_resolution.ToPair(),
requested_resolution.PixelCount(),
absl::nullopt);
int cropped_width, cropped_height;
int out_width = 0, out_height = 0;
if (!adapter.AdaptFrameResolution(frame_width, frame_height, 0,
&cropped_width, &cropped_height, &out_width,
&out_height)) {
RTC_LOG(LS_ERROR) << "AdaptFrameResolution returned false!";
}
return {.width = out_width, .height = out_height};
}
} // namespace cricket

18
video/config/encoder_stream_factory.h
View File

@ -15,6 +15,7 @@
#include "api/transport/field_trial_based_config.h"
#include "api/units/data_rate.h"
#include "api/video_codecs/video_encoder.h"
#include "video/config/video_encoder_config.h"
namespace cricket {
@ -22,21 +23,18 @@ namespace cricket {
class EncoderStreamFactory
: public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
public:
// Note: this constructor is used by testcase in downstream.
EncoderStreamFactory(std::string codec_name,
int max_qp,
bool is_screenshare,
bool conference_mode)
: EncoderStreamFactory(codec_name,
max_qp,
is_screenshare,
conference_mode,
nullptr) {}
bool conference_mode);
EncoderStreamFactory(std::string codec_name,
int max_qp,
bool is_screenshare,
bool conference_mode,
const webrtc::FieldTrialsView* trials);
const webrtc::VideoEncoder::EncoderInfo& encoder_info,
const webrtc::FieldTrialsView* trials = nullptr);
private:
std::vector<webrtc::VideoStream> CreateEncoderStreams(
@ -57,6 +55,11 @@ class EncoderStreamFactory
const webrtc::VideoEncoderConfig& encoder_config,
const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const;
webrtc::Resolution GetLayerResolutionFromRequestedResolution(
int in_frame_width,
int in_frame_height,
webrtc::Resolution requested_resolution) const;
const std::string codec_name_;
const int max_qp_;
const bool is_screenshare_;
@ -65,6 +68,7 @@ class EncoderStreamFactory
const bool conference_mode_;
const webrtc::FieldTrialBasedConfig fallback_trials_;
const webrtc::FieldTrialsView& trials_;
const int encoder_info_requested_resolution_alignment_;
};
} // namespace cricket

4
video/config/video_encoder_config.h
View File

@ -135,8 +135,8 @@ class VideoEncoderConfig {
// The size of the vector may not be larger than
// `encoder_config.number_of_streams`.
virtual std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const VideoEncoderConfig& encoder_config) = 0;
protected:

3
video/end_to_end_tests/resolution_bitrate_limits_tests.cc
View File

@ -129,6 +129,7 @@ class InitEncodeTest : public test::EndToEndTest,
VideoSendStream::Config* send_config,
std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
VideoEncoderConfig* encoder_config) override {
webrtc::VideoEncoder::EncoderInfo encoder_info;
send_config->encoder_settings.encoder_factory = &encoder_factory_;
send_config->rtp.payload_name = payload_name_;
send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
@ -137,7 +138,7 @@ class InitEncodeTest : public test::EndToEndTest,
encoder_config->video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
payload_name_, /*max qp*/ 0, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
encoder_config->max_bitrate_bps = -1;
if (configs_.size() == 1 && configs_[0].bitrate.max)
encoder_config->max_bitrate_bps = configs_[0].bitrate.max->bps();

6
video/end_to_end_tests/rtp_rtcp_tests.cc
View File

@ -138,11 +138,11 @@ void RtpRtcpEndToEndTest::TestRtpStatePreservation(
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const VideoEncoderConfig& encoder_config) override {
std::vector<VideoStream> streams =
test::CreateVideoStreams(width, height, encoder_config);
test::CreateVideoStreams(frame_width, frame_height, encoder_config);
if (encoder_config.number_of_streams > 1) {
// Lower bitrates so that all streams send initially.

18
video/video_quality_test.cc
View File

@ -108,13 +108,13 @@ class VideoStreamFactory
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const VideoEncoderConfig& encoder_config) override {
// The highest layer must match the incoming resolution.
std::vector<VideoStream> streams = streams_;
streams[streams_.size() - 1].height = height;
streams[streams_.size() - 1].width = width;
streams[streams_.size() - 1].height = frame_height;
streams[streams_.size() - 1].width = frame_width;
streams[0].bitrate_priority = encoder_config.bitrate_priority;
return streams;
@ -613,6 +613,7 @@ void VideoQualityTest::FillScalabilitySettings(
const std::vector<std::string>& sl_descriptors) {
if (params->ss[video_idx].streams.empty() &&
params->ss[video_idx].infer_streams) {
webrtc::VideoEncoder::EncoderInfo encoder_info;
webrtc::VideoEncoderConfig encoder_config;
encoder_config.codec_type =
PayloadStringToCodecType(params->video[video_idx].codec);
@ -629,11 +630,11 @@ void VideoQualityTest::FillScalabilitySettings(
encoder_config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
params->video[video_idx].codec, kDefaultMaxQp,
params->screenshare[video_idx].enabled, true);
params->screenshare[video_idx].enabled, true, encoder_info);
params->ss[video_idx].streams =
encoder_config.video_stream_factory->CreateEncoderStreams(
static_cast<int>(params->video[video_idx].width),
static_cast<int>(params->video[video_idx].height), encoder_config);
params->video[video_idx].width, params->video[video_idx].height,
encoder_config);
} else {
// Read VideoStream and SpatialLayer elements from a list of comma separated
// lists. To use a default value for an element, use -1 or leave empty.
@ -707,6 +708,7 @@ void VideoQualityTest::SetupVideo(Transport* send_transport,
RTC_CHECK(num_video_streams_ > 0);
video_encoder_configs_.resize(num_video_streams_);
std::string generic_codec_name;
webrtc::VideoEncoder::EncoderInfo encoder_info;
for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) {
video_send_configs_.push_back(VideoSendStream::Config(send_transport));
video_encoder_configs_.push_back(VideoEncoderConfig());
@ -804,7 +806,7 @@ void VideoQualityTest::SetupVideo(Transport* send_transport,
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
params_.video[video_idx].codec,
params_.ss[video_idx].streams[0].max_qp,
params_.screenshare[video_idx].enabled, true);
params_.screenshare[video_idx].enabled, true, encoder_info);
video_encoder_configs_[video_idx].spatial_layers =
params_.ss[video_idx].spatial_layers;

9
video/video_send_stream_tests.cc
View File

@ -2625,11 +2625,11 @@ TEST_F(VideoSendStreamTest, TranslatesTwoLayerScreencastToTargetBitrate) {
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const VideoEncoderConfig& encoder_config) override {
std::vector<VideoStream> streams =
test::CreateVideoStreams(width, height, encoder_config);
test::CreateVideoStreams(frame_width, frame_height, encoder_config);
RTC_CHECK_GT(streams[0].max_bitrate_bps,
kScreencastMaxTargetBitrateDeltaKbps);
streams[0].target_bitrate_bps =
@ -4080,6 +4080,7 @@ void VideoSendStreamTest::TestTemporalLayers(
VideoSendStream::Config* send_config,
std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
VideoEncoderConfig* encoder_config) override {
webrtc::VideoEncoder::EncoderInfo encoder_info;
send_config->encoder_settings.encoder_factory = encoder_factory_;
send_config->rtp.payload_name = payload_name_;
send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
@ -4088,7 +4089,7 @@ void VideoSendStreamTest::TestTemporalLayers(
encoder_config->video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
payload_name_, /*max_qp=*/56, /*is_screenshare=*/false,
/*conference_mode=*/false);
/*conference_mode=*/false, encoder_info);
encoder_config->max_bitrate_bps = kMaxBitrateBps;
if (absl::EqualsIgnoreCase(payload_name_, "VP9")) {
encoder_config->encoder_specific_settings = rtc::make_ref_counted<

4
video/video_stream_encoder.cc
View File

@ -932,6 +932,7 @@ void VideoStreamEncoder::ReconfigureEncoder() {
encoder_reset_required = true;
}
// TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
// Possibly adjusts scale_resolution_down_by in `encoder_config_` to limit the
// alignment value.
AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
@ -948,12 +949,13 @@ void VideoStreamEncoder::ReconfigureEncoder() {
encoder_config_.video_format.name, encoder_config_.max_qp,
encoder_config_.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen,
encoder_config_.legacy_conference_mode);
encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo());
streams = factory->CreateEncoderStreams(
last_frame_info_->width, last_frame_info_->height, encoder_config_);
}
// TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory
// Get alignment when actual number of layers are known.
int alignment = AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors(
encoder_->GetEncoderInfo(), &encoder_config_, streams.size());

41
video/video_stream_encoder_unittest.cc
View File

@ -500,11 +500,12 @@ class CroppingVideoStreamFactory
private:
std::vector<VideoStream> CreateEncoderStreams(
int width,
int height,
int frame_width,
int frame_height,
const VideoEncoderConfig& encoder_config) override {
std::vector<VideoStream> streams = test::CreateVideoStreams(
width - width % 4, height - height % 4, encoder_config);
frame_width - frame_width % 4, frame_height - frame_height % 4,
encoder_config);
return streams;
}
};
@ -2190,6 +2191,7 @@ TEST_F(VideoStreamEncoderTest,
// Two streams, highest stream active.
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
const int kNumStreams = 2;
test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
config.max_bitrate_bps = 0;
@ -2198,7 +2200,7 @@ TEST_F(VideoStreamEncoderTest,
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
// The encoder bitrate limits for 270p should be used.
@ -2254,6 +2256,7 @@ TEST_F(VideoStreamEncoderTest,
DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
// Two streams, highest stream active.
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
const int kNumStreams = 2;
test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
config.max_bitrate_bps = 0;
@ -2262,7 +2265,7 @@ TEST_F(VideoStreamEncoderTest,
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
// Default bitrate limits for 270p should be used.
@ -2327,6 +2330,7 @@ TEST_F(VideoStreamEncoderTest,
// Three streams, middle stream active.
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
const int kNumStreams = 3;
test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
config.simulcast_layers[0].active = false;
@ -2335,7 +2339,7 @@ TEST_F(VideoStreamEncoderTest,
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
// The encoder bitrate limits for 360p should be used.
@ -2371,6 +2375,7 @@ TEST_F(VideoStreamEncoderTest,
// Three streams, lowest stream active.
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
const int kNumStreams = 3;
test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
config.simulcast_layers[0].active = true;
@ -2379,7 +2384,7 @@ TEST_F(VideoStreamEncoderTest,
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
// Resolution on lowest stream lower than 270p. The encoder limits not applied
@ -2407,6 +2412,7 @@ TEST_F(VideoStreamEncoderTest,
// Two streams, highest stream active.
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
const int kNumStreams = 2;
test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
config.simulcast_layers[0].active = false;
@ -2415,7 +2421,7 @@ TEST_F(VideoStreamEncoderTest,
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
// The encoder bitrate limits for 270p should be used.
@ -2496,6 +2502,7 @@ TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
// Fill config with the scaling factor by which to reduce encoding size.
const int num_streams = scale_factors_.size();
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
for (int i = 0; i < num_streams; ++i) {
config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
@ -2503,7 +2510,7 @@ TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
@ -5647,12 +5654,13 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
// Trigger QVGA "singlecast"
// Update the config.
VideoEncoderConfig video_encoder_config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
&video_encoder_config);
video_encoder_config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
for (auto& layer : video_encoder_config.simulcast_layers) {
layer.num_temporal_layers = 1;
layer.max_framerate = kDefaultFramerate;
@ -7971,6 +7979,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
const int number_layers =
sizeof(downscale_factors) / sizeof(downscale_factors[0]);
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
for (int i = 0; i < number_layers; ++i) {
config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
@ -7979,7 +7988,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
0, 0, 0);
@ -8096,6 +8105,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
@ -8104,7 +8114,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
"VP8", /*max qp*/ 56, /*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
0, 0, 0);
@ -8772,6 +8782,7 @@ TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
test::FillEncoderConfiguration(codec_type_, 1, &config);
config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
@ -8825,7 +8836,7 @@ TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
/*screencast*/ false,
/*screenshare enabled*/ false);
/*screenshare enabled*/ false, encoder_info);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
0, 0, 0);
@ -8969,13 +8980,15 @@ class ReconfigureEncoderTest : public VideoStreamEncoderTest {
void ConfigureEncoder(const VideoStream& stream) {
VideoEncoderConfig config;
webrtc::VideoEncoder::EncoderInfo encoder_info;
test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
config.max_bitrate_bps = stream.max_bitrate_bps;
config.simulcast_layers[0] = stream;
config.video_stream_factory =
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
/*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
/*conference_mode=*/false);
/*conference_mode=*/false, encoder_info);
video_stream_encoder_->ConfigureEncoder(std::move(config),
kMaxPayloadLength);
}