#include "media/gpu/android/media_codec_video_decoder.h"
#include <memory>
#include <variant>
#include "base/android/android_info.h"
#include "base/command_line.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/functional/callback_helpers.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/memory/weak_ptr.h"
#include "base/metrics/histogram_macros.h"
#include "base/no_destructor.h"
#include "base/strings/string_number_conversions.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/time/time.h"
#include "base/trace_event/trace_event.h"
#include "media/base/android/media_codec_bridge_impl.h"
#include "media/base/android/media_codec_util.h"
#include "media/base/async_destroy_video_decoder.h"
#include "media/base/decoder_buffer.h"
#include "media/base/media_log.h"
#include "media/base/media_switches.h"
#include "media/base/scoped_async_trace.h"
#include "media/base/status.h"
#include "media/base/supported_types.h"
#include "media/base/supported_video_decoder_config.h"
#include "media/base/video_aspect_ratio.h"
#include "media/base/video_codecs.h"
#include "media/base/video_decoder_config.h"
#include "media/base/video_frame.h"
#include "media/gpu/android/android_video_surface_chooser.h"
#include "media/gpu/android/codec_allocator.h"
#include "media/gpu/android/video_accelerator_util.h"
#include "media/media_buildflags.h"
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
#include "media/base/android/extract_sps_and_pps.h"
#endif
namespace media {
namespace {
void OutputBufferReleased(base::RepeatingClosure pump_cb, bool has_work) {
if (has_work) {
pump_cb.Run();
}
}
std::vector<SupportedVideoDecoderConfig> GenerateSupportedConfigs(
DeviceInfo* device_info,
bool allow_media_codec_sw_decoder) {
std::vector<SupportedVideoDecoderConfig> supported_configs;
for (const auto& info : GetDecoderInfoCache()) {
if (base::EndsWith(info.name, ".low_latency",
base::CompareCase::INSENSITIVE_ASCII)) {
continue;
}
const auto codec = VideoCodecProfileToVideoCodec(info.profile);
constexpr bool kAllowEncrypted = true;
if ((codec == VideoCodec::kVP8 && device_info->IsVp8DecoderAvailable()) ||
(codec == VideoCodec::kH264 && IsDecoderBuiltInVideoCodec(codec)) ||
codec == VideoCodec::kVP9 || codec == VideoCodec::kAV1) {
const bool can_use_builtin_software_decoder =
info.profile != VP9PROFILE_PROFILE2 &&
info.profile != VP9PROFILE_PROFILE3 &&
info.secure_codec_capability == SecureCodecCapability::kClear;
const bool is_os_software_decoder_allowed =
!can_use_builtin_software_decoder || allow_media_codec_sw_decoder;
if (info.is_software_codec && !is_os_software_decoder_allowed) {
supported_configs.emplace_back(info.profile, info.profile,
info.coded_size_min, info.coded_size_max,
kAllowEncrypted,
true);
continue;
}
supported_configs.emplace_back(
info.profile, info.profile, info.coded_size_min, info.coded_size_max,
kAllowEncrypted,
info.secure_codec_capability ==
SecureCodecCapability::kEncrypted);
continue;
}
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
if (codec == VideoCodec::kH264
#if BUILDFLAG(ENABLE_PLATFORM_HEVC)
|| (codec == VideoCodec::kHEVC &&
base::FeatureList::IsEnabled(kPlatformHEVCDecoderSupport))
#endif
#if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
|| codec == VideoCodec::kDolbyVision
#endif
) {
supported_configs.emplace_back(
info.profile, info.profile, info.coded_size_min, info.coded_size_max,
kAllowEncrypted,
info.secure_codec_capability ==
SecureCodecCapability::kEncrypted);
}
#endif
}
return supported_configs;
}
bool SelectMediaCodec(const VideoDecoderConfig& config,
bool requires_secure_codec,
bool requires_low_latency_codec,
std::string* out_codec_name) {
*out_codec_name = "";
std::string software_decoder;
for (const auto& info : GetDecoderInfoCache()) {
VideoCodec codec = VideoCodecProfileToVideoCodec(info.profile);
if (config.codec() != codec) {
continue;
}
if (config.profile() != VIDEO_CODEC_PROFILE_UNKNOWN &&
config.profile() != info.profile) {
continue;
}
if (config.level() != kNoVideoCodecLevel &&
info.level != kNoVideoCodecLevel && config.level() > info.level) {
continue;
}
if (config.coded_size().width() < info.coded_size_min.width() ||
config.coded_size().height() < info.coded_size_min.height() ||
config.coded_size().width() > info.coded_size_max.width() ||
config.coded_size().height() > info.coded_size_max.height()) {
continue;
}
if (!requires_secure_codec &&
info.secure_codec_capability == SecureCodecCapability::kEncrypted) {
continue;
}
if (requires_secure_codec &&
info.secure_codec_capability == SecureCodecCapability::kClear) {
continue;
}
if (!requires_low_latency_codec &&
info.low_latency_capability == LowLatencyCapability::kRequired) {
continue;
}
if (requires_low_latency_codec &&
info.low_latency_capability == LowLatencyCapability::kNone) {
continue;
}
if (info.is_software_codec) {
if (software_decoder.empty()) {
software_decoder = info.name;
}
continue;
}
*out_codec_name = info.name;
return true;
}
if (!(config.is_encrypted()
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
|| config.codec() == VideoCodec::kH264
#if BUILDFLAG(ENABLE_PLATFORM_HEVC)
|| (config.codec() == VideoCodec::kHEVC &&
base::FeatureList::IsEnabled(kPlatformHEVCDecoderSupport))
#endif
#if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION)
|| config.codec() == VideoCodec::kDolbyVision
#endif
#endif
|| config.profile() == VP9PROFILE_PROFILE2 ||
config.profile() == VP9PROFILE_PROFILE3)) {
DVLOG(2) << "Can't find proper video decoder from decoder info cache, "
"fallback to the default decoder selection path.";
return false;
}
*out_codec_name = software_decoder;
return true;
}
}
constexpr static float kReallocateThreshold = 3.9;
PendingDecode PendingDecode::CreateEos() {
return {DecoderBuffer::CreateEOSBuffer(), base::DoNothing()};
}
PendingDecode::PendingDecode(scoped_refptr<DecoderBuffer> buffer,
VideoDecoder::DecodeCB decode_cb)
: buffer(std::move(buffer)), decode_cb(std::move(decode_cb)) {}
PendingDecode::PendingDecode(PendingDecode&& other) = default;
PendingDecode::~PendingDecode() = default;
std::vector<SupportedVideoDecoderConfig>
MediaCodecVideoDecoder::GetSupportedConfigs() {
static const base::NoDestructor<std::vector<SupportedVideoDecoderConfig>>
configs(GenerateSupportedConfigs(
DeviceInfo::GetInstance(),
base::FeatureList::IsEnabled(
media::kAllowMediaCodecSoftwareDecoder)));
return *configs;
}
MediaCodecVideoDecoder::MediaCodecVideoDecoder(
const gpu::GpuPreferences& gpu_preferences,
bool is_surface_control_enabled,
std::unique_ptr<MediaLog> media_log,
DeviceInfo* device_info,
CodecAllocator* codec_allocator,
std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
AndroidOverlayMojoFactoryCB overlay_factory_cb,
RequestOverlayInfoCB request_overlay_info_cb,
std::unique_ptr<VideoFrameFactory> video_frame_factory,
scoped_refptr<gpu::RefCountedLock> drdc_lock)
: gpu::RefCountedLockHelperDrDc(std::move(drdc_lock)),
media_log_(std::move(media_log)),
codec_allocator_(codec_allocator),
request_overlay_info_cb_(std::move(request_overlay_info_cb)),
is_surface_control_enabled_(is_surface_control_enabled),
surface_chooser_helper_(
std::move(surface_chooser),
base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kForceVideoOverlays),
base::FeatureList::IsEnabled(media::kUseAndroidOverlayForSecureOnly),
is_surface_control_enabled_),
video_frame_factory_(std::move(video_frame_factory)),
overlay_factory_cb_(std::move(overlay_factory_cb)),
device_info_(device_info),
enable_threaded_texture_mailboxes_(
gpu_preferences.enable_threaded_texture_mailboxes),
use_block_model_(device_info_->SdkVersion() >=
base::android::android_info::SDK_VERSION_V &&
base::FeatureList::IsEnabled(kMediaCodecBlockModel)) {
DVLOG(2) << __func__;
surface_chooser_helper_.chooser()->SetClientCallbacks(
base::BindRepeating(&MediaCodecVideoDecoder::OnSurfaceChosen,
weak_factory_.GetWeakPtr()),
base::BindRepeating(&MediaCodecVideoDecoder::OnSurfaceChosen,
weak_factory_.GetWeakPtr(), nullptr));
}
std::unique_ptr<VideoDecoder> MediaCodecVideoDecoder::Create(
const gpu::GpuPreferences& gpu_preferences,
bool is_surface_control_enabled,
std::unique_ptr<MediaLog> media_log,
DeviceInfo* device_info,
CodecAllocator* codec_allocator,
std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser,
AndroidOverlayMojoFactoryCB overlay_factory_cb,
RequestOverlayInfoCB request_overlay_info_cb,
std::unique_ptr<VideoFrameFactory> video_frame_factory,
scoped_refptr<gpu::RefCountedLock> drdc_lock) {
auto* decoder = new MediaCodecVideoDecoder(
gpu_preferences, is_surface_control_enabled, std::move(media_log),
device_info, codec_allocator, std::move(surface_chooser),
std::move(overlay_factory_cb), std::move(request_overlay_info_cb),
std::move(video_frame_factory), std::move(drdc_lock));
return std::make_unique<AsyncDestroyVideoDecoder<MediaCodecVideoDecoder>>(
base::WrapUnique(decoder));
}
MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
TRACE_EVENT0("media", "MediaCodecVideoDecoder::~MediaCodecVideoDecoder");
ReleaseCodec();
}
void MediaCodecVideoDecoder::DestroyAsync(
std::unique_ptr<MediaCodecVideoDecoder> decoder) {
DVLOG(1) << __func__;
TRACE_EVENT0("media", "MediaCodecVideoDecoder::Destroy");
DCHECK(decoder);
auto* self = decoder.release();
self->weak_factory_.InvalidateWeakPtrs();
if (self->media_crypto_context_) {
self->event_cb_registration_.reset();
self->media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback());
self->media_crypto_context_ = nullptr;
}
if (self->reset_cb_)
std::move(self->reset_cb_).Run();
self->codec_allocator_weak_factory_.InvalidateWeakPtrs();
self->CancelPendingDecodes(DecoderStatus::Codes::kAborted);
self->StartDrainingCodec(DrainType::kForDestroy);
DCHECK(!self->drain_type_.has_value());
}
void MediaCodecVideoDecoder::Initialize(const VideoDecoderConfig& config,
bool low_delay,
CdmContext* cdm_context,
InitCB init_cb,
const OutputCB& output_cb,
const WaitingCB& waiting_cb) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(output_cb);
DCHECK(waiting_cb);
const bool first_init = !decoder_config_.IsValidConfig();
DVLOG(1) << (first_init ? "Initializing" : "Reinitializing")
<< " MCVD with config: " << config.AsHumanReadableString()
<< ", cdm_context = " << cdm_context;
if (!config.IsValidConfig()) {
MEDIA_LOG(INFO, media_log_) << "Video configuration is not valid: "
<< config.AsHumanReadableString();
DVLOG(1) << "Invalid configuration.";
base::BindPostTaskToCurrentDefault(std::move(init_cb))
.Run(DecoderStatus::Codes::kUnsupportedConfig);
return;
}
if (!IsVideoDecoderConfigSupported(GetSupportedConfigsInternal(), config) &&
IsDecoderBuiltInVideoCodec(config.codec())) {
MEDIA_LOG(INFO, media_log_) << "Video configuration is not valid: "
<< config.AsHumanReadableString();
base::BindPostTaskToCurrentDefault(std::move(init_cb))
.Run(DecoderStatus::Codes::kUnsupportedConfig);
return;
}
if (!first_init && decoder_config_.codec() != config.codec()) {
DVLOG(1) << "Codec changed: cannot reinitialize";
MEDIA_LOG(INFO, media_log_) << "Cannot change codec during re-init: "
<< decoder_config_.AsHumanReadableString()
<< " -> " << config.AsHumanReadableString();
base::BindPostTaskToCurrentDefault(std::move(init_cb))
.Run(DecoderStatus::Codes::kCantChangeCodec);
return;
}
const auto old_size = decoder_config_.coded_size();
decoder_config_ = config;
low_delay_ = low_delay;
surface_chooser_helper_.SetVideoRotation(
decoder_config_.video_transformation().rotation);
output_cb_ = output_cb;
waiting_cb_ = waiting_cb;
#if BUILDFLAG(USE_PROPRIETARY_CODECS)
if (config.codec() == VideoCodec::kH264)
ExtractSpsAndPps(config.extra_data(), &csd0_, &csd1_);
#endif
const int width = decoder_config_.coded_size().width();
if (first_init && cdm_context && cdm_context->GetMediaCryptoContext()) {
DCHECK(media_crypto_.is_null());
last_width_ = width;
SetCdm(cdm_context, std::move(init_cb));
return;
}
if (config.is_encrypted() && media_crypto_.is_null()) {
DVLOG(1) << "No MediaCrypto to handle encrypted config";
MEDIA_LOG(INFO, media_log_) << "No MediaCrypto to handle encrypted config";
base::BindPostTaskToCurrentDefault(std::move(init_cb))
.Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
return;
}
base::BindPostTaskToCurrentDefault(std::move(init_cb))
.Run(DecoderStatus::Codes::kOk);
if (first_init) {
last_width_ = width;
} else if (CodecNeedsReallocation(width)) {
MEDIA_LOG(INFO, media_log_)
<< "Queuing deferred codec reallocation for resolution change from "
<< old_size.ToString() << " to "
<< decoder_config_.coded_size().ToString();
deferred_flush_pending_ = true;
deferred_reallocation_pending_ = true;
should_retry_codec_allocation_ = true;
last_width_ = width;
} else {
MEDIA_LOG(INFO, media_log_)
<< "Reusing codec without reallocation for resolution change from "
<< old_size.ToString() << " to "
<< decoder_config_.coded_size().ToString();
}
}
void MediaCodecVideoDecoder::SetCdm(CdmContext* cdm_context, InitCB init_cb) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__;
DCHECK(cdm_context) << "No CDM provided";
DCHECK(cdm_context->GetMediaCryptoContext());
media_crypto_context_ = cdm_context->GetMediaCryptoContext();
event_cb_registration_ = cdm_context->RegisterEventCB(base::BindRepeating(
&MediaCodecVideoDecoder::OnCdmContextEvent, weak_factory_.GetWeakPtr()));
media_crypto_context_->SetMediaCryptoReadyCB(
base::BindPostTaskToCurrentDefault(
base::BindOnce(&MediaCodecVideoDecoder::OnMediaCryptoReady,
weak_factory_.GetWeakPtr(), std::move(init_cb))));
}
void MediaCodecVideoDecoder::OnMediaCryptoReady(
InitCB init_cb,
base::android::ScopedJavaGlobalRef<jobject> media_crypto,
bool requires_secure_video_codec) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(1) << __func__
<< ": requires_secure_video_codec = " << requires_secure_video_codec;
DCHECK(state_ == State::kInitializing);
if (!media_crypto) {
media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback());
media_crypto_context_ = nullptr;
if (decoder_config_.is_encrypted()) {
LOG(ERROR) << "MediaCrypto is not available";
EnterTerminalState(State::kError, {DecoderStatus::Codes::kFailed,
"MediaCrypto is not available"});
std::move(init_cb).Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
return;
}
std::move(init_cb).Run(DecoderStatus::Codes::kOk);
return;
}
media_crypto_ = std::move(media_crypto);
requires_secure_codec_ = requires_secure_video_codec;
surface_chooser_helper_.SetSecureSurfaceMode(
requires_secure_video_codec
? SurfaceChooserHelper::SecureSurfaceMode::kRequired
: SurfaceChooserHelper::SecureSurfaceMode::kRequested);
std::move(init_cb).Run(DecoderStatus::Codes::kOk);
}
void MediaCodecVideoDecoder::OnCdmContextEvent(CdmContext::Event event) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
if (event != CdmContext::Event::kHasAdditionalUsableKey)
return;
waiting_for_key_ = false;
PumpCodec();
}
void MediaCodecVideoDecoder::StartLazyInit() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
TRACE_EVENT0("media", "MediaCodecVideoDecoder::StartLazyInit");
lazy_init_pending_ = false;
const bool want_promotion_hints = !enable_threaded_texture_mailboxes_;
VideoFrameFactory::OverlayMode overlay_mode =
VideoFrameFactory::OverlayMode::kDontRequestPromotionHints;
if (is_surface_control_enabled_) {
overlay_mode =
requires_secure_codec_
? VideoFrameFactory::OverlayMode::kSurfaceControlSecure
: VideoFrameFactory::OverlayMode::kSurfaceControlInsecure;
} else if (want_promotion_hints) {
overlay_mode = VideoFrameFactory::OverlayMode::kRequestPromotionHints;
}
video_frame_factory_->Initialize(
overlay_mode, base::BindRepeating(
&MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized,
weak_factory_.GetWeakPtr()));
}
void MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized(
scoped_refptr<gpu::TextureOwner> texture_owner) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
TRACE_EVENT0("media",
"MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized");
if (!texture_owner) {
EnterTerminalState(State::kError, {DecoderStatus::Codes::kFailed,
"Could not allocated TextureOwner"});
return;
}
texture_owner_bundle_ =
new CodecSurfaceBundle(std::move(texture_owner), GetDrDcLock());
if (enable_threaded_texture_mailboxes_) {
OnSurfaceChosen(nullptr);
return;
}
std::move(request_overlay_info_cb_)
.Run(base::BindRepeating(&MediaCodecVideoDecoder::OnOverlayInfoChanged,
weak_factory_.GetWeakPtr()));
}
void MediaCodecVideoDecoder::OnOverlayInfoChanged(
const OverlayInfo& overlay_info) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
DCHECK(!enable_threaded_texture_mailboxes_);
if (InTerminalState())
return;
bool overlay_changed = !overlay_info_.RefersToSameOverlayAs(overlay_info);
overlay_info_ = overlay_info;
surface_chooser_helper_.SetIsFullscreen(overlay_info_.is_fullscreen);
surface_chooser_helper_.SetIsPersistentVideo(
overlay_info_.is_persistent_video);
surface_chooser_helper_.UpdateChooserState(
overlay_changed ? std::make_optional(CreateOverlayFactoryCb())
: std::nullopt);
}
void MediaCodecVideoDecoder::OnSurfaceChosen(
std::unique_ptr<AndroidOverlay> overlay) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
TRACE_EVENT1("media", "MediaCodecVideoDecoder::OnSurfaceChosen", "overlay",
overlay ? "yes" : "no");
if (overlay) {
overlay->AddSurfaceDestroyedCallback(
base::BindOnce(&MediaCodecVideoDecoder::OnSurfaceDestroyed,
weak_factory_.GetWeakPtr()));
target_surface_bundle_ = new CodecSurfaceBundle(std::move(overlay));
} else {
target_surface_bundle_ = texture_owner_bundle_;
}
if (state_ == State::kInitializing) {
state_ = State::kRunning;
CreateCodec();
}
}
void MediaCodecVideoDecoder::OnSurfaceDestroyed(AndroidOverlay* overlay) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
DCHECK_NE(state_, State::kInitializing);
TRACE_EVENT0("media", "MediaCodecVideoDecoder::OnSurfaceDestroyed");
if (target_surface_bundle_ && target_surface_bundle_->overlay() == overlay)
target_surface_bundle_ = texture_owner_bundle_;
if (requires_secure_codec_ &&
target_surface_bundle_ == texture_owner_bundle_) {
DVLOG(2) << "surface destroyed for secure codec, resetting MediaCodec.";
video_frame_factory_->SetSurfaceBundle(nullptr);
ReleaseCodec();
waiting_cb_.Run(WaitingReason::kSecureSurfaceLost);
return;
}
if (SurfaceTransitionPending())
TransitionToTargetSurface();
}
bool MediaCodecVideoDecoder::SurfaceTransitionPending() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return codec_ && codec_->SurfaceBundle() != target_surface_bundle_;
}
void MediaCodecVideoDecoder::TransitionToTargetSurface() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
DCHECK(SurfaceTransitionPending());
if (!codec_->SetSurface(target_surface_bundle_)) {
video_frame_factory_->SetSurfaceBundle(nullptr);
EnterTerminalState(State::kError,
{DecoderStatus::Codes::kFailed,
"Could not switch codec output surface"});
return;
}
video_frame_factory_->SetSurfaceBundle(target_surface_bundle_);
CacheFrameInformation();
}
void MediaCodecVideoDecoder::CreateCodec() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(!codec_);
DCHECK(target_surface_bundle_);
DCHECK_EQ(state_, State::kRunning);
auto config = std::make_unique<VideoCodecConfig>();
if (requires_secure_codec_)
config->codec_type = CodecType::kSecure;
config->codec = decoder_config_.codec();
config->csd0 = csd0_;
config->csd1 = csd1_;
config->surface = target_surface_bundle_->GetJavaSurface();
config->media_crypto = media_crypto_;
config->initial_expected_coded_size = decoder_config_.coded_size();
config->container_color_space = decoder_config_.color_space_info();
config->hdr_metadata = decoder_config_.hdr_metadata();
config->use_block_model = use_block_model_;
config->profile = decoder_config_.profile();
bool found_codec = false;
if (base::FeatureList::IsEnabled(kMediaCodecLowDelayMode) && low_delay_) {
found_codec = SelectMediaCodec(decoder_config_, requires_secure_codec_,
low_delay_, &config->name);
if (found_codec) {
config->use_low_latency_mode = low_delay_;
} else {
low_delay_ = false;
}
}
if (!found_codec) {
SelectMediaCodec(decoder_config_, requires_secure_codec_,
false, &config->name);
}
config->on_buffers_available_cb =
base::BindPostTaskToCurrentDefault(base::BindRepeating(
&MediaCodecVideoDecoder::PumpCodec, weak_factory_.GetWeakPtr()));
video_frame_factory_->SetSurfaceBundle(target_surface_bundle_);
codec_allocator_->CreateMediaCodecAsync(
base::BindOnce(&MediaCodecVideoDecoder::OnCodecConfiguredInternal,
codec_allocator_weak_factory_.GetWeakPtr(),
codec_allocator_, target_surface_bundle_),
std::move(config));
}
void MediaCodecVideoDecoder::OnCodecConfiguredInternal(
base::WeakPtr<MediaCodecVideoDecoder> weak_this,
CodecAllocator* codec_allocator,
scoped_refptr<CodecSurfaceBundle> surface_bundle,
std::unique_ptr<MediaCodecBridge> codec) {
if (!weak_this) {
if (codec) {
codec_allocator->ReleaseMediaCodec(
std::move(codec),
base::BindOnce(
&base::SequencedTaskRunner::ReleaseSoon<CodecSurfaceBundle>,
base::SequencedTaskRunner::GetCurrentDefault(), FROM_HERE,
std::move(surface_bundle)));
}
return;
}
weak_this->OnCodecConfigured(std::move(surface_bundle), std::move(codec));
}
void MediaCodecVideoDecoder::OnCodecConfigured(
scoped_refptr<CodecSurfaceBundle> surface_bundle,
std::unique_ptr<MediaCodecBridge> codec) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(!codec_);
DCHECK_EQ(state_, State::kRunning);
bool should_retry_codec_allocation = should_retry_codec_allocation_;
should_retry_codec_allocation_ = false;
if (!codec && should_retry_codec_allocation &&
device_info_->SdkVersion() >=
base::android::android_info::SDK_VERSION_R &&
device_info_->SdkVersion() <= 32
) {
CreateCodec();
return;
}
if (!codec) {
EnterTerminalState(State::kError, {DecoderStatus::Codes::kFailed,
"Unable to allocate codec"});
return;
}
codec_name_ = codec->GetName();
MEDIA_LOG(INFO, media_log_)
<< "Created MediaCodec " << codec_name_
<< ", is_software_codec=" << codec->IsSoftwareCodec()
<< ", use_block_model_=" << use_block_model_ << ", low_delay_="
<< (base::FeatureList::IsEnabled(kMediaCodecLowDelayMode) && low_delay_);
std::optional<gfx::Size> coded_size_alignment =
MediaCodecUtil::LookupCodedSizeAlignment(codec_name_);
if (coded_size_alignment) {
MEDIA_LOG(INFO, media_log_) << "Using a coded size alignment of "
<< coded_size_alignment->ToString();
} else {
MEDIA_LOG(WARNING, media_log_)
<< "Unable to lookup coded size alignment for codec " << codec_name_;
}
max_input_size_ = codec->GetMaxInputSize();
codec_ = std::make_unique<CodecWrapper>(
CodecSurfacePair(std::move(codec), std::move(surface_bundle)),
base::BindRepeating(
&OutputBufferReleased,
base::BindPostTaskToCurrentDefault(base::BindRepeating(
&MediaCodecVideoDecoder::PumpCodec, weak_factory_.GetWeakPtr()))),
decoder_config_.coded_size(),
decoder_config_.color_space_info().ToGfxColorSpace(),
coded_size_alignment);
if (SurfaceTransitionPending())
TransitionToTargetSurface();
CacheFrameInformation();
PumpCodec();
}
void MediaCodecVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
DecodeCB decode_cb) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << ": " << buffer->AsHumanReadableString();
if (state_ == State::kError) {
std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
return;
}
if (!DecoderBuffer::DoSubsamplesMatch(*buffer)) {
std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
return;
}
pending_decodes_.emplace_back(std::move(buffer), std::move(decode_cb));
if (state_ == State::kInitializing) {
if (lazy_init_pending_)
StartLazyInit();
return;
}
PumpCodec();
}
void MediaCodecVideoDecoder::FlushCodec() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
deferred_flush_pending_ = false;
if (deferred_reallocation_pending_) {
deferred_reallocation_pending_ = false;
ReleaseCodec();
should_retry_codec_allocation_ = !SurfaceTransitionPending();
CreateCodec();
}
if (!codec_ || codec_->IsFlushed())
return;
DVLOG(2) << "Flushing codec";
if (!codec_->Flush())
EnterTerminalState(State::kError,
{DecoderStatus::Codes::kFailed, "Codec flush failed"});
}
void MediaCodecVideoDecoder::PumpCodec() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(4) << __func__;
if (state_ != State::kRunning) {
return;
}
bool did_input = false, did_output = false;
do {
did_input = QueueInput();
did_output = DequeueOutput();
} while (did_input || did_output);
}
bool MediaCodecVideoDecoder::QueueInput() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(4) << __func__;
if (!codec_ || waiting_for_key_)
return false;
if (codec_->IsDrained() || deferred_flush_pending_) {
if (!codec_->HasUnreleasedOutputBuffers() && !pending_decodes_.empty()) {
FlushCodec();
return true;
}
return false;
}
if (pending_decodes_.empty())
return false;
auto pending_buffer = pending_decodes_.front().buffer;
if (!use_block_model_ && !pending_buffer->end_of_stream() &&
pending_buffer->is_key_frame() &&
pending_buffer->size() > max_input_size_) {
if (decoder_config_.coded_size().width() == last_width_) {
const size_t compression_ratio =
(decoder_config_.codec() == VideoCodec::kH264 ||
decoder_config_.codec() == VideoCodec::kVP8)
? 2
: 4;
const size_t max_pixels =
(pending_buffer->size() * compression_ratio * 2) / 3;
if (max_pixels > 8294400)
decoder_config_.set_coded_size(gfx::Size(7680, 4320));
else if (max_pixels > 2088960)
decoder_config_.set_coded_size(gfx::Size(3840, 2160));
else
decoder_config_.set_coded_size(gfx::Size(1920, 1080));
}
if (decoder_config_.coded_size().width() != last_width_) {
deferred_flush_pending_ = true;
deferred_reallocation_pending_ = true;
last_width_ = decoder_config_.coded_size().width();
return true;
}
}
if (pending_buffer->end_of_stream() && pending_buffer->next_config()) {
const auto new_config =
std::get<VideoDecoderConfig>(*pending_buffer->next_config());
const bool can_reuse_codec = [&]() {
std::string codec_name;
SelectMediaCodec(new_config, requires_secure_codec_, low_delay_,
&codec_name);
return !codec_name_.empty() && codec_name == codec_name_ &&
!CodecNeedsReallocation(new_config.coded_size().width());
}();
if (can_reuse_codec) {
MEDIA_LOG(INFO, media_log_)
<< "Eliding EOS buffer and flush for resolution change from "
<< decoder_config_.coded_size().ToString() << " to "
<< new_config.coded_size().ToString();
std::move(pending_decodes_.front().decode_cb)
.Run(DecoderStatus::Codes::kElidedEndOfStreamForConfigChange);
pending_decodes_.pop_front();
DCHECK(pending_decodes_.empty());
decoder_config_ = new_config;
return true;
}
}
auto status = codec_->QueueInputBuffer(*pending_buffer);
DVLOG((status.code() == CodecWrapper::QueueStatus::Codes::kTryAgainLater ||
status.is_ok()
? 3
: 2))
<< "QueueInput(" << pending_buffer->AsHumanReadableString()
<< ") status=" << status.message();
switch (status.code()) {
case CodecWrapper::QueueStatus::Codes::kOk:
break;
case CodecWrapper::QueueStatus::Codes::kTryAgainLater:
return false;
case CodecWrapper::QueueStatus::Codes::kNoKey:
waiting_for_key_ = true;
waiting_cb_.Run(WaitingReason::kNoDecryptionKey);
return false;
case CodecWrapper::QueueStatus::Codes::kError:
EnterTerminalState(State::kError,
{DecoderStatus::Codes::kFailed,
"QueueInputBuffer failed", std::move(status)});
return false;
}
if (pending_buffer->end_of_stream()) {
DCHECK(!eos_decode_cb_);
eos_decode_cb_ = std::move(pending_decodes_.front().decode_cb);
} else {
std::move(pending_decodes_.front().decode_cb)
.Run(DecoderStatus::Codes::kOk);
}
pending_decodes_.pop_front();
return true;
}
bool MediaCodecVideoDecoder::DequeueOutput() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(4) << __func__;
if (!codec_ || codec_->IsDrained() || waiting_for_key_)
return false;
if (SurfaceTransitionPending()) {
if (!codec_->HasUnreleasedOutputBuffers()) {
TransitionToTargetSurface();
return true;
}
return false;
}
base::TimeDelta presentation_time;
bool eos = false;
std::unique_ptr<CodecOutputBuffer> output_buffer;
auto status =
codec_->DequeueOutputBuffer(&presentation_time, &eos, &output_buffer);
switch (status.code()) {
case CodecWrapper::DequeueStatus::Codes::kOk:
break;
case CodecWrapper::DequeueStatus::Codes::kTryAgainLater:
return false;
case CodecWrapper::DequeueStatus::Codes::kError:
DVLOG(1) << "DequeueOutputBuffer() error";
EnterTerminalState(State::kError,
{DecoderStatus::Codes::kFailed,
"DequeueOutputBuffer failed", std::move(status)});
return false;
}
DVLOG(3) << "DequeueOutputBuffer(): pts="
<< (eos ? "EOS"
: base::NumberToString(presentation_time.InMilliseconds()));
if (eos) {
if (eos_decode_cb_) {
video_frame_factory_->RunAfterPendingVideoFrames(
base::BindOnce(&MediaCodecVideoDecoder::RunEosDecodeCb,
weak_factory_.GetWeakPtr(), reset_generation_));
}
if (drain_type_)
OnCodecDrained();
return false;
}
if (drain_type_ || deferred_flush_pending_)
return true;
if (output_buffer->size().GetArea() > decoder_config_.coded_size().GetArea())
decoder_config_.set_coded_size(output_buffer->size());
gfx::Rect visible_rect(output_buffer->size());
std::unique_ptr<ScopedAsyncTrace> async_trace =
ScopedAsyncTrace::CreateIfEnabled(
"MediaCodecVideoDecoder::CreateVideoFrame");
if (!is_surface_control_enabled_) {
output_buffer->set_render_cb(
base::BindPostTaskToCurrentDefault(base::BindOnce(
&MediaCodecVideoDecoder::PumpCodec, weak_factory_.GetWeakPtr())));
}
video_frame_factory_->CreateVideoFrame(
std::move(output_buffer), presentation_time,
decoder_config_.aspect_ratio().GetNaturalSize(visible_rect),
CreatePromotionHintCB(),
base::BindOnce(&MediaCodecVideoDecoder::ForwardVideoFrame,
weak_factory_.GetWeakPtr(), reset_generation_,
std::move(async_trace), base::TimeTicks::Now()));
return true;
}
void MediaCodecVideoDecoder::RunEosDecodeCb(int reset_generation) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (reset_generation == reset_generation_ && eos_decode_cb_)
std::move(eos_decode_cb_).Run(DecoderStatus::Codes::kOk);
}
void MediaCodecVideoDecoder::ForwardVideoFrame(
int reset_generation,
std::unique_ptr<ScopedAsyncTrace> async_trace,
base::TimeTicks started_at,
scoped_refptr<VideoFrame> frame) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(3) << __func__ << " : "
<< (frame ? frame->AsHumanReadableString() : "null");
if (!frame) {
DLOG(ERROR) << __func__ << " |frame| is null";
EnterTerminalState(State::kError, {DecoderStatus::Codes::kFailed,
"Could not create VideoFrame"});
return;
}
if (frame->ColorSpace().IsHDR() && decoder_config_.hdr_metadata()) {
frame->set_hdr_metadata(decoder_config_.hdr_metadata());
}
if (media_crypto_context_) {
frame->metadata().protected_video = true;
if (requires_secure_codec_) {
frame->metadata().hw_protected = true;
}
}
if (reset_generation == reset_generation_) {
frame->metadata().power_efficient = true;
output_cb_.Run(std::move(frame));
}
}
void MediaCodecVideoDecoder::Reset(base::OnceClosure closure) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
DCHECK(!reset_cb_);
reset_generation_++;
reset_cb_ = std::move(closure);
CancelPendingDecodes(DecoderStatus::Codes::kAborted);
StartDrainingCodec(DrainType::kForReset);
}
void MediaCodecVideoDecoder::StartDrainingCodec(DrainType drain_type) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
TRACE_EVENT0("media", "MediaCodecVideoDecoder::StartDrainingCodec");
DCHECK(pending_decodes_.empty());
drain_type_ = drain_type;
if (codec_)
codec_->DiscardOutputBuffers();
deferred_flush_pending_ =
codec_ && !codec_->IsDrained() && !codec_->IsFlushed();
OnCodecDrained();
}
void MediaCodecVideoDecoder::OnCodecDrained() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__;
TRACE_EVENT0("media", "MediaCodecVideoDecoder::OnCodecDrained");
DrainType drain_type = *drain_type_;
drain_type_.reset();
if (drain_type == DrainType::kForDestroy) {
base::SequencedTaskRunner::GetCurrentDefault()->DeleteSoon(FROM_HERE, this);
return;
}
std::move(reset_cb_).Run();
if (codec_ && !codec_->IsFlushed() && !codec_->IsDrained() &&
!deferred_flush_pending_) {
FlushCodec();
}
}
void MediaCodecVideoDecoder::EnterTerminalState(State state,
DecoderStatus reason) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__ << " " << static_cast<int>(state) << " "
<< reason.message();
state_ = state;
DCHECK(InTerminalState());
codec_allocator_weak_factory_.InvalidateWeakPtrs();
ReleaseCodec();
target_surface_bundle_ = nullptr;
texture_owner_bundle_ = nullptr;
if (state == State::kError) {
CancelPendingDecodes(reason);
} else {
MEDIA_LOG(INFO, media_log_)
<< "Entering Terminal State: " << reason.message();
}
if (drain_type_) {
OnCodecDrained();
}
}
bool MediaCodecVideoDecoder::InTerminalState() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return state_ == State::kSurfaceDestroyed || state_ == State::kError;
}
void MediaCodecVideoDecoder::CancelPendingDecodes(DecoderStatus status) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
for (auto& pending_decode : pending_decodes_)
std::move(pending_decode.decode_cb).Run(status);
pending_decodes_.clear();
if (eos_decode_cb_)
std::move(eos_decode_cb_).Run(status);
}
void MediaCodecVideoDecoder::ReleaseCodec() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!codec_)
return;
auto pair = codec_->TakeCodecSurfacePair();
codec_ = nullptr;
codec_allocator_->ReleaseMediaCodec(
std::move(pair.first),
base::BindOnce(
&base::SequencedTaskRunner::ReleaseSoon<CodecSurfaceBundle>,
base::SequencedTaskRunner::GetCurrentDefault(), FROM_HERE,
std::move(pair.second)));
}
AndroidOverlayFactoryCB MediaCodecVideoDecoder::CreateOverlayFactoryCb() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!overlay_factory_cb_ || !overlay_info_.HasValidRoutingToken())
return AndroidOverlayFactoryCB();
return base::BindRepeating(overlay_factory_cb_, *overlay_info_.routing_token);
}
VideoDecoderType MediaCodecVideoDecoder::GetDecoderType() const {
return VideoDecoderType::kMediaCodec;
}
bool MediaCodecVideoDecoder::NeedsBitstreamConversion() const {
return true;
}
bool MediaCodecVideoDecoder::CanReadWithoutStalling() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
const auto buffer_count =
codec_ ? codec_->GetUnreleasedOutputBufferCount() : 0;
return !video_frame_factory_->IsStalled() && buffer_count < 2;
}
int MediaCodecVideoDecoder::GetMaxDecodeRequests() const {
return 2;
}
PromotionHintAggregator::NotifyPromotionHintCB
MediaCodecVideoDecoder::CreatePromotionHintCB() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return base::BindPostTaskToCurrentDefault(base::BindRepeating(
[](base::WeakPtr<MediaCodecVideoDecoder> mcvd,
CodecSurfaceBundle::ScheduleLayoutCB layout_cb,
PromotionHintAggregator::Hint hint) {
if (hint.is_promotable)
layout_cb.Run(hint.screen_rect);
if (mcvd)
mcvd->NotifyPromotionHint(hint);
},
weak_factory_.GetWeakPtr(),
codec_->SurfaceBundle()->GetScheduleLayoutCB()));
}
bool MediaCodecVideoDecoder::IsUsingOverlay() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return codec_ && codec_->SurfaceBundle() &&
codec_->SurfaceBundle()->overlay();
}
void MediaCodecVideoDecoder::NotifyPromotionHint(
PromotionHintAggregator::Hint hint) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
surface_chooser_helper_.NotifyPromotionHintAndUpdateChooser(hint,
IsUsingOverlay());
}
void MediaCodecVideoDecoder::CacheFrameInformation() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
cached_frame_information_ =
surface_chooser_helper_.ComputeFrameInformation(IsUsingOverlay());
}
bool MediaCodecVideoDecoder::CodecNeedsReallocation(int new_width) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return !use_block_model_ && new_width > last_width_ * kReallocateThreshold &&
device_info_ &&
device_info_->SdkVersion() >
base::android::android_info::SDK_VERSION_P;
}
std::vector<SupportedVideoDecoderConfig>
MediaCodecVideoDecoder::GetSupportedConfigsInternal() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (device_info_ != DeviceInfo::GetInstance()) {
return GenerateSupportedConfigs(device_info_,
true);
}
return GetSupportedConfigs();
}
}