#ifdef UNSAFE_BUFFERS_BUILD
#pragma allow_unsafe_buffers
#endif
#include "media/gpu/v4l2/v4l2_stateful_video_decoder.h"
#include <fcntl.h>
#include <libdrm/drm_fourcc.h>
#include <poll.h>
#include <sys/eventfd.h>
#include <sys/ioctl.h>
#include "base/containers/contains.h"
#include "base/containers/heap_array.h"
#include "base/location.h"
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram_macros.h"
#include "base/notimplemented.h"
#include "base/posix/eintr_wrapper.h"
#include "base/task/bind_post_task.h"
#include "base/task/sequenced_task_runner.h"
#include "base/task/thread_pool.h"
#include "base/trace_event/trace_event.h"
#include "media/base/media_log.h"
#include "media/base/media_switches.h"
#include "media/gpu/chromeos/video_frame_resource.h"
#include "media/gpu/macros.h"
#include "media/gpu/v4l2/v4l2_device.h"
#include "media/gpu/v4l2/v4l2_framerate_control.h"
#include "media/gpu/v4l2/v4l2_queue.h"
#include "media/gpu/v4l2/v4l2_utils.h"
#include "media/parsers/h264_parser.h"
#include "third_party/abseil-cpp/absl/cleanup/cleanup.h"
#include "ui/gfx/geometry/size.h"
namespace {
constexpr int kIoctlOk = 0;
int HandledIoctl(int fd, int request, void* arg) {
return HANDLE_EINTR(ioctl(fd, request, arg));
}
void* Mmap(int fd,
void* addr,
unsigned int len,
int prot,
int flags,
unsigned int offset) {
return mmap(addr, len, prot, flags, fd, offset);
}
void WaitOnceForEvents(int device_fd,
int wake_event,
base::OnceClosure dequeue_callback,
base::OnceClosure resolution_change_callback) {
VLOGF(5) << "Going to poll()";
struct pollfd pollfds[] = {{.fd = device_fd, .events = POLLIN | POLLPRI},
{.fd = wake_event, .events = POLLIN}};
constexpr int kInfiniteTimeout = -1;
if (HANDLE_EINTR(poll(pollfds, std::size(pollfds), kInfiniteTimeout)) <
kIoctlOk) {
PLOG(ERROR) << "Poll()ing for events failed";
return;
}
const auto events_from_device = pollfds[0].revents;
const auto other_events = pollfds[1].revents;
const auto pollin_or_pollpri_event = events_from_device & (POLLIN | POLLPRI);
if (pollin_or_pollpri_event) {
if (events_from_device & POLLIN) {
std::move(dequeue_callback).Run();
}
if (events_from_device & POLLPRI) {
VLOGF(2) << "Resolution change event";
struct v4l2_event event;
memset(&event, 0, sizeof(event));
if (HandledIoctl(device_fd, VIDIOC_DQEVENT, &event) != kIoctlOk) {
PLOG(ERROR) << "Failed dequeing an event";
return;
}
DCHECK_EQ(event.type,
static_cast<unsigned int>(V4L2_EVENT_SOURCE_CHANGE));
DCHECK(event.u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION);
std::move(resolution_change_callback).Run();
}
return;
}
if (other_events & POLLIN) {
return;
}
CHECK((events_from_device & (POLLERR | POLLHUP | POLLNVAL)) ||
(other_events & (POLLERR | POLLHUP | POLLNVAL)));
VLOG(2) << "Unhandled |events_from_device|: 0x" << std::hex
<< events_from_device << ", or |other_events|: 0x" << other_events;
}
bool IsNewH264Frame(const media::H264SPS* sps,
const media::H264PPS* pps,
const media::H264SliceHeader* prev_slice_header,
const media::H264SliceHeader* curr_slice_header) {
if (curr_slice_header->frame_num != prev_slice_header->frame_num ||
curr_slice_header->pic_parameter_set_id != pps->pic_parameter_set_id ||
curr_slice_header->nal_ref_idc != prev_slice_header->nal_ref_idc ||
curr_slice_header->idr_pic_flag != prev_slice_header->idr_pic_flag ||
(curr_slice_header->idr_pic_flag &&
(curr_slice_header->idr_pic_id != prev_slice_header->idr_pic_id ||
curr_slice_header->first_mb_in_slice == 0))) {
return true;
}
if (sps->pic_order_cnt_type == 0) {
if (curr_slice_header->pic_order_cnt_lsb !=
prev_slice_header->pic_order_cnt_lsb ||
curr_slice_header->delta_pic_order_cnt_bottom !=
prev_slice_header->delta_pic_order_cnt_bottom) {
return true;
}
} else if (sps->pic_order_cnt_type == 1) {
if (curr_slice_header->delta_pic_order_cnt0 !=
prev_slice_header->delta_pic_order_cnt0 ||
curr_slice_header->delta_pic_order_cnt1 !=
prev_slice_header->delta_pic_order_cnt1) {
return true;
}
}
return false;
}
scoped_refptr<media::DecoderBuffer> ReassembleFragments(
std::vector<scoped_refptr<media::DecoderBuffer>>& fragments) {
size_t frame_size = 0;
for (const auto& fragment : fragments) {
frame_size += fragment->size();
}
auto temp_buffer = base::HeapArray<uint8_t>::Uninit(frame_size);
uint8_t* dst = temp_buffer.data();
for (const auto& fragment : fragments) {
auto fragment_span = base::span(*fragment);
memcpy(dst, fragment_span.data(), fragment_span.size());
dst += fragment_span.size();
}
auto reassembled_frame =
media::DecoderBuffer::FromArray(std::move(temp_buffer));
reassembled_frame->set_timestamp(fragments.back()->timestamp());
fragments.clear();
return reassembled_frame;
}
}
namespace media {
class H264FrameReassembler {
public:
H264FrameReassembler() = default;
~H264FrameReassembler() = default;
H264FrameReassembler(const H264FrameReassembler&) = delete;
H264FrameReassembler& operator=(const H264FrameReassembler&) = delete;
std::vector<std::pair<scoped_refptr<DecoderBuffer>, VideoDecoder::DecodeCB>>
Process(scoped_refptr<DecoderBuffer> buffer,
VideoDecoder::DecodeCB decode_cb);
scoped_refptr<DecoderBuffer> AssembleAndFlushFragments() {
return ReassembleFragments(frame_fragments_);
}
bool HasFragments() const { return !frame_fragments_.empty(); }
private:
struct FrameBoundaryInfo {
bool is_whole_frame;
bool is_start_of_new_frame;
size_t nalu_size;
};
std::optional<struct FrameBoundaryInfo> FindH264FrameBoundary(
const uint8_t* const data,
size_t size);
H264Parser h264_parser_;
static constexpr int kInvalidSPS = -1;
int sps_id_ = kInvalidSPS;
static constexpr int kInvalidPPS = -1;
int pps_id_ = kInvalidPPS;
std::unique_ptr<H264SliceHeader> previous_slice_header_;
std::vector<scoped_refptr<DecoderBuffer>> frame_fragments_;
};
base::AtomicRefCount V4L2StatefulVideoDecoder::num_decoder_instances_(0);
std::unique_ptr<VideoDecoderMixin> V4L2StatefulVideoDecoder::Create(
std::unique_ptr<MediaLog> media_log,
scoped_refptr<base::SequencedTaskRunner> task_runner,
base::WeakPtr<VideoDecoderMixin::Client> client) {
DCHECK(task_runner->RunsTasksInCurrentSequence());
DCHECK(client);
return base::WrapUnique<VideoDecoderMixin>(new V4L2StatefulVideoDecoder(
std::move(media_log), std::move(task_runner), std::move(client)));
}
void V4L2StatefulVideoDecoder::Initialize(const VideoDecoderConfig& config,
bool ,
CdmContext* cdm_context,
InitCB init_cb,
const PipelineOutputCB& output_cb,
const WaitingCB& ) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(config.IsValidConfig());
DVLOGF(1) << config.AsHumanReadableString();
if (config.is_encrypted() || !!cdm_context) {
std::move(init_cb).Run(DecoderStatus::Codes::kUnsupportedEncryptionMode);
return;
}
static const auto decoder_instances_limit =
V4L2StatefulVideoDecoder::GetMaxNumDecoderInstances();
const bool can_create_decoder =
num_decoder_instances_.Increment() < decoder_instances_limit;
if (!can_create_decoder) {
num_decoder_instances_.Decrement();
LOG(ERROR) << "Too many decoder instances, max=" << decoder_instances_limit;
std::move(init_cb).Run(DecoderStatus::Codes::kTooManyDecoders);
return;
}
if (supported_configs_.empty()) {
supported_configs_ = GetSupportedV4L2DecoderConfigs().value_or(
SupportedVideoDecoderConfigs());
DCHECK(!supported_configs_.empty());
}
if (!IsVideoDecoderConfigSupported(supported_configs_, config)) {
VLOGF(1) << "Video configuration is not supported: "
<< config.AsHumanReadableString();
MEDIA_LOG(INFO, media_log_) << "Video configuration is not supported: "
<< config.AsHumanReadableString();
std::move(init_cb).Run(DecoderStatus::Codes::kUnsupportedConfig);
return;
}
if (!device_fd_.is_valid()) {
device_fd_ = V4L2Device::OpenFDForType(V4L2Device::Type::kDecoder);
if (!device_fd_.is_valid()) {
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
wake_event_.reset(eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC));
if (!wake_event_.is_valid()) {
PLOG(ERROR) << "Failed to create an eventfd.";
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
struct v4l2_capability caps = {};
if (HandledIoctl(device_fd_.get(), VIDIOC_QUERYCAP, &caps) != kIoctlOk) {
PLOG(ERROR) << "Failed querying caps";
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
is_mtk8173_ = base::Contains(
std::string(reinterpret_cast<const char*>(caps.card)), "8173");
DVLOGF_IF(1, is_mtk8173_) << "This is an MTK8173 device (Hana, Oak)";
}
if (IsInitialized()) {
DCHECK(decoder_buffer_and_callbacks_.empty());
weak_ptr_factory_for_events_.InvalidateWeakPtrs();
weak_ptr_factory_for_CAPTURE_availability_.InvalidateWeakPtrs();
cancelable_task_tracker_.TryCancelAll();
encoding_timestamps_.clear();
if (OUTPUT_queue_ && !OUTPUT_queue_->Streamoff()) {
LOG(ERROR) << "Failed to stop (VIDIOC_STREAMOFF) |OUTPUT_queue_|.";
}
if (CAPTURE_queue_ && !CAPTURE_queue_->Streamoff()) {
LOG(ERROR) << "Failed to stop (VIDIOC_STREAMOFF) |CAPTURE_queue_|.";
}
}
framerate_control_ = std::make_unique<V4L2FrameRateControl>(
base::BindRepeating(&HandledIoctl, device_fd_.get()),
base::SequencedTaskRunner::GetCurrentDefault());
OUTPUT_queue_ = base::MakeRefCounted<V4L2Queue>(
V4L2Queue::PassKey::Get(),
base::BindRepeating(&HandledIoctl, device_fd_.get()),
base::DoNothing(),
base::BindRepeating(&Mmap, device_fd_.get()),
AllocateSecureBufferAsCallback(), V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
base::DoNothing());
const auto profile_as_v4l2_fourcc =
VideoCodecProfileToV4L2PixFmt(config.profile(), false);
constexpr size_t kMiB = 1024 * 1024;
constexpr int kFullHDNumPixels = 1920 * 1080;
const size_t kInputBufferInMBs =
(config.coded_size().GetArea() <= kFullHDNumPixels) ? 2 : 4;
const auto v4l2_format = OUTPUT_queue_->SetFormat(
profile_as_v4l2_fourcc, gfx::Size(), kInputBufferInMBs * kMiB);
if (!v4l2_format) {
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
DCHECK_EQ(v4l2_format->fmt.pix_mp.pixelformat, profile_as_v4l2_fourcc);
const bool is_h264 =
VideoCodecProfileToVideoCodec(config.profile()) == VideoCodec::kH264;
constexpr size_t kNumInputBuffersH264 = 16;
constexpr size_t kNumInputBuffersVPx = 2;
const auto num_input_buffers =
is_h264 ? kNumInputBuffersH264 : kNumInputBuffersVPx;
if (OUTPUT_queue_->AllocateBuffers(num_input_buffers, V4L2_MEMORY_MMAP,
false) <
num_input_buffers) {
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
if (!OUTPUT_queue_->Streamon()) {
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
client_->NotifyEstimatedMaxDecodeRequests(base::checked_cast<int>(
std::min(static_cast<size_t>(4), num_input_buffers)));
struct v4l2_event_subscription sub = {.type = V4L2_EVENT_SOURCE_CHANGE};
if (HandledIoctl(device_fd_.get(), VIDIOC_SUBSCRIBE_EVENT, &sub) !=
kIoctlOk) {
PLOG(ERROR) << "Failed to subscribe to V4L2_EVENT_SOURCE_CHANGE";
std::move(init_cb).Run(DecoderStatus::Codes::kFailedToCreateDecoder);
return;
}
config_ = config;
output_cb_ = std::move(output_cb);
if (is_h264) {
h264_frame_reassembler_ = std::make_unique<H264FrameReassembler>();
}
std::move(init_cb).Run(DecoderStatus::Codes::kOk);
}
void V4L2StatefulVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer,
DecodeCB decode_cb) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
VLOGF(3) << buffer->AsHumanReadableString(false);
if (!IsInitialized()) {
DecoderStatus init_result;
Initialize(
config_, false, nullptr,
base::BindOnce([](DecoderStatus* out, DecoderStatus in) { *out = in; },
&init_result),
output_cb_,
base::DoNothing());
if (!init_result.is_ok()) {
OUTPUT_queue_.reset();
std::move(decode_cb).Run(init_result);
return;
}
}
if (buffer->end_of_stream()) {
if (!event_task_runner_) {
std::move(decode_cb).Run(DecoderStatus::Codes::kOk);
return;
}
if (h264_frame_reassembler_ && h264_frame_reassembler_->HasFragments()) {
decoder_buffer_and_callbacks_.emplace(
h264_frame_reassembler_->AssembleAndFlushFragments(),
base::DoNothing());
TryAndEnqueueOUTPUTQueueBuffers();
}
const bool is_pending_work = !decoder_buffer_and_callbacks_.empty();
const bool decoding = !!CAPTURE_queue_;
if (is_pending_work || !decoding) {
decoder_buffer_and_callbacks_.emplace(std::move(buffer),
std::move(decode_cb));
return;
}
if (!OUTPUT_queue_->SendStopCommand()) {
std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
return;
}
RearmCAPTUREQueueMonitoring();
flush_cb_ = std::move(decode_cb);
return;
}
PrintAndTraceQueueStates(FROM_HERE);
if (VideoCodecProfileToVideoCodec(config_.profile()) == VideoCodec::kH264) {
auto processed_buffer_and_decode_cbs = h264_frame_reassembler_->Process(
std::move(buffer), std::move(decode_cb));
if (processed_buffer_and_decode_cbs.empty()) {
return;
}
for (auto& a : processed_buffer_and_decode_cbs) {
decoder_buffer_and_callbacks_.push(std::move(a));
}
} else if (VideoCodecProfileToVideoCodec(config_.profile()) ==
VideoCodec::kHEVC) {
NOTIMPLEMENTED();
std::move(decode_cb).Run(DecoderStatus::Codes::kUnsupportedCodec);
return;
} else {
decoder_buffer_and_callbacks_.emplace(std::move(buffer),
std::move(decode_cb));
}
if (!TryAndEnqueueOUTPUTQueueBuffers()) {
return;
}
if (!event_task_runner_) {
CHECK(!CAPTURE_queue_);
event_task_runner_ = base::ThreadPool::CreateSingleThreadTaskRunner(
{base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN},
base::SingleThreadTaskRunnerThreadMode::DEDICATED);
CHECK(event_task_runner_);
}
RearmCAPTUREQueueMonitoring();
}
void V4L2StatefulVideoDecoder::Reset(base::OnceClosure closure) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOGF(2);
absl::Cleanup scoped_trampoline_reset = [closure =
std::move(closure)]() mutable {
base::SequencedTaskRunner::GetCurrentDefault()->PostTask(
FROM_HERE, std::move(closure));
};
weak_ptr_factory_for_events_.InvalidateWeakPtrs();
weak_ptr_factory_for_CAPTURE_availability_.InvalidateWeakPtrs();
cancelable_task_tracker_.TryCancelAll();
if (wake_event_.is_valid()) {
const uint64_t buf = 1;
const auto res = HANDLE_EINTR(write(wake_event_.get(), &buf, sizeof(buf)));
PLOG_IF(ERROR, res < 0) << "Error writing to |wake_event_|";
}
if (h264_frame_reassembler_) {
h264_frame_reassembler_ = std::make_unique<H264FrameReassembler>();
}
while (!decoder_buffer_and_callbacks_.empty()) {
auto media_decode_cb =
std::move(decoder_buffer_and_callbacks_.front().second);
decoder_buffer_and_callbacks_.pop();
std::move(media_decode_cb).Run(DecoderStatus::Codes::kAborted);
}
OUTPUT_queue_.reset();
CAPTURE_queue_.reset();
device_fd_.reset();
event_task_runner_.reset();
num_decoder_instances_.Decrement();
encoding_timestamps_.clear();
if (flush_cb_) {
std::move(flush_cb_).Run(DecoderStatus::Codes::kAborted);
}
}
bool V4L2StatefulVideoDecoder::NeedsBitstreamConversion() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NOTREACHED() << "Our only owner VideoDecoderPipeline never calls here";
}
bool V4L2StatefulVideoDecoder::CanReadWithoutStalling() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NOTREACHED() << "Our only owner VideoDecoderPipeline never calls here";
}
int V4L2StatefulVideoDecoder::GetMaxDecodeRequests() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NOTREACHED() << "Our only owner VideoDecoderPipeline never calls here";
}
VideoDecoderType V4L2StatefulVideoDecoder::GetDecoderType() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NOTREACHED() << "Our only owner VideoDecoderPipeline never calls here";
}
bool V4L2StatefulVideoDecoder::IsPlatformDecoder() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NOTREACHED() << "Our only owner VideoDecoderPipeline never calls here";
}
void V4L2StatefulVideoDecoder::ApplyResolutionChange() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOGF(2);
if (IsInitialized())
InitializeCAPTUREQueue();
}
size_t V4L2StatefulVideoDecoder::GetMaxOutputFramePoolSize() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return VIDEO_MAX_FRAME;
}
void V4L2StatefulVideoDecoder::SetDmaIncoherentV4L2(bool incoherent) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
NOTIMPLEMENTED();
}
V4L2StatefulVideoDecoder::V4L2StatefulVideoDecoder(
std::unique_ptr<MediaLog> media_log,
scoped_refptr<base::SequencedTaskRunner> task_runner,
base::WeakPtr<VideoDecoderMixin::Client> client)
: VideoDecoderMixin(std::move(media_log),
std::move(task_runner),
std::move(client)),
weak_ptr_factory_for_events_(this),
weak_ptr_factory_for_CAPTURE_availability_(this) {
DCHECK(decoder_task_runner_->RunsTasksInCurrentSequence());
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOGF(1);
}
V4L2StatefulVideoDecoder::~V4L2StatefulVideoDecoder() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOGF(1);
weak_ptr_factory_for_events_.InvalidateWeakPtrs();
weak_ptr_factory_for_CAPTURE_availability_.InvalidateWeakPtrs();
cancelable_task_tracker_.TryCancelAll();
if (wake_event_.is_valid()) {
const uint64_t buf = 1;
const auto res = HANDLE_EINTR(write(wake_event_.get(), &buf, sizeof(buf)));
PLOG_IF(ERROR, res < 0) << "Error writing to |wake_event_|";
}
CAPTURE_queue_.reset();
OUTPUT_queue_.reset();
num_decoder_instances_.Decrement();
if (event_task_runner_) {
event_task_runner_->PostTask(
FROM_HERE,
base::BindOnce([](base::ScopedFD fd) {}, std::move(device_fd_)));
event_task_runner_->PostTask(
FROM_HERE,
base::BindOnce([](base::ScopedFD fd) {}, std::move(wake_event_)));
}
}
bool V4L2StatefulVideoDecoder::InitializeCAPTUREQueue() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(IsInitialized()) << "V4L2StatefulVideoDecoder must be Initialize()d";
CAPTURE_queue_ = base::MakeRefCounted<V4L2Queue>(
V4L2Queue::PassKey::Get(),
base::BindRepeating(&HandledIoctl, device_fd_.get()),
base::DoNothing(),
base::BindRepeating(&Mmap, device_fd_.get()),
AllocateSecureBufferAsCallback(), V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
base::DoNothing());
const auto v4l2_format_or_error = CAPTURE_queue_->GetFormat();
if (!v4l2_format_or_error.first || v4l2_format_or_error.second != kIoctlOk) {
return false;
}
const struct v4l2_format v4l2_format = *(v4l2_format_or_error.first);
VLOG(3) << "Out-of-the-box |CAPTURE_queue_| configuration: "
<< V4L2FormatToString(v4l2_format);
const gfx::Size coded_size(v4l2_format.fmt.pix_mp.width,
v4l2_format.fmt.pix_mp.height);
std::vector<ImageProcessor::PixelLayoutCandidate> candidates =
EnumeratePixelLayoutCandidates(coded_size);
std::optional<gfx::Rect> visible_rect = CAPTURE_queue_->GetVisibleRect();
if (!visible_rect) {
return false;
}
CHECK(gfx::Rect(coded_size).Contains(*visible_rect));
visible_rect_ = *visible_rect;
const auto num_codec_reference_frames = GetNumberOfReferenceFrames();
CroStatus::Or<ImageProcessor::PixelLayoutCandidate> status_or_output_format =
client_->PickDecoderOutputFormat(
candidates, *visible_rect,
config_.aspect_ratio().GetNaturalSize(*visible_rect),
std::nullopt, num_codec_reference_frames,
false, false,
std::nullopt);
if (!status_or_output_format.has_value()) {
return false;
}
const ImageProcessor::PixelLayoutCandidate output_format =
std::move(status_or_output_format).value();
auto chosen_fourcc = output_format.fourcc;
const auto chosen_size = output_format.size;
const auto chosen_modifier = output_format.modifier;
const bool use_v4l2_allocated_buffers = !client_->GetVideoFramePool();
const v4l2_memory buffer_type =
use_v4l2_allocated_buffers ? V4L2_MEMORY_MMAP : V4L2_MEMORY_DMABUF;
const size_t v4l2_num_buffers = use_v4l2_allocated_buffers
? num_codec_reference_frames + 2
: VIDEO_MAX_FRAME;
if (!use_v4l2_allocated_buffers) {
std::optional<GpuBufferLayout> layout =
client_->GetVideoFramePool()->GetGpuBufferLayout();
if (!layout.has_value()) {
return false;
}
if (layout->modifier() == DRM_FORMAT_MOD_QCOM_COMPRESSED) {
if (!CAPTURE_queue_
->SetFormat(V4L2_PIX_FMT_QC08C, chosen_size, 0)
.has_value()) {
return false;
}
chosen_fourcc = Fourcc::FromV4L2PixFmt(V4L2_PIX_FMT_QC08C).value();
}
}
VLOG(2) << "Chosen |CAPTURE_queue_| format: " << chosen_fourcc.ToString()
<< " " << chosen_size.ToString() << " (modifier: 0x" << std::hex
<< chosen_modifier << std::dec << "). Using " << v4l2_num_buffers
<< " |CAPTURE_queue_| slots.";
const auto allocated_buffers = CAPTURE_queue_->AllocateBuffers(
v4l2_num_buffers, buffer_type, false);
if (allocated_buffers < v4l2_num_buffers) {
LOGF(ERROR) << "Failed to allocate enough CAPTURE buffers, requested= "
<< v4l2_num_buffers << " actual= " << allocated_buffers;
return false;
}
if (!CAPTURE_queue_->Streamon()) {
return false;
}
TryAndEnqueueCAPTUREQueueBuffers();
TryAndEnqueueOUTPUTQueueBuffers();
RearmCAPTUREQueueMonitoring();
return true;
}
std::vector<ImageProcessor::PixelLayoutCandidate>
V4L2StatefulVideoDecoder::EnumeratePixelLayoutCandidates(
const gfx::Size& coded_size) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(CAPTURE_queue_) << "|CAPTURE_queue_| must be created at this point";
const auto v4l2_pix_fmts = EnumerateSupportedPixFmts(
base::BindRepeating(&HandledIoctl, device_fd_.get()),
V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
std::vector<ImageProcessor::PixelLayoutCandidate> candidates;
for (const uint32_t& pixfmt : v4l2_pix_fmts) {
const auto candidate_fourcc = Fourcc::FromV4L2PixFmt(pixfmt);
if (!candidate_fourcc) {
continue;
}
candidates.emplace_back(ImageProcessor::PixelLayoutCandidate{
.fourcc = *candidate_fourcc, .size = coded_size});
VLOG(2) << "CAPTURE queue candidate format: "
<< candidate_fourcc->ToString() << ", " << coded_size.ToString();
}
return candidates;
}
size_t V4L2StatefulVideoDecoder::GetNumberOfReferenceFrames() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(CAPTURE_queue_) << "|CAPTURE_queue_| must be created at this point";
constexpr size_t kDefaultNumReferenceFrames = 8;
constexpr size_t kDefaultNumReferenceFramesMTK8173 = 16;
size_t num_codec_reference_frames = is_mtk8173_
? kDefaultNumReferenceFramesMTK8173
: kDefaultNumReferenceFrames;
struct v4l2_ext_control ctrl = {.id = V4L2_CID_MIN_BUFFERS_FOR_CAPTURE};
struct v4l2_ext_controls ext_ctrls = {.count = 1, .controls = &ctrl};
if (HandledIoctl(device_fd_.get(), VIDIOC_G_EXT_CTRLS, &ext_ctrls) ==
kIoctlOk) {
num_codec_reference_frames = std::max(
base::checked_cast<size_t>(ctrl.value), num_codec_reference_frames);
}
VLOG(2) << "Driver wants: " << ctrl.value
<< " CAPTURE buffers. We'll use: " << num_codec_reference_frames;
CHECK_LE(num_codec_reference_frames, 18u);
return num_codec_reference_frames;
}
void V4L2StatefulVideoDecoder::RearmCAPTUREQueueMonitoring() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
auto dequeue_callback = base::BindPostTaskToCurrentDefault(base::BindOnce(
&V4L2StatefulVideoDecoder::TryAndDequeueCAPTUREQueueBuffers,
weak_ptr_factory_for_events_.GetWeakPtr()));
auto resolution_change_callback =
base::BindPostTaskToCurrentDefault(base::BindOnce(
[](base::WeakPtr<VideoDecoderMixin::Client> client,
base::WeakPtr<V4L2StatefulVideoDecoder> weak_this) {
if (weak_this && client) {
client->PrepareChangeResolution();
}
},
client_, weak_ptr_factory_for_events_.GetWeakPtr()));
cancelable_task_tracker_.PostTask(
event_task_runner_.get(), FROM_HERE,
base::BindOnce(&WaitOnceForEvents, device_fd_.get(), wake_event_.get(),
std::move(dequeue_callback),
std::move(resolution_change_callback)));
}
void V4L2StatefulVideoDecoder::TryAndDequeueCAPTUREQueueBuffers() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(CAPTURE_queue_) << "|CAPTURE_queue_| must be created at this point";
const v4l2_memory queue_type = CAPTURE_queue_->GetMemoryType();
DCHECK(queue_type == V4L2_MEMORY_MMAP || queue_type == V4L2_MEMORY_DMABUF);
const bool use_v4l2_allocated_buffers = !client_->GetVideoFramePool();
DCHECK((queue_type == V4L2_MEMORY_MMAP && use_v4l2_allocated_buffers) ||
(queue_type == V4L2_MEMORY_DMABUF && !use_v4l2_allocated_buffers));
bool success;
scoped_refptr<V4L2ReadableBuffer> dequeued_buffer;
for (std::tie(success, dequeued_buffer) = CAPTURE_queue_->DequeueBuffer();
success && dequeued_buffer;
std::tie(success, dequeued_buffer) = CAPTURE_queue_->DequeueBuffer()) {
PrintAndTraceQueueStates(FROM_HERE);
const int64_t flat_timespec =
TimeValToTimeDelta(dequeued_buffer->GetTimeStamp()).InMilliseconds();
if (base::Contains(encoding_timestamps_, flat_timespec)) {
UMA_HISTOGRAM_TIMES(
"Media.PlatformVideoDecoding.Decode",
base::TimeTicks::Now() - encoding_timestamps_[flat_timespec]);
encoding_timestamps_.erase(flat_timespec);
}
if (dequeued_buffer->IsLast()) {
VLOGF(3) << "Buffer marked LAST in |CAPTURE_queue_|";
if (!DrainOUTPUTQueue()) {
LOG(ERROR) << "Failed to drain resources from |OUTPUT_queue_|.";
}
if (!CAPTURE_queue_->SendStartCommand()) {
VLOGF(3) << "Failed to resume decoding after flush";
}
const bool has_pending_OUTPUT_queue_work =
OUTPUT_queue_->QueuedBuffersCount();
if (flush_cb_ && !has_pending_OUTPUT_queue_work) {
std::move(flush_cb_).Run(DecoderStatus::Codes::kOk);
}
return;
} else if (!dequeued_buffer->IsError()) {
scoped_refptr<FrameResource> frame = dequeued_buffer->GetFrameResource();
CHECK(frame);
frame->set_timestamp(TimeValToTimeDelta(dequeued_buffer->GetTimeStamp()));
frame->set_color_space(config_.color_space_info().ToGfxColorSpace());
frame->set_hdr_metadata(config_.hdr_metadata());
if (queue_type == V4L2_MEMORY_MMAP) {
CHECK(gfx::Rect(frame->coded_size()).Contains(visible_rect_));
CHECK(frame->visible_rect().Contains(visible_rect_));
auto wrapped_frame =
frame->CreateWrappingFrame(visible_rect_,
visible_rect_.size());
wrapped_frame->AddDestructionObserver(
base::BindPostTaskToCurrentDefault(base::BindOnce(
[](scoped_refptr<V4L2ReadableBuffer> buffer,
base::WeakPtr<V4L2StatefulVideoDecoder> weak_this) {
if (weak_this) {
weak_this->TryAndEnqueueCAPTUREQueueBuffers();
weak_this->PrintAndTraceQueueStates(FROM_HERE);
}
},
std::move(dequeued_buffer),
weak_ptr_factory_for_CAPTURE_availability_.GetWeakPtr())));
CHECK(wrapped_frame);
VLOGF(3) << wrapped_frame->AsHumanReadableString();
output_cb_.Run(std::move(wrapped_frame));
} else {
DCHECK_EQ(queue_type, V4L2_MEMORY_DMABUF);
VLOGF(3) << frame->AsHumanReadableString();
framerate_control_->AttachToFrameResource(frame);
output_cb_.Run(std::move(frame));
}
if (!DrainOUTPUTQueue()) {
LOG(ERROR) << "Failed to drain resources from |OUTPUT_queue_|.";
}
}
}
LOG_IF(ERROR, !success) << "Failed dequeueing from |CAPTURE_queue_|";
TryAndEnqueueCAPTUREQueueBuffers();
TryAndEnqueueOUTPUTQueueBuffers();
RearmCAPTUREQueueMonitoring();
}
void V4L2StatefulVideoDecoder::TryAndEnqueueCAPTUREQueueBuffers() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(CAPTURE_queue_) << "|CAPTURE_queue_| must be created at this point";
const v4l2_memory queue_type = CAPTURE_queue_->GetMemoryType();
DCHECK(queue_type == V4L2_MEMORY_MMAP || queue_type == V4L2_MEMORY_DMABUF);
if (queue_type == V4L2_MEMORY_MMAP) {
while (auto v4l2_buffer = CAPTURE_queue_->GetFreeBuffer()) {
if (!std::move(*v4l2_buffer).QueueMMap()) {
LOG(ERROR) << "CAPTURE queue failed to enqueue an MMAP buffer.";
return;
}
}
} else {
while (true) {
if (client_->GetVideoFramePool()->IsExhausted()) {
client_->GetVideoFramePool()->NotifyWhenFrameAvailable(base::BindOnce(
base::IgnoreResult(&base::SequencedTaskRunner::PostTask),
base::SequencedTaskRunner::GetCurrentDefault(), FROM_HERE,
base::BindOnce(
&V4L2StatefulVideoDecoder::TryAndEnqueueCAPTUREQueueBuffers,
weak_ptr_factory_for_CAPTURE_availability_.GetWeakPtr())));
return;
}
auto frame = client_->GetVideoFramePool()->GetFrame();
CHECK(frame);
auto v4l2_buffer = CAPTURE_queue_->GetFreeBuffer();
if (!v4l2_buffer) {
VLOGF(1) << "|CAPTURE_queue_| has no buffers";
return;
}
if (!std::move(*v4l2_buffer).QueueDMABuf(std::move(frame))) {
LOG(ERROR) << "CAPTURE queue failed to enqueue a DmaBuf buffer.";
return;
}
}
}
}
bool V4L2StatefulVideoDecoder::DrainOUTPUTQueue() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(IsInitialized()) << "V4L2StatefulVideoDecoder must be Initialize()d";
bool success;
scoped_refptr<V4L2ReadableBuffer> dequeued_buffer;
for (std::tie(success, dequeued_buffer) = OUTPUT_queue_->DequeueBuffer();
success && dequeued_buffer;
std::tie(success, dequeued_buffer) = OUTPUT_queue_->DequeueBuffer()) {
PrintAndTraceQueueStates(FROM_HERE);
}
return success;
}
bool V4L2StatefulVideoDecoder::TryAndEnqueueOUTPUTQueueBuffers() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(IsInitialized()) << "V4L2StatefulVideoDecoder must be Initialize()d";
if (!DrainOUTPUTQueue()) {
PLOG(ERROR) << "Failed to drain resources from |OUTPUT_queue_|.";
return false;
}
for (std::optional<V4L2WritableBufferRef> v4l2_buffer =
OUTPUT_queue_->GetFreeBuffer();
v4l2_buffer && !decoder_buffer_and_callbacks_.empty();
v4l2_buffer = OUTPUT_queue_->GetFreeBuffer()) {
PrintAndTraceQueueStates(FROM_HERE);
auto media_buffer = std::move(decoder_buffer_and_callbacks_.front().first);
auto media_decode_cb =
std::move(decoder_buffer_and_callbacks_.front().second);
decoder_buffer_and_callbacks_.pop();
if (media_buffer) {
if (media_buffer->end_of_stream()) {
if (!OUTPUT_queue_->SendStopCommand()) {
std::move(media_decode_cb).Run(DecoderStatus::Codes::kFailed);
return false;
}
flush_cb_ = std::move(media_decode_cb);
return true;
}
CHECK_EQ(v4l2_buffer->PlanesCount(), 1u);
uint8_t* dst = static_cast<uint8_t*>(v4l2_buffer->GetPlaneMapping(0));
auto media_buffer_span = base::span(*media_buffer);
CHECK_GE(v4l2_buffer->GetPlaneSize(0),
media_buffer_span.size());
memcpy(dst, media_buffer_span.data(), media_buffer_span.size());
v4l2_buffer->SetPlaneBytesUsed(0, media_buffer_span.size());
VLOGF(4) << "Enqueuing " << media_buffer_span.size() << " bytes.";
v4l2_buffer->SetTimeStamp(TimeDeltaToTimeVal(media_buffer->timestamp()));
const int64_t flat_timespec = media_buffer->timestamp().InMilliseconds();
encoding_timestamps_[flat_timespec] = base::TimeTicks::Now();
if (!std::move(*v4l2_buffer).QueueMMap()) {
LOG(ERROR) << "Error while queuing input |media_buffer|!";
std::move(media_decode_cb)
.Run(DecoderStatus::Codes::kPlatformDecodeFailure);
return false;
}
}
std::move(media_decode_cb).Run(DecoderStatus::Codes::kOk);
}
return true;
}
void V4L2StatefulVideoDecoder::PrintAndTraceQueueStates(
const base::Location& from_here) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(IsInitialized()) << "V4L2StatefulVideoDecoder must be Initialize()d";
VLOG(4) << from_here.function_name() << "(): |OUTPUT_queue_| "
<< OUTPUT_queue_->QueuedBuffersCount() << "/"
<< OUTPUT_queue_->AllocatedBuffersCount() << ", |CAPTURE_queue_| "
<< (CAPTURE_queue_ ? CAPTURE_queue_->QueuedBuffersCount() : 0) << "/"
<< (CAPTURE_queue_ ? CAPTURE_queue_->AllocatedBuffersCount() : 0);
TRACE_COUNTER_ID1(
"media,gpu", "V4L2 OUTPUT Q used buffers", this,
base::checked_cast<int32_t>(OUTPUT_queue_->QueuedBuffersCount()));
TRACE_COUNTER_ID1("media,gpu", "V4L2 CAPTURE Q free buffers", this,
(CAPTURE_queue_ ? base::checked_cast<int32_t>(
CAPTURE_queue_->QueuedBuffersCount())
: 0));
}
bool V4L2StatefulVideoDecoder::IsInitialized() const {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
return !!OUTPUT_queue_;
}
int V4L2StatefulVideoDecoder::GetMaxNumDecoderInstances() {
if (!base::FeatureList::IsEnabled(media::kLimitConcurrentDecoderInstances)) {
return std::numeric_limits<int>::max();
}
auto device_fd = V4L2Device::OpenFDForType(V4L2Device::Type::kDecoder);
if (!device_fd.is_valid()) {
return std::numeric_limits<int>::max();
}
struct v4l2_capability caps = {};
if (HandledIoctl(device_fd.get(), VIDIOC_QUERYCAP, &caps) != kIoctlOk) {
PLOG(ERROR) << "Failed querying caps";
return std::numeric_limits<int>::max();
}
const bool is_mtk8173 = base::Contains(
std::string(reinterpret_cast<const char*>(caps.card)), "8173");
return is_mtk8173 ? 10 : 15;
}
std::vector<std::pair<scoped_refptr<DecoderBuffer>, VideoDecoder::DecodeCB>>
H264FrameReassembler::Process(scoped_refptr<DecoderBuffer> buffer,
VideoDecoder::DecodeCB decode_cb) {
std::vector<std::pair<scoped_refptr<DecoderBuffer>, VideoDecoder::DecodeCB>>
frames;
auto remaining = base::span(*buffer);
do {
const auto nalu_info =
FindH264FrameBoundary(remaining.data(), remaining.size());
if (!nalu_info.has_value()) {
LOG(ERROR) << "Failed parsing H.264 DecoderBuffer";
std::move(decode_cb).Run(DecoderStatus::Codes::kFailed);
return {};
}
const size_t found_nalu_size =
base::checked_cast<size_t>(nalu_info->nalu_size);
if (nalu_info->is_start_of_new_frame && HasFragments()) {
VLOGF(4) << frame_fragments_.size()
<< " currently stored frame fragment(s) can be reassembled.";
frames.emplace_back(ReassembleFragments(frame_fragments_),
base::DoNothing());
}
if (nalu_info->is_whole_frame) {
VLOGF(3) << "Found a whole frame, size=" << found_nalu_size << " bytes";
frames.emplace_back(
DecoderBuffer::CopyFrom(remaining.take_first(found_nalu_size)),
base::DoNothing());
frames.back().first->set_timestamp(buffer->timestamp());
continue;
}
VLOGF(4) << "This was a frame fragment; storing it for later reassembly.";
frame_fragments_.emplace_back(
DecoderBuffer::CopyFrom(remaining.take_first(found_nalu_size)));
frame_fragments_.back()->set_timestamp(buffer->timestamp());
} while (!remaining.empty());
if (frames.empty()) {
frames.emplace_back(nullptr, std::move(decode_cb));
} else {
frames.back().second = std::move(decode_cb);
}
return frames;
}
std::optional<struct H264FrameReassembler::FrameBoundaryInfo>
H264FrameReassembler::FindH264FrameBoundary(const uint8_t* const data,
size_t data_size) {
h264_parser_.SetStream(data, data_size);
while (true) {
H264NALU nalu = {};
H264Parser::Result result = h264_parser_.AdvanceToNextNALU(&nalu);
if (result == H264Parser::kInvalidStream ||
result == H264Parser::kUnsupportedStream) {
LOG(ERROR) << "Could not parse bitstream.";
return std::nullopt;
}
if (result == H264Parser::kEOStream) {
return FrameBoundaryInfo{.is_whole_frame = true,
.is_start_of_new_frame = true,
.nalu_size = nalu.data.size()};
}
DCHECK_EQ(result, H264Parser::kOk);
static const char* kKnownNALUNames[] = {
"Unspecified", "NonIDRSlice", "SliceDataA",
"SliceDataB", "SliceDataC", "IDRSlice",
"SEIMessage", "SPS", "PPS",
"AUD", "EOSeq", "EOStream",
"Filler", "SPSExt", "Prefix",
"SubsetSPS", "DPS", "Reserved17",
"Reserved18", "CodedSliceAux", "CodedSliceExtension",
};
constexpr auto kMaxNALUTypeValue = std::size(kKnownNALUNames);
if (base::checked_cast<size_t>(nalu.nal_unit_type) >= kMaxNALUTypeValue) {
LOG(ERROR) << "NALU type unknown.";
return std::nullopt;
}
CHECK_GE(nalu.data.data(), data);
CHECK_LE(nalu.data.data(), data + data_size);
const auto nalu_size = nalu.data.data() - data + nalu.data.size();
VLOGF(4) << "H264NALU type " << kKnownNALUNames[nalu.nal_unit_type]
<< ", NALU size=" << nalu_size
<< " bytes, payload size=" << nalu.data.size() << " bytes";
switch (nalu.nal_unit_type) {
case H264NALU::kSPS:
result = h264_parser_.ParseSPS(&sps_id_);
if (result != H264Parser::kOk) {
LOG(ERROR) << "Could not parse SPS header.";
return std::nullopt;
}
previous_slice_header_.reset();
return FrameBoundaryInfo{.is_whole_frame = true,
.is_start_of_new_frame = true,
.nalu_size = nalu_size};
case H264NALU::kPPS:
result = h264_parser_.ParsePPS(&pps_id_);
if (result != H264Parser::kOk) {
LOG(ERROR) << "Could not parse PPS header.";
return std::nullopt;
}
previous_slice_header_.reset();
return FrameBoundaryInfo{.is_whole_frame = true,
.is_start_of_new_frame = true,
.nalu_size = nalu_size};
case H264NALU::kNonIDRSlice:
case H264NALU::kIDRSlice: {
H264SliceHeader curr_slice_header;
result = h264_parser_.ParseSliceHeader(nalu, &curr_slice_header);
if (result != H264Parser::kOk) {
LOG(WARNING) << "Could not parse NALU header.";
return FrameBoundaryInfo{.is_whole_frame = true,
.is_start_of_new_frame = false,
.nalu_size = nalu_size};
}
const bool is_new_frame =
previous_slice_header_ &&
IsNewH264Frame(h264_parser_.GetSPS(sps_id_),
h264_parser_.GetPPS(pps_id_),
previous_slice_header_.get(), &curr_slice_header);
previous_slice_header_ =
std::make_unique<H264SliceHeader>(curr_slice_header);
return FrameBoundaryInfo{.is_whole_frame = false,
.is_start_of_new_frame = is_new_frame,
.nalu_size = nalu_size};
}
case H264NALU::kSEIMessage:
case H264NALU::kAUD:
case H264NALU::kEOSeq:
case H264NALU::kEOStream:
case H264NALU::kFiller:
case H264NALU::kSPSExt:
case H264NALU::kPrefix:
case H264NALU::kSubsetSPS:
case H264NALU::kDPS:
case H264NALU::kReserved17:
case H264NALU::kReserved18:
previous_slice_header_.reset();
return FrameBoundaryInfo{.is_whole_frame = true,
.is_start_of_new_frame = true,
.nalu_size = nalu_size};
default:
VLOGF(4) << "Unsupported NALU " << kKnownNALUNames[nalu.nal_unit_type];
}
}
}
}