#ifdef UNSAFE_BUFFERS_BUILD
#pragma allow_unsafe_buffers
#endif
#include "media/video/gpu_memory_buffer_video_frame_pool.h"
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <stddef.h>
#include <stdint.h>
#include <algorithm>
#include <array>
#include <atomic>
#include <cstdint>
#include <list>
#include <memory>
#include <utility>
#include "base/barrier_closure.h"
#include "base/bits.h"
#include "base/command_line.h"
#include "base/containers/circular_deque.h"
#include "base/containers/span.h"
#include "base/feature_list.h"
#include "base/functional/bind.h"
#include "base/functional/callback_helpers.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/memory/raw_ptr.h"
#include "base/memory/scoped_refptr.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/not_fatal_until.h"
#include "base/numerics/safe_conversions.h"
#include "base/strings/stringprintf.h"
#include "base/task/sequenced_task_runner.h"
#include "base/time/default_tick_clock.h"
#include "base/time/time.h"
#include "base/trace_event/memory_dump_manager.h"
#include "base/trace_event/memory_dump_provider.h"
#include "base/trace_event/trace_event.h"
#include "build/build_config.h"
#include "components/viz/common/resources/shared_image_format.h"
#include "components/viz/common/resources/shared_image_format_utils.h"
#include "gpu/GLES2/gl2extchromium.h"
#include "gpu/command_buffer/client/client_shared_image.h"
#include "gpu/command_buffer/client/shared_image_interface.h"
#include "gpu/command_buffer/common/shared_image_capabilities.h"
#include "gpu/command_buffer/common/shared_image_usage.h"
#include "gpu/config/gpu_finch_features.h"
#include "gpu/config/gpu_switches.h"
#include "media/base/media_switches.h"
#include "media/base/video_types.h"
#include "media/base/video_util.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "third_party/libyuv/include/libyuv.h"
#include "third_party/perfetto/include/perfetto/tracing/track.h"
#include "ui/gfx/buffer_types.h"
#include "ui/gfx/color_space.h"
#include "ui/gl/trace_util.h"
#if BUILDFLAG(IS_MAC)
#include "base/mac/mac_util.h"
#include "media/base/mac/video_frame_mac.h"
#endif
namespace media {
class GpuMemoryBufferVideoFramePool::PoolImpl
: public base::RefCountedThreadSafe<
GpuMemoryBufferVideoFramePool::PoolImpl>,
public base::trace_event::MemoryDumpProvider {
public:
REQUIRE_ADOPTION_FOR_REFCOUNTED_TYPE();
PoolImpl(const scoped_refptr<base::SequencedTaskRunner>& media_task_runner,
const scoped_refptr<base::TaskRunner>& worker_task_runner,
GpuVideoAcceleratorFactories* const gpu_factories)
: media_task_runner_(media_task_runner),
worker_task_runner_(worker_task_runner),
gpu_factories_(gpu_factories),
output_format_(GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED),
tick_clock_(base::DefaultTickClock::GetInstance()) {
DCHECK(media_task_runner_);
DCHECK(worker_task_runner_);
static std::atomic_uint32_t id = 0;
pool_id_ = ++id;
}
PoolImpl(const PoolImpl&) = delete;
PoolImpl& operator=(const PoolImpl&) = delete;
void CreateHardwareFrame(scoped_refptr<VideoFrame> video_frame,
FrameReadyCB cb);
bool OnMemoryDump(const base::trace_event::MemoryDumpArgs& args,
base::trace_event::ProcessMemoryDump* pmd) override;
void Abort();
void Shutdown();
void SetTickClockForTesting(const base::TickClock* tick_clock);
private:
friend class base::RefCountedThreadSafe<
GpuMemoryBufferVideoFramePool::PoolImpl>;
~PoolImpl() override;
struct FrameResource {
explicit FrameResource(const gfx::Size& size,
gfx::BufferUsage usage,
const gfx::ColorSpace& color_space)
: size(size), usage(usage), color_space(color_space) {}
void MarkUsed() {
is_used_ = true;
last_use_time_ = base::TimeTicks();
}
void MarkUnused(base::TimeTicks last_use_time) {
is_used_ = false;
last_use_time_ = last_use_time;
}
bool is_used() const { return is_used_; }
base::TimeTicks last_use_time() const { return last_use_time_; }
const gfx::Size size;
const gfx::BufferUsage usage;
const gfx::ColorSpace color_space;
int32_t buffer_id = -1;
scoped_refptr<gpu::ClientSharedImage> shared_image;
std::unique_ptr<gpu::ClientSharedImage::ScopedMapping> scoped_mapping;
gpu::SyncToken sync_token;
private:
bool is_used_ = true;
base::TimeTicks last_use_time_;
};
struct VideoFrameCopyRequest {
VideoFrameCopyRequest(scoped_refptr<VideoFrame> video_frame,
FrameReadyCB frame_ready_cb,
bool passthrough)
: video_frame(std::move(video_frame)),
frame_ready_cb(std::move(frame_ready_cb)),
passthrough(passthrough) {}
scoped_refptr<VideoFrame> video_frame;
FrameReadyCB frame_ready_cb;
bool passthrough;
};
void StartCopy();
void CopyVideoFrameToGpuMemoryBuffer(scoped_refptr<VideoFrame> video_frame,
FrameResource* frame_resource);
void OnCopiesDone(bool copy_failed,
scoped_refptr<VideoFrame> video_frame,
FrameResource* frame_resource);
void OnCopiesDoneOnMediaThread(bool copy_failed,
scoped_refptr<VideoFrame> video_frame,
FrameResource* frame_resource);
static void CopyRowsToBuffer(
GpuVideoAcceleratorFactories::OutputFormat output_format,
const size_t row,
const size_t rows_to_copy,
const gfx::Size coded_size,
const VideoFrame* video_frame,
FrameResource* frame_resource,
base::OnceClosure done);
scoped_refptr<VideoFrame> BindAndCreateMailboxHardwareFrameResource(
FrameResource* frame_resource,
const gfx::Size& coded_size,
const gfx::Rect& visible_rect,
const gfx::Size& natural_size,
const gfx::ColorSpace& color_space,
base::TimeDelta timestamp,
bool video_frame_allow_overlay);
static bool IsFrameResourceCompatible(const FrameResource* resource,
const gfx::Size& size,
gfx::BufferUsage usage,
const gfx::ColorSpace& color_space) {
return size == resource->size && usage == resource->usage &&
color_space == resource->color_space;
}
FrameResource* GetOrCreateFrameResource(const gfx::Size& size,
gfx::BufferUsage usage,
const gfx::ColorSpace& color_space);
void CompleteCopyRequestAndMaybeStartNextCopy(
scoped_refptr<VideoFrame> video_frame);
void MailboxHolderReleased(FrameResource* frame_resource,
const gpu::SyncToken& sync_token);
static void DeleteFrameResource(
GpuVideoAcceleratorFactories* const gpu_factories,
FrameResource* frame_resource);
const scoped_refptr<base::SequencedTaskRunner> media_task_runner_;
const scoped_refptr<base::TaskRunner> worker_task_runner_;
const raw_ptr<GpuVideoAcceleratorFactories> gpu_factories_;
std::list<raw_ptr<FrameResource, CtnExperimental>> resources_pool_;
GpuVideoAcceleratorFactories::OutputFormat output_format_;
raw_ptr<const base::TickClock> tick_clock_;
base::circular_deque<VideoFrameCopyRequest> frame_copy_requests_;
bool in_shutdown_ = false;
uint32_t pool_id_ = 0;
uint32_t buffer_id_ = 0;
};
namespace {
constexpr size_t kBytesPerCopyTarget = 1024 * 1024;
viz::SharedImageFormat OutputFormatToSharedImageFormat(
GpuVideoAcceleratorFactories::OutputFormat format) {
switch (format) {
case GpuVideoAcceleratorFactories::OutputFormat::YV12:
return viz::MultiPlaneFormat::kYV12;
case GpuVideoAcceleratorFactories::OutputFormat::P010:
return viz::MultiPlaneFormat::kP010;
case GpuVideoAcceleratorFactories::OutputFormat::NV12:
return viz::MultiPlaneFormat::kNV12;
case GpuVideoAcceleratorFactories::OutputFormat::XR30:
return viz::SinglePlaneFormat::kBGRA_1010102;
case GpuVideoAcceleratorFactories::OutputFormat::XB30:
return viz::SinglePlaneFormat::kRGBA_1010102;
case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
NOTREACHED();
}
}
VideoPixelFormat VideoFormat(
GpuVideoAcceleratorFactories::OutputFormat format) {
switch (format) {
case GpuVideoAcceleratorFactories::OutputFormat::YV12:
return PIXEL_FORMAT_YV12;
case GpuVideoAcceleratorFactories::OutputFormat::NV12:
return PIXEL_FORMAT_NV12;
case GpuVideoAcceleratorFactories::OutputFormat::P010:
return PIXEL_FORMAT_P010LE;
case GpuVideoAcceleratorFactories::OutputFormat::XR30:
return PIXEL_FORMAT_XR30;
case GpuVideoAcceleratorFactories::OutputFormat::XB30:
return PIXEL_FORMAT_XB30;
case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
NOTREACHED();
}
}
int RowsPerCopy(VideoPixelFormat format, int width) {
int bytes_per_row = VideoFrame::RowBytes(0, format, width);
if (format == PIXEL_FORMAT_NV12) {
bytes_per_row += VideoFrame::RowBytes(1, format, width);
}
return std::max<size_t>((kBytesPerCopyTarget / bytes_per_row) & ~1, 1);
}
void CopyRowsToI420Buffer(size_t first_row,
size_t rows,
size_t bytes_per_row,
size_t bit_depth,
const uint8_t* source,
size_t source_stride,
base::span<uint8_t> output,
size_t dest_stride) {
TRACE_EVENT2("media", "CopyRowsToI420Buffer", "bytes_per_row", bytes_per_row,
"rows", rows);
if (output.empty()) {
return;
}
DCHECK_NE(dest_stride, 0u);
DCHECK_LE(bytes_per_row, dest_stride);
DCHECK_LE(bytes_per_row, source_stride);
DCHECK_GE(bit_depth, 8u);
if (bit_depth == 8u) {
libyuv::CopyPlane(source + source_stride * first_row, source_stride,
output.subspan(dest_stride * first_row).data(),
dest_stride, bytes_per_row, rows);
} else {
const int scale = 0x10000 >> (bit_depth - 8u);
libyuv::Convert16To8Plane(
reinterpret_cast<const uint16_t*>(source + source_stride * first_row),
source_stride / 2, output.subspan(dest_stride * first_row).data(),
dest_stride, scale, bytes_per_row, rows);
}
}
void CopyRowsToP010Buffer(int first_row,
int rows,
int width,
const VideoFrame* source_frame,
base::span<uint8_t> dest_y,
int dest_stride_y,
base::span<uint8_t> dest_uv,
int dest_stride_uv) {
TRACE_EVENT2("media", "CopyRowsToP010Buffer", "width", width, "rows", rows);
if (dest_y.empty() || dest_uv.empty()) {
return;
}
DCHECK_NE(dest_stride_y, 0);
DCHECK_NE(dest_stride_uv, 0);
DCHECK_EQ(0, first_row % 2);
DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_YUV420P10);
DCHECK_LE(static_cast<size_t>(width * 2),
source_frame->stride(VideoFrame::Plane::kY));
const uint16_t* y_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kY) +
first_row * source_frame->stride(VideoFrame::Plane::kY));
const size_t y_plane_stride = source_frame->stride(VideoFrame::Plane::kY) / 2;
const uint16_t* u_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kU) +
(first_row / 2) * source_frame->stride(VideoFrame::Plane::kU));
const size_t u_plane_stride = source_frame->stride(VideoFrame::Plane::kU) / 2;
const uint16_t* v_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kV) +
(first_row / 2) * source_frame->stride(VideoFrame::Plane::kV));
const size_t v_plane_stride = source_frame->stride(VideoFrame::Plane::kV) / 2;
libyuv::I010ToP010(
y_plane, y_plane_stride, u_plane, u_plane_stride, v_plane, v_plane_stride,
reinterpret_cast<uint16_t*>(
dest_y.subspan(base::checked_cast<size_t>(first_row * dest_stride_y))
.data()),
dest_stride_y / 2,
reinterpret_cast<uint16_t*>(dest_uv
.subspan(base::checked_cast<size_t>(
(first_row / 2) * dest_stride_uv))
.data()),
dest_stride_uv / 2, width, rows);
}
void CopyRowsToNV12Buffer(int first_row,
int rows,
int width,
size_t bit_depth,
const VideoFrame* source_frame,
base::span<uint8_t> dest_y,
int dest_stride_y,
base::span<uint8_t> dest_uv,
int dest_stride_uv) {
TRACE_EVENT2("media", "CopyRowsToNV12Buffer", "width", width, "rows", rows);
if (dest_y.empty() || dest_uv.empty()) {
return;
}
DCHECK_NE(dest_stride_y, 0);
DCHECK_NE(dest_stride_uv, 0);
DCHECK_EQ(0, first_row % 2);
DCHECK(source_frame->format() == PIXEL_FORMAT_I420 ||
source_frame->format() == PIXEL_FORMAT_YV12 ||
source_frame->format() == PIXEL_FORMAT_NV12 ||
source_frame->format() == PIXEL_FORMAT_YUV420P10);
if (bit_depth == 8) {
const int rows_y =
VideoFrame::Rows(VideoFrame::Plane::kY, PIXEL_FORMAT_NV12, rows);
const int rows_uv =
VideoFrame::Rows(VideoFrame::Plane::kUV, PIXEL_FORMAT_NV12, rows);
const int bytes_per_row_y =
VideoFrame::RowBytes(VideoFrame::Plane::kY, PIXEL_FORMAT_NV12, width);
const int bytes_per_row_uv =
VideoFrame::RowBytes(VideoFrame::Plane::kUV, PIXEL_FORMAT_NV12, width);
DCHECK_LE(bytes_per_row_y, std::abs(dest_stride_y));
DCHECK_LE(bytes_per_row_uv, std::abs(dest_stride_uv));
if (source_frame->format() == PIXEL_FORMAT_NV12) {
libyuv::CopyPlane(
source_frame->visible_data(VideoFrame::Plane::kY) +
first_row * source_frame->stride(VideoFrame::Plane::kY),
source_frame->stride(VideoFrame::Plane::kY),
dest_y.subspan(base::checked_cast<size_t>(first_row * dest_stride_y))
.data(),
dest_stride_y, bytes_per_row_y, rows_y);
libyuv::CopyPlane(
source_frame->visible_data(VideoFrame::Plane::kUV) +
first_row / 2 * source_frame->stride(VideoFrame::Plane::kUV),
source_frame->stride(VideoFrame::Plane::kUV),
dest_uv
.subspan(
base::checked_cast<size_t>(first_row / 2 * dest_stride_uv))
.data(),
dest_stride_uv, bytes_per_row_uv, rows_uv);
return;
}
libyuv::I420ToNV12(
source_frame->visible_data(VideoFrame::Plane::kY) +
first_row * source_frame->stride(VideoFrame::Plane::kY),
source_frame->stride(VideoFrame::Plane::kY),
source_frame->visible_data(VideoFrame::Plane::kU) +
first_row / 2 * source_frame->stride(VideoFrame::Plane::kU),
source_frame->stride(VideoFrame::Plane::kU),
source_frame->visible_data(VideoFrame::Plane::kV) +
first_row / 2 * source_frame->stride(VideoFrame::Plane::kV),
source_frame->stride(VideoFrame::Plane::kV),
dest_y.subspan(base::checked_cast<size_t>(first_row * dest_stride_y))
.data(),
dest_stride_y,
dest_uv
.subspan(base::checked_cast<size_t>(first_row / 2 * dest_stride_uv))
.data(),
dest_stride_uv, bytes_per_row_y, rows_y);
} else {
DCHECK_LE(static_cast<size_t>(width * 2),
source_frame->stride(VideoFrame::Plane::kY));
const uint16_t* y_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kY) +
first_row * source_frame->stride(VideoFrame::Plane::kY));
const size_t y_plane_stride =
source_frame->stride(VideoFrame::Plane::kY) / 2;
const uint16_t* u_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kU) +
(first_row / 2) * source_frame->stride(VideoFrame::Plane::kU));
const size_t u_plane_stride =
source_frame->stride(VideoFrame::Plane::kU) / 2;
const uint16_t* v_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kV) +
(first_row / 2) * source_frame->stride(VideoFrame::Plane::kV));
const size_t v_plane_stride =
source_frame->stride(VideoFrame::Plane::kV) / 2;
libyuv::I010ToNV12(
y_plane, y_plane_stride, u_plane, u_plane_stride, v_plane,
v_plane_stride,
dest_y.subspan(base::checked_cast<size_t>(first_row * dest_stride_y))
.data(),
dest_stride_y,
dest_uv
.subspan(
base::checked_cast<size_t>((first_row / 2) * dest_stride_uv))
.data(),
dest_stride_uv, width, rows);
}
}
void CopyRowsToRGB10Buffer(bool is_rgba,
int first_row,
int rows,
int width,
const VideoFrame* source_frame,
base::span<uint8_t> output,
int dest_stride) {
TRACE_EVENT2("media", "CopyRowsToRGB10Buffer", "bytes_per_row", width * 2,
"rows", rows);
if (output.empty()) {
return;
}
DCHECK_NE(dest_stride, 0);
DCHECK_LE(width, std::abs(dest_stride / 2));
DCHECK_EQ(0, first_row % 2);
DCHECK_EQ(source_frame->format(), PIXEL_FORMAT_YUV420P10);
const auto* y_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kY) +
first_row * source_frame->stride(VideoFrame::Plane::kY));
const auto* u_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kU) +
first_row / 2 * source_frame->stride(VideoFrame::Plane::kU));
const auto* v_plane = reinterpret_cast<const uint16_t*>(
source_frame->visible_data(VideoFrame::Plane::kV) +
first_row / 2 * source_frame->stride(VideoFrame::Plane::kV));
size_t y_plane_stride = source_frame->stride(VideoFrame::Plane::kY) / 2;
size_t u_plane_stride = source_frame->stride(VideoFrame::Plane::kU) / 2;
size_t v_plane_stride = source_frame->stride(VideoFrame::Plane::kV) / 2;
uint8_t* dest_rgb10 =
output.subspan(base::checked_cast<size_t>(first_row * dest_stride))
.data();
SkYUVColorSpace yuv_cs = kRec601_Limited_SkYUVColorSpace;
source_frame->ColorSpace().ToSkYUVColorSpace(source_frame->BitDepth(),
&yuv_cs);
const bool is_libyuv_abgr = is_rgba;
const auto* matrix = GetYuvContantsForColorSpace(
yuv_cs, !is_libyuv_abgr);
if (is_libyuv_abgr) {
std::swap(u_plane, v_plane);
std::swap(u_plane_stride, v_plane_stride);
}
libyuv::I010ToAR30Matrix(y_plane, y_plane_stride, u_plane, u_plane_stride,
v_plane, v_plane_stride, dest_rgb10, dest_stride,
matrix, width, rows);
}
gfx::Size CodedSize(const VideoFrame* video_frame,
GpuVideoAcceleratorFactories::OutputFormat output_format) {
DCHECK(gfx::Rect(video_frame->coded_size())
.Contains(video_frame->visible_rect()));
size_t width = video_frame->visible_rect().width();
size_t height = video_frame->visible_rect().height();
gfx::Size output;
switch (output_format) {
case GpuVideoAcceleratorFactories::OutputFormat::YV12:
case GpuVideoAcceleratorFactories::OutputFormat::P010:
case GpuVideoAcceleratorFactories::OutputFormat::NV12:
DCHECK_EQ(video_frame->visible_rect().x() % 2, 0);
DCHECK_EQ(video_frame->visible_rect().y() % 2, 0);
if (!viz::IsOddSizeMultiPlanarBuffersAllowed()) {
width = base::bits::AlignUp(width, size_t{2});
height = base::bits::AlignUp(height, size_t{2});
}
output = gfx::Size(width, height);
break;
case GpuVideoAcceleratorFactories::OutputFormat::XR30:
case GpuVideoAcceleratorFactories::OutputFormat::XB30:
output = gfx::Size(base::bits::AlignUp(width, size_t{2}), height);
break;
case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
NOTREACHED();
}
DCHECK(gfx::Rect(video_frame->coded_size()).Contains(gfx::Rect(output)));
return output;
}
void SetPrefersExternalSampler(viz::SharedImageFormat& format) {
if (format.is_multi_plane()) {
#if BUILDFLAG(IS_OZONE)
format.SetPrefersExternalSampler();
#endif
}
}
gfx::ColorSpace GetOutputColorSpace(
const gfx::ColorSpace& source_cs,
GpuVideoAcceleratorFactories::OutputFormat output_format) {
switch (output_format) {
case GpuVideoAcceleratorFactories::OutputFormat::YV12:
case GpuVideoAcceleratorFactories::OutputFormat::P010:
case GpuVideoAcceleratorFactories::OutputFormat::NV12:
return source_cs;
case GpuVideoAcceleratorFactories::OutputFormat::XR30:
case GpuVideoAcceleratorFactories::OutputFormat::XB30:
return source_cs.GetAsFullRangeRGB();
case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
NOTREACHED();
}
}
}
void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
scoped_refptr<VideoFrame> video_frame,
FrameReadyCB frame_ready_cb) {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
#if BUILDFLAG(IS_ARKWEB)
std::move(frame_ready_cb).Run(std::move(video_frame));
return;
#else
const VideoPixelFormat pixel_format = video_frame->format();
if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED) {
output_format_ = gpu_factories_->VideoFrameOutputFormat(pixel_format);
}
if (output_format_ != gpu_factories_->VideoFrameOutputFormat(pixel_format)) {
std::move(frame_ready_cb).Run(std::move(video_frame));
return;
}
bool is_software_backed_video_frame = !video_frame->HasSharedImage();
#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
is_software_backed_video_frame &= !video_frame->HasDmaBufs();
#endif
bool passthrough = false;
#if BUILDFLAG(IS_APPLE)
if (!IOSurfaceCanSetColorSpace(video_frame->ColorSpace())) {
passthrough = true;
}
#endif
if (!video_frame->IsMappable()) {
passthrough = true;
}
if (output_format_ == GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED) {
passthrough = true;
}
switch (pixel_format) {
case PIXEL_FORMAT_YV12:
case PIXEL_FORMAT_I420:
case PIXEL_FORMAT_YUV420P10:
case PIXEL_FORMAT_NV12:
case PIXEL_FORMAT_NV12A:
break;
case PIXEL_FORMAT_I420A:
case PIXEL_FORMAT_I422:
case PIXEL_FORMAT_I444:
case PIXEL_FORMAT_NV21:
case PIXEL_FORMAT_UYVY:
case PIXEL_FORMAT_YUY2:
case PIXEL_FORMAT_ARGB:
case PIXEL_FORMAT_BGRA:
case PIXEL_FORMAT_XRGB:
case PIXEL_FORMAT_RGB24:
case PIXEL_FORMAT_MJPEG:
case PIXEL_FORMAT_YUV422P10:
case PIXEL_FORMAT_YUV444P10:
case PIXEL_FORMAT_YUV420P12:
case PIXEL_FORMAT_YUV422P12:
case PIXEL_FORMAT_YUV444P12:
case PIXEL_FORMAT_Y16:
case PIXEL_FORMAT_ABGR:
case PIXEL_FORMAT_XBGR:
case PIXEL_FORMAT_NV16:
case PIXEL_FORMAT_NV24:
case PIXEL_FORMAT_P010LE:
case PIXEL_FORMAT_P210LE:
case PIXEL_FORMAT_P410LE:
case PIXEL_FORMAT_XR30:
case PIXEL_FORMAT_XB30:
case PIXEL_FORMAT_RGBAF16:
case PIXEL_FORMAT_I422A:
case PIXEL_FORMAT_I444A:
case PIXEL_FORMAT_YUV420AP10:
case PIXEL_FORMAT_YUV422AP10:
case PIXEL_FORMAT_YUV444AP10:
case PIXEL_FORMAT_UNKNOWN:
if (is_software_backed_video_frame) {
UMA_HISTOGRAM_ENUMERATION(
"Media.GpuMemoryBufferVideoFramePool.UnsupportedFormat",
pixel_format, PIXEL_FORMAT_MAX + 1);
}
passthrough = true;
}
if (video_frame->visible_rect().x() % 2 ||
video_frame->visible_rect().y() % 2) {
passthrough = true;
}
if (video_frame->coded_size().width() % 2 &&
!viz::IsOddSizeMultiPlanarBuffersAllowed()) {
passthrough = true;
}
if (video_frame->coded_size().height() % 2 &&
!viz::IsOddSizeMultiPlanarBuffersAllowed()) {
passthrough = true;
}
frame_copy_requests_.emplace_back(std::move(video_frame),
std::move(frame_ready_cb), passthrough);
if (frame_copy_requests_.size() == 1u) {
StartCopy();
}
#endif
}
bool GpuMemoryBufferVideoFramePool::PoolImpl::OnMemoryDump(
const base::trace_event::MemoryDumpArgs& args,
base::trace_event::ProcessMemoryDump* pmd) {
const int kImportance = 2;
for (const FrameResource* frame_resource : resources_pool_) {
scoped_refptr<gpu::ClientSharedImage> shared_image =
frame_resource->shared_image;
if (shared_image) {
std::string dump_name =
base::StringPrintf("media/video_frame_memory_%d/buffer_%d", pool_id_,
frame_resource->buffer_id);
base::trace_event::MemoryAllocatorDump* dump =
pmd->CreateAllocatorDump(dump_name);
auto size = frame_resource->size;
size_t buffer_size_in_bytes =
shared_image->format().EstimatedSizeInBytes(size);
dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
buffer_size_in_bytes);
dump->AddScalar("free_size",
base::trace_event::MemoryAllocatorDump::kUnitsBytes,
frame_resource->is_used() ? 0 : buffer_size_in_bytes);
shared_image->OnMemoryDump(pmd, dump->guid(), kImportance);
}
}
return true;
}
void GpuMemoryBufferVideoFramePool::PoolImpl::Abort() {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
if (frame_copy_requests_.size() <= 1u) {
return;
}
frame_copy_requests_.erase(frame_copy_requests_.begin() + 1,
frame_copy_requests_.end());
}
void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDone(
bool copy_failed,
scoped_refptr<VideoFrame> video_frame,
FrameResource* frame_resource) {
TRACE_EVENT_END("media",
perfetto::NamedTrack(
"CopyVideoFrameToGpuMemoryBuffer",
video_frame->timestamp().InNanoseconds()));
media_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&PoolImpl::OnCopiesDoneOnMediaThread, this, copy_failed,
std::move(video_frame), frame_resource));
}
void GpuMemoryBufferVideoFramePool::PoolImpl::StartCopy() {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
DCHECK(!frame_copy_requests_.empty());
while (!frame_copy_requests_.empty()) {
VideoFrameCopyRequest& request = frame_copy_requests_.front();
auto output_color_space =
request.passthrough
? request.video_frame->ColorSpace()
: GetOutputColorSpace(request.video_frame->ColorSpace(),
output_format_);
FrameResource* frame_resource =
request.passthrough
? nullptr
: GetOrCreateFrameResource(
CodedSize(request.video_frame.get(), output_format_),
gfx::BufferUsage::SCANOUT_CPU_READ_WRITE, output_color_space);
if (!frame_resource || !frame_resource->shared_image ||
!(frame_resource->scoped_mapping =
frame_resource->shared_image->Map())) {
if (frame_resource) {
DLOG(ERROR) << "Could not get or map buffer.";
}
std::move(request.frame_ready_cb).Run(std::move(request.video_frame));
frame_copy_requests_.pop_front();
continue;
}
worker_task_runner_->PostTask(
FROM_HERE, base::BindOnce(&PoolImpl::CopyVideoFrameToGpuMemoryBuffer,
this, request.video_frame, frame_resource));
break;
}
}
void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffer(
scoped_refptr<VideoFrame> video_frame,
FrameResource* frame_resource) {
CHECK(frame_resource);
CHECK(frame_resource->shared_image);
CHECK(frame_resource->scoped_mapping);
auto on_copies_done =
base::BindOnce(&PoolImpl::OnCopiesDone, this, false,
video_frame, frame_resource);
TRACE_EVENT_BEGIN(
"media", "CopyVideoFrameToGpuMemoryBuffer",
perfetto::NamedTrack("CopyVideoFrameToGpuMemoryBuffer",
video_frame->timestamp().InNanoseconds()));
const gfx::Size coded_size = CodedSize(video_frame.get(), output_format_);
size_t copies = 0;
const int rows = VideoFrame::Rows(0, VideoFormat(output_format_),
coded_size.height());
const int rows_per_copy =
RowsPerCopy(VideoFormat(output_format_), coded_size.width());
copies += rows / rows_per_copy;
if (rows % rows_per_copy) {
++copies;
}
if (copies == 1) {
DCHECK_LE(rows, rows_per_copy);
CopyRowsToBuffer(output_format_, 0, rows, coded_size,
video_frame.get(), frame_resource,
std::move(on_copies_done));
return;
}
const base::RepeatingClosure barrier =
base::BarrierClosure(copies, std::move(on_copies_done));
for (int row = 0; row < rows; row += rows_per_copy) {
const int rows_to_copy = std::min(rows_per_copy, rows - row);
worker_task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&CopyRowsToBuffer, output_format_, row, rows_to_copy,
coded_size, base::Unretained(video_frame.get()),
frame_resource, barrier));
}
}
void GpuMemoryBufferVideoFramePool::PoolImpl::CopyRowsToBuffer(
GpuVideoAcceleratorFactories::OutputFormat output_format,
const size_t row,
const size_t rows_to_copy,
const gfx::Size coded_size,
const VideoFrame* video_frame,
FrameResource* frame_resource,
base::OnceClosure done) {
base::ScopedClosureRunner done_runner(std::move(done));
auto* scoped_mapping = frame_resource->scoped_mapping.get();
base::span<uint8_t> memory_ptr0 = scoped_mapping->GetMemoryForPlane(0);
size_t stride0 = scoped_mapping->Stride(0);
switch (output_format) {
case GpuVideoAcceleratorFactories::OutputFormat::YV12: {
DCHECK(video_frame->format() == PIXEL_FORMAT_I420 ||
video_frame->format() == PIXEL_FORMAT_YUV420P10)
<< VideoPixelFormatToString(video_frame->format());
VideoPixelFormat pixel_format = VideoFormat(output_format);
for (int dst_plane = 0; dst_plane < 3; ++dst_plane) {
constexpr static std::array<VideoFrame::Plane, 3> kSrcPlanes = {
VideoFrame::Plane::kY,
VideoFrame::Plane::kV,
VideoFrame::Plane::kU,
};
VideoFrame::Plane src_plane = kSrcPlanes[dst_plane];
const size_t plane_row_start =
row / VideoFrame::SampleSize(pixel_format, src_plane).height();
const size_t plane_rows_to_copy =
VideoFrame::Rows(src_plane, pixel_format, rows_to_copy);
const size_t plane_bytes_per_row =
VideoFrame::RowBytes(src_plane, pixel_format, coded_size.width());
CopyRowsToI420Buffer(plane_row_start, plane_rows_to_copy,
plane_bytes_per_row, video_frame->BitDepth(),
video_frame->visible_data(src_plane),
video_frame->stride(src_plane),
scoped_mapping->GetMemoryForPlane(dst_plane),
scoped_mapping->Stride(dst_plane));
}
break;
}
case GpuVideoAcceleratorFactories::OutputFormat::P010:
CopyRowsToP010Buffer(row, rows_to_copy, coded_size.width(), video_frame,
memory_ptr0, stride0,
scoped_mapping->GetMemoryForPlane(1),
scoped_mapping->Stride(1));
break;
case GpuVideoAcceleratorFactories::OutputFormat::NV12:
CopyRowsToNV12Buffer(row, rows_to_copy, coded_size.width(),
video_frame->BitDepth(), video_frame, memory_ptr0,
stride0, scoped_mapping->GetMemoryForPlane(1),
scoped_mapping->Stride(1));
break;
case GpuVideoAcceleratorFactories::OutputFormat::XB30:
case GpuVideoAcceleratorFactories::OutputFormat::XR30: {
const bool is_rgba =
output_format == GpuVideoAcceleratorFactories::OutputFormat::XB30;
CopyRowsToRGB10Buffer(is_rgba, row, rows_to_copy, coded_size.width(),
video_frame, memory_ptr0, stride0);
break;
}
case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
NOTREACHED();
}
}
void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDoneOnMediaThread(
bool copy_failed,
scoped_refptr<VideoFrame> video_frame,
FrameResource* frame_resource) {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
if (frame_resource->scoped_mapping) {
frame_resource->scoped_mapping.reset();
}
if (copy_failed) {
if (!in_shutdown_) {
auto it = std::ranges::find(resources_pool_, frame_resource);
CHECK(it != resources_pool_.end());
resources_pool_.erase(it);
}
DeleteFrameResource(gpu_factories_, frame_resource);
delete frame_resource;
CompleteCopyRequestAndMaybeStartNextCopy(std::move(video_frame));
return;
}
scoped_refptr<VideoFrame> frame = BindAndCreateMailboxHardwareFrameResource(
frame_resource, CodedSize(video_frame.get(), output_format_),
gfx::Rect(video_frame->visible_rect().size()),
video_frame->natural_size(), video_frame->ColorSpace(),
video_frame->timestamp(), video_frame->metadata().allow_overlay);
if (!frame) {
CompleteCopyRequestAndMaybeStartNextCopy(std::move(video_frame));
return;
}
bool new_allow_overlay = frame->metadata().allow_overlay;
bool new_read_lock_fences_enabled =
frame->metadata().read_lock_fences_enabled;
frame->set_hdr_metadata(video_frame->hdr_metadata());
frame->metadata().MergeMetadataFrom(video_frame->metadata());
frame->metadata().allow_overlay = new_allow_overlay;
frame->metadata().read_lock_fences_enabled = new_read_lock_fences_enabled;
CompleteCopyRequestAndMaybeStartNextCopy(std::move(frame));
}
scoped_refptr<VideoFrame> GpuMemoryBufferVideoFramePool::PoolImpl::
BindAndCreateMailboxHardwareFrameResource(
FrameResource* frame_resource,
const gfx::Size& coded_size,
const gfx::Rect& visible_rect,
const gfx::Size& natural_size,
const gfx::ColorSpace& color_space,
base::TimeDelta timestamp,
bool video_frame_allow_overlay) {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
gpu::SharedImageInterface* sii = gpu_factories_->SharedImageInterface();
if (!sii) {
frame_resource->MarkUnused(tick_clock_->NowTicks());
return nullptr;
}
bool is_webgpu_compatible = false;
CHECK(frame_resource->shared_image);
auto handle = frame_resource->shared_image->CloneGpuMemoryBufferHandle();
auto name = (handle.type == gfx::GpuMemoryBufferType::SHARED_MEMORY_BUFFER)
? std::string("Media.GPU.OutputFormatSoftwareGmb")
: std::string("Media.GPU.OutputFormatHardwareGmb");
base::UmaHistogramEnumeration(name, output_format_);
#if BUILDFLAG(IS_MAC)
is_webgpu_compatible =
media::IOSurfaceIsWebGPUCompatible(handle.io_surface().get());
#endif
#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
is_webgpu_compatible =
handle.type == gfx::NATIVE_PIXMAP &&
handle.native_pixmap_handle().supports_zero_copy_webgpu_import;
#endif
sii->UpdateSharedImage(frame_resource->sync_token,
frame_resource->shared_image->mailbox());
gpu::SyncToken sync_token = sii->GenUnverifiedSyncToken();
VideoPixelFormat frame_format = VideoFormat(output_format_);
scoped_refptr<VideoFrame> frame = VideoFrame::WrapSharedImage(
frame_format, frame_resource->shared_image, sync_token,
VideoFrame::ReleaseMailboxCB(), coded_size, visible_rect, natural_size,
timestamp);
if (!frame) {
frame_resource->MarkUnused(tick_clock_->NowTicks());
MailboxHolderReleased(frame_resource, sync_token);
return nullptr;
}
frame->SetReleaseMailboxCB(
base::BindOnce(&PoolImpl::MailboxHolderReleased, this, frame_resource));
frame->set_color_space(frame_resource->shared_image->color_space());
bool allow_overlay = false;
if (frame_resource->shared_image->usage().Has(
gpu::SHARED_IMAGE_USAGE_SCANOUT)) {
#if BUILDFLAG(IS_WIN)
allow_overlay =
output_format_ == GpuVideoAcceleratorFactories::OutputFormat::NV12;
#else
switch (output_format_) {
case GpuVideoAcceleratorFactories::OutputFormat::YV12:
allow_overlay = video_frame_allow_overlay;
break;
case GpuVideoAcceleratorFactories::OutputFormat::P010:
case GpuVideoAcceleratorFactories::OutputFormat::NV12:
allow_overlay = true;
break;
case GpuVideoAcceleratorFactories::OutputFormat::XR30:
case GpuVideoAcceleratorFactories::OutputFormat::XB30:
#if BUILDFLAG(IS_APPLE)
allow_overlay = IOSurfaceCanSetColorSpace(color_space);
#else
allow_overlay = false;
#endif
break;
case GpuVideoAcceleratorFactories::OutputFormat::UNDEFINED:
break;
}
#endif
}
frame->metadata().allow_overlay = allow_overlay;
frame->metadata().read_lock_fences_enabled = true;
frame->metadata().is_webgpu_compatible = is_webgpu_compatible;
return frame;
}
GpuMemoryBufferVideoFramePool::PoolImpl::~PoolImpl() {
DCHECK(in_shutdown_);
}
void GpuMemoryBufferVideoFramePool::PoolImpl::Shutdown() {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
Abort();
in_shutdown_ = true;
for (FrameResource* frame_resource : resources_pool_) {
if (frame_resource->is_used()) {
continue;
}
media_task_runner_->PostTask(
FROM_HERE, base::BindOnce(&PoolImpl::DeleteFrameResource,
gpu_factories_, base::Owned(frame_resource)));
}
resources_pool_.clear();
}
void GpuMemoryBufferVideoFramePool::PoolImpl::SetTickClockForTesting(
const base::TickClock* tick_clock) {
tick_clock_ = tick_clock;
}
GpuMemoryBufferVideoFramePool::PoolImpl::FrameResource*
GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResource(
const gfx::Size& size,
gfx::BufferUsage usage,
const gfx::ColorSpace& color_space) {
DCHECK(media_task_runner_->RunsTasksInCurrentSequence());
auto it = resources_pool_.begin();
while (it != resources_pool_.end()) {
FrameResource* frame_resource = *it;
if (!frame_resource->is_used()) {
if (IsFrameResourceCompatible(frame_resource, size, usage, color_space)) {
frame_resource->MarkUsed();
return frame_resource;
} else {
resources_pool_.erase(it++);
DeleteFrameResource(gpu_factories_, frame_resource);
delete frame_resource;
}
} else {
it++;
}
}
FrameResource* frame_resource = new FrameResource(size, usage, color_space);
resources_pool_.push_back(frame_resource);
frame_resource->buffer_id = ++buffer_id_;
if (auto* sii = gpu_factories_->SharedImageInterface()) {
viz::SharedImageFormat si_format =
OutputFormatToSharedImageFormat(output_format_);
SetPrefersExternalSampler(si_format);
gpu::SharedImageUsageSet si_usage = gpu::SHARED_IMAGE_USAGE_GLES2_READ |
gpu::SHARED_IMAGE_USAGE_RASTER_READ |
gpu::SHARED_IMAGE_USAGE_DISPLAY_READ;
auto si_caps = sii->GetCapabilities();
#if BUILDFLAG(IS_WIN)
bool add_scanout_usage =
si_caps.supports_scanout_shared_images_for_software_video_frames;
#else
bool add_scanout_usage = si_caps.supports_scanout_shared_images;
#endif
if (add_scanout_usage) {
si_usage |= gpu::SHARED_IMAGE_USAGE_SCANOUT;
}
#if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
if (base::CommandLine::ForCurrentProcess()->HasSwitch(
switches::kEnableUnsafeWebGPU)) {
si_usage |= gpu::SHARED_IMAGE_USAGE_WEBGPU_READ;
}
#elif BUILDFLAG(IS_MAC)
si_usage |= gpu::SHARED_IMAGE_USAGE_WEBGPU_READ;
#endif
frame_resource->shared_image =
sii->CreateSharedImage({si_format, size, color_space, si_usage,
"MediaGmbVideoFramePoolMappableSI"},
gpu::kNullSurfaceHandle, usage);
return frame_resource;
}
return nullptr;
}
void GpuMemoryBufferVideoFramePool::PoolImpl::
CompleteCopyRequestAndMaybeStartNextCopy(
scoped_refptr<VideoFrame> video_frame) {
DCHECK(!frame_copy_requests_.empty());
std::move(frame_copy_requests_.front().frame_ready_cb)
.Run(std::move(video_frame));
frame_copy_requests_.pop_front();
if (!frame_copy_requests_.empty()) {
StartCopy();
}
}
void GpuMemoryBufferVideoFramePool::PoolImpl::DeleteFrameResource(
GpuVideoAcceleratorFactories* const gpu_factories,
FrameResource* frame_resource) {
if (!gpu_factories->SharedImageInterface()) {
return;
}
if (frame_resource->shared_image) {
frame_resource->shared_image->UpdateDestructionSyncToken(
frame_resource->sync_token);
}
}
void GpuMemoryBufferVideoFramePool::PoolImpl::MailboxHolderReleased(
FrameResource* frame_resource,
const gpu::SyncToken& release_sync_token) {
if (!media_task_runner_->RunsTasksInCurrentSequence()) {
media_task_runner_->PostTask(
FROM_HERE, base::BindOnce(&PoolImpl::MailboxHolderReleased, this,
frame_resource, release_sync_token));
return;
}
frame_resource->sync_token = release_sync_token;
if (in_shutdown_) {
DeleteFrameResource(gpu_factories_, frame_resource);
delete frame_resource;
return;
}
const base::TimeTicks now = tick_clock_->NowTicks();
frame_resource->MarkUnused(now);
auto it = resources_pool_.begin();
while (it != resources_pool_.end()) {
FrameResource* resource = *it;
constexpr base::TimeDelta kStaleFrameLimit = base::Seconds(10);
if (!resource->is_used() &&
now - resource->last_use_time() > kStaleFrameLimit) {
resources_pool_.erase(it++);
DeleteFrameResource(gpu_factories_, resource);
delete resource;
} else {
it++;
}
}
}
GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool() = default;
GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool(
const scoped_refptr<base::SequencedTaskRunner>& media_task_runner,
const scoped_refptr<base::TaskRunner>& worker_task_runner,
GpuVideoAcceleratorFactories* gpu_factories)
: pool_impl_(base::MakeRefCounted<PoolImpl>(media_task_runner,
worker_task_runner,
gpu_factories)) {
base::trace_event::MemoryDumpManager::GetInstance()
->RegisterDumpProviderWithSequencedTaskRunner(
pool_impl_.get(), "GpuMemoryBufferVideoFramePool", media_task_runner,
base::trace_event::MemoryDumpProvider::Options());
}
GpuMemoryBufferVideoFramePool::~GpuMemoryBufferVideoFramePool() {
if (!pool_impl_) {
return;
}
pool_impl_->Shutdown();
base::trace_event::MemoryDumpManager::GetInstance()->UnregisterDumpProvider(
pool_impl_.get());
}
void GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame(
scoped_refptr<VideoFrame> video_frame,
FrameReadyCB frame_ready_cb) {
DCHECK(video_frame);
pool_impl_->CreateHardwareFrame(std::move(video_frame),
std::move(frame_ready_cb));
}
void GpuMemoryBufferVideoFramePool::Abort() {
pool_impl_->Abort();
}
void GpuMemoryBufferVideoFramePool::SetTickClockForTesting(
const base::TickClock* tick_clock) {
pool_impl_->SetTickClockForTesting(tick_clock);
}
}