#ifdef UNSAFE_BUFFERS_BUILD
#pragma allow_unsafe_buffers
#endif
#include "media/capture/video/fuchsia/video_capture_device_fuchsia.h"
#include <zircon/status.h>
#include "base/fuchsia/fuchsia_logging.h"
#include "base/strings/stringprintf.h"
#include "base/time/time.h"
#include "media/base/video_types.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/video_common.h"
namespace media {
namespace {
size_t RoundUp(size_t value, size_t alignment) {
return ((value + alignment - 1) / alignment) * alignment;
}
libyuv::FourCC GetFourccForPixelFormat(
fuchsia::images2::PixelFormat src_pixel_format) {
switch (src_pixel_format) {
case fuchsia::images2::PixelFormat::I420:
return libyuv::FourCC::FOURCC_I420;
case fuchsia::images2::PixelFormat::YV12:
return libyuv::FourCC::FOURCC_YV12;
case fuchsia::images2::PixelFormat::NV12:
return libyuv::FourCC::FOURCC_NV12;
default:
NOTREACHED();
}
}
libyuv::RotationMode CameraOrientationToLibyuvRotation(
fuchsia::camera3::Orientation orientation,
bool* flip_y) {
switch (orientation) {
case fuchsia::camera3::Orientation::UP:
*flip_y = false;
return libyuv::RotationMode::kRotate0;
case fuchsia::camera3::Orientation::DOWN:
*flip_y = false;
return libyuv::RotationMode::kRotate180;
case fuchsia::camera3::Orientation::LEFT:
*flip_y = false;
return libyuv::RotationMode::kRotate270;
case fuchsia::camera3::Orientation::RIGHT:
*flip_y = false;
return libyuv::RotationMode::kRotate90;
case fuchsia::camera3::Orientation::UP_FLIPPED:
*flip_y = true;
return libyuv::RotationMode::kRotate180;
case fuchsia::camera3::Orientation::DOWN_FLIPPED:
*flip_y = true;
return libyuv::RotationMode::kRotate0;
case fuchsia::camera3::Orientation::LEFT_FLIPPED:
*flip_y = true;
return libyuv::RotationMode::kRotate90;
case fuchsia::camera3::Orientation::RIGHT_FLIPPED:
*flip_y = true;
return libyuv::RotationMode::kRotate270;
}
}
gfx::Size RotateSize(gfx::Size size, libyuv::RotationMode rotation) {
switch (rotation) {
case libyuv::RotationMode::kRotate0:
case libyuv::RotationMode::kRotate180:
return size;
case libyuv::RotationMode::kRotate90:
case libyuv::RotationMode::kRotate270:
return gfx::Size(size.height(), size.width());
}
}
}
VideoPixelFormat VideoCaptureDeviceFuchsia::GetConvertedPixelFormat(
fuchsia::images2::PixelFormat format) {
switch (format) {
case fuchsia::images2::PixelFormat::I420:
case fuchsia::images2::PixelFormat::YV12:
case fuchsia::images2::PixelFormat::NV12:
return PIXEL_FORMAT_I420;
default:
LOG(ERROR) << "Camera uses unsupported pixel format "
<< static_cast<int>(format);
return PIXEL_FORMAT_UNKNOWN;
}
}
VideoPixelFormat VideoCaptureDeviceFuchsia::GetConvertedPixelFormat(
fuchsia::sysmem::PixelFormatType format) {
auto images2_pixel_format =
static_cast<fuchsia::images2::PixelFormat>(fidl::ToUnderlying(format));
return GetConvertedPixelFormat(images2_pixel_format);
}
bool VideoCaptureDeviceFuchsia::IsSupportedPixelFormat(
fuchsia::images2::PixelFormat format) {
return GetConvertedPixelFormat(format) != PIXEL_FORMAT_UNKNOWN;
}
VideoCaptureDeviceFuchsia::VideoCaptureDeviceFuchsia(
fidl::InterfaceHandle<fuchsia::camera3::Device> device)
: sysmem_allocator_("CrVideoCaptureDeviceFuchsia") {
device_.Bind(std::move(device));
device_.set_error_handler(
fit::bind_member(this, &VideoCaptureDeviceFuchsia::OnDeviceError));
}
VideoCaptureDeviceFuchsia::~VideoCaptureDeviceFuchsia() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
}
void VideoCaptureDeviceFuchsia::AllocateAndStart(
const VideoCaptureParams& params,
std::unique_ptr<Client> client) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DCHECK_EQ(params.requested_format.pixel_format, PIXEL_FORMAT_I420);
DCHECK(!client_);
DCHECK(!stream_);
client_ = std::move(client);
if (!device_) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaCameraDeviceDisconnected,
"fuchsia.camera3.Device disconnected");
return;
}
start_time_ = base::TimeTicks::Now();
frames_received_ = 0;
device_->ConnectToStream(0, stream_.NewRequest());
stream_.set_error_handler(
fit::bind_member(this, &VideoCaptureDeviceFuchsia::OnStreamError));
WatchResolution();
WatchOrientation();
stream_->SetBufferCollection2(sysmem_allocator_.CreateNewToken());
WatchBufferCollection();
}
void VideoCaptureDeviceFuchsia::StopAndDeAllocate() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DisconnectStream();
client_.reset();
}
void VideoCaptureDeviceFuchsia::OnDeviceError(zx_status_t status) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaCameraDeviceDisconnected,
base::StringPrintf("fuchsia.camera3.Device disconnected: %s (%d)",
zx_status_get_string(status), status));
}
void VideoCaptureDeviceFuchsia::OnStreamError(zx_status_t status) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaCameraStreamDisconnected,
base::StringPrintf("fuchsia.camera3.Stream disconnected: %s (%d)",
zx_status_get_string(status), status));
}
void VideoCaptureDeviceFuchsia::DisconnectStream() {
stream_.Unbind();
buffer_collection_.reset();
buffers_.clear();
frame_size_.reset();
}
void VideoCaptureDeviceFuchsia::OnError(base::Location location,
VideoCaptureError error,
const std::string& reason) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DisconnectStream();
if (client_) {
client_->OnError(error, location, reason);
}
}
void VideoCaptureDeviceFuchsia::WatchResolution() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
stream_->WatchResolution(fit::bind_member(
this, &VideoCaptureDeviceFuchsia::OnWatchResolutionResult));
}
void VideoCaptureDeviceFuchsia::OnWatchResolutionResult(
fuchsia::math::Size frame_size) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
frame_size_ = gfx::Size(frame_size.width, frame_size.height);
WatchResolution();
}
void VideoCaptureDeviceFuchsia::WatchOrientation() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
stream_->WatchOrientation(fit::bind_member(
this, &VideoCaptureDeviceFuchsia::OnWatchOrientationResult));
}
void VideoCaptureDeviceFuchsia::OnWatchOrientationResult(
fuchsia::camera3::Orientation orientation) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
orientation_ = orientation;
WatchOrientation();
}
void VideoCaptureDeviceFuchsia::WatchBufferCollection() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
stream_->WatchBufferCollection2(
[this](fidl::InterfaceHandle<fuchsia::sysmem2::BufferCollectionToken>
token_handle) {
InitializeBufferCollection(std::move(token_handle));
WatchBufferCollection();
});
}
void VideoCaptureDeviceFuchsia::InitializeBufferCollection(
fidl::InterfaceHandle<fuchsia::sysmem2::BufferCollectionToken>
token_handle) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
buffer_collection_.reset();
buffers_.clear();
fuchsia::sysmem2::BufferCollectionTokenPtr token;
token.Bind(std::move(token_handle));
const size_t kMaxUsedOutputFrames = 1;
constexpr uint32_t kNamePriority = 10;
fuchsia::sysmem2::BufferCollectionConstraints constraints =
VmoBuffer::GetRecommendedConstraints(kMaxUsedOutputFrames,
std::nullopt,
false);
buffer_collection_ = sysmem_allocator_.BindSharedCollection(std::move(token));
buffer_collection_->Initialize(std::move(constraints), "CrVideoCaptureDevice",
kNamePriority);
buffer_collection_->AcquireBuffers(base::BindOnce(
&VideoCaptureDeviceFuchsia::OnBuffersAcquired, base::Unretained(this)));
}
void VideoCaptureDeviceFuchsia::OnBuffersAcquired(
std::vector<VmoBuffer> buffers,
const fuchsia::sysmem2::SingleBufferSettings& buffer_settings) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
if (buffers.empty()) {
buffer_collection_.reset();
return;
}
buffers_ = std::move(buffers);
if (!buffer_settings.has_image_format_constraints()) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaSysmemDidNotSetImageFormat,
"Sysmem created buffer without image format constraints");
return;
}
auto pixel_format = buffer_settings.image_format_constraints().pixel_format();
if (!IsSupportedPixelFormat(pixel_format)) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaUnsupportedPixelFormat,
base::StringPrintf("Unsupported video frame format: %d",
static_cast<int>(pixel_format)));
return;
}
buffers_format_ = fidl::Clone(buffer_settings.image_format_constraints());
if (!started_) {
started_ = true;
client_->OnStarted();
ReceiveNextFrame();
}
}
void VideoCaptureDeviceFuchsia::ReceiveNextFrame() {
stream_->GetNextFrame([this](fuchsia::camera3::FrameInfo frame_info) {
ProcessNewFrame(std::move(frame_info));
ReceiveNextFrame();
});
}
void VideoCaptureDeviceFuchsia::ProcessNewFrame(
fuchsia::camera3::FrameInfo frame_info) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DCHECK(client_);
if (buffers_.empty()) {
DLOG(WARNING) << "Dropping frame received before sysmem collection has "
"been initialized.";
return;
}
size_t index = frame_info.buffer_index;
if (index >= buffers_.size()) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaSysmemInvalidBufferIndex,
base::StringPrintf("Received frame with invalid buffer_index=%zu",
index));
return;
}
size_t src_coded_width =
RoundUp(std::max(buffers_format_.min_size().width,
buffers_format_.required_max_size().width),
buffers_format_.size_alignment().width);
size_t src_coded_height =
RoundUp(std::max(buffers_format_.min_size().height,
buffers_format_.required_max_size().height),
buffers_format_.size_alignment().height);
size_t src_stride =
RoundUp(std::max(static_cast<size_t>(buffers_format_.min_bytes_per_row()),
src_coded_width),
buffers_format_.bytes_per_row_divisor());
gfx::Size visible_size =
frame_size_.value_or(gfx::Size(src_coded_width, src_coded_height));
gfx::Size nonrotated_output_size((visible_size.width() + 1) & ~1,
(visible_size.height() + 1) & ~1);
bool flip_y;
libyuv::RotationMode rotation =
CameraOrientationToLibyuvRotation(orientation_, &flip_y);
gfx::Size output_size = RotateSize(nonrotated_output_size, rotation);
visible_size = RotateSize(visible_size, rotation);
base::TimeTicks reference_time =
base::TimeTicks::FromZxTime(frame_info.timestamp);
base::TimeDelta timestamp =
std::max(reference_time - start_time_, base::TimeDelta());
++frames_received_;
float frame_rate =
(timestamp.is_positive())
? static_cast<float>(frames_received_) / timestamp.InSecondsF()
: 0.0;
VideoCaptureFormat capture_format(output_size, frame_rate, PIXEL_FORMAT_I420);
Client::Buffer buffer;
Client::ReserveResult result = client_->ReserveOutputBuffer(
capture_format.frame_size, capture_format.pixel_format,
0, &buffer, nullptr,
nullptr);
if (result != Client::ReserveResult::kSucceeded) {
DLOG(WARNING) << "Failed to allocate output buffer for a video frame";
return;
}
auto src_span = buffers_[index].GetMemory();
if (src_span.empty()) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaFailedToMapSysmemBuffer,
"Failed to map buffers allocated by sysmem");
return;
}
size_t src_buffer_size = src_coded_height * src_stride * 3 / 2;
if (src_span.size() < src_buffer_size) {
OnError(FROM_HERE, VideoCaptureError::kFuchsiaSysmemInvalidBufferSize,
"Sysmem allocated buffer that's smaller than expected");
return;
}
std::unique_ptr<VideoCaptureBufferHandle> output_handle =
buffer.handle_provider->GetHandleForInProcessAccess();
uint8_t* dst_y = output_handle->data().data();
int dst_stride_y = output_size.width();
size_t dst_y_plane_size = output_size.width() * output_size.height();
uint8_t* dst_u = dst_y + dst_y_plane_size;
int dst_stride_u = output_size.width() / 2;
uint8_t* dst_v = dst_u + dst_y_plane_size / 4;
int dst_stride_v = output_size.width() / 2;
const uint8_t* dst_end = dst_v + dst_y_plane_size / 4;
CHECK_LE(dst_end,
output_handle->data().data() + output_handle->mapped_size());
int flipped_src_height = static_cast<int>(src_coded_height);
if (flip_y)
flipped_src_height = -flipped_src_height;
auto four_cc = GetFourccForPixelFormat(buffers_format_.pixel_format());
libyuv::ConvertToI420(src_span.data(), src_span.size(), dst_y, dst_stride_y,
dst_u, dst_stride_u, dst_v, dst_stride_v,
0, 0, src_stride,
flipped_src_height, nonrotated_output_size.width(),
nonrotated_output_size.height(), rotation, four_cc);
client_->OnIncomingCapturedBufferExt(
std::move(buffer), capture_format, gfx::ColorSpace(), reference_time,
timestamp, std::nullopt, gfx::Rect(visible_size), VideoFrameMetadata());
}
}