#include "device/vr/android/cardboard/cardboard_render_loop.h"
#include <memory>
#include "base/task/bind_post_task.h"
#include "device/vr/android/cardboard/cardboard_image_transport.h"
#include "device/vr/android/cardboard/cardboard_sdk.h"
#include "device/vr/android/xr_java_coordinator.h"
#include "device/vr/public/mojom/isolated_xr_service.mojom.h"
#include "device/vr/public/mojom/vr_service.mojom-shared.h"
#include "mojo/public/cpp/bindings/associated_receiver.h"
#include "ui/gfx/geometry/transform.h"
#include "ui/gl/gl_bindings.h"
#include "ui/gl/gl_bindings_autogen_gl.h"
#include "ui/gl/gl_context.h"
#include "ui/gl/gl_fence_android_native_fence_sync.h"
#include "ui/gl/gl_surface.h"
#include "ui/gl/gl_utils.h"
#include "ui/gl/init/gl_factory.h"
namespace device {
CardboardRenderLoop::CardboardRenderLoop(
std::unique_ptr<CardboardImageTransportFactory>
cardboard_image_transport_factory,
std::unique_ptr<MailboxToSurfaceBridge> mailbox_bridge)
: base::android::JavaHandlerThread("CardboardRenderLoop"),
cardboard_image_transport_factory_(
std::move(cardboard_image_transport_factory)),
mailbox_bridge_(std::move(mailbox_bridge)),
webxr_(std::make_unique<WebXrPresentationState>()) {}
CardboardRenderLoop::~CardboardRenderLoop() {
Stop();
}
void CardboardRenderLoop::GetEnvironmentIntegrationProvider(
mojo::PendingAssociatedReceiver<
device::mojom::XREnvironmentIntegrationProvider> environment_provider) {
frame_data_receiver_.ReportBadMessage(
"Environment integration is not supported.");
}
void CardboardRenderLoop::SetInputSourceButtonListener(
mojo::PendingAssociatedRemote<device::mojom::XRInputSourceButtonListener>) {
frame_data_receiver_.ReportBadMessage("Input eventing is not supported.");
}
void CardboardRenderLoop::CreateSession(
CardboardRequestSessionCallback session_request_callback,
base::OnceClosure session_shutdown_callback,
XrJavaCoordinator* java_coordinator,
CardboardSdk* cardboard_sdk,
gfx::AcceleratedWidget drawing_widget,
const gfx::Size& frame_size,
display::Display::Rotation display_rotation,
mojom::XRRuntimeSessionOptionsPtr options) {
DCHECK(task_runner()->BelongsToCurrentThread());
CHECK(!session_request_callback_);
CHECK(!frame_size.IsEmpty());
DVLOG(1) << __func__;
cardboard_image_transport_ =
cardboard_image_transport_factory_->Create(std::move(mailbox_bridge_));
cardboard_sdk_ = cardboard_sdk;
session_request_callback_ = std::move(session_request_callback);
session_shutdown_callback_ = std::move(session_shutdown_callback);
texture_size_ = frame_size;
enabled_features_.insert(options->required_features.begin(),
options->required_features.end());
enabled_features_.insert(options->optional_features.begin(),
options->optional_features.end());
left_eye_ = mojom::XRView::New();
left_eye_->eye = mojom::XREye::kLeft;
left_eye_->viewport =
gfx::Rect(0, 0, texture_size_.width() / 2, texture_size_.height());
right_eye_ = mojom::XRView::New();
right_eye_->eye = mojom::XREye::kRight;
right_eye_->viewport =
gfx::Rect(texture_size_.width() / 2, 0, texture_size_.width() / 2,
texture_size_.height());
left_eye_->mojo_from_view = gfx::Transform();
left_eye_->field_of_view =
cardboard_image_transport_->GetFOV(CardboardEye::kLeft, texture_size_);
right_eye_->mojo_from_view = gfx::Transform();
right_eye_->field_of_view =
cardboard_image_transport_->GetFOV(CardboardEye::kRight, texture_size_);
if (!InitializeGl(drawing_widget)) {
std::move(session_request_callback_).Run(nullptr);
return;
}
base::android::ScopedJavaLocalRef<jobject> application_context =
java_coordinator->GetApplicationContext();
if (!application_context.obj()) {
DLOG(ERROR) << "Unable to retrieve the Java context/activity!";
std::move(session_request_callback_).Run(nullptr);
return;
}
cardboard_sdk_->Initialize(application_context.obj());
cardboard_image_transport_->Initialize(
webxr_.get(),
base::BindOnce(&CardboardRenderLoop::OnCardboardImageTransportReady,
weak_ptr_factory_.GetWeakPtr()));
}
bool CardboardRenderLoop::InitializeGl(gfx::AcceleratedWidget drawing_widget) {
DVLOG(1) << __func__;
DCHECK(task_runner()->BelongsToCurrentThread());
CHECK(drawing_widget);
gl::init::DisableANGLE();
gl::GLDisplay* display = nullptr;
if (gl::GetGLImplementation() == gl::kGLImplementationNone) {
display = gl::init::InitializeGLOneOff(
gl::GpuPreference::kDefault);
if (!display) {
DLOG(ERROR) << "gl::init::InitializeGLOneOff failed";
return false;
}
} else {
display = gl::GetDefaultDisplayEGL();
}
scoped_refptr<gl::GLSurface> surface =
gl::init::CreateViewGLSurface(display, drawing_widget);
DVLOG(3) << "surface=" << surface.get();
if (!surface.get()) {
DLOG(ERROR) << "gl::init::CreateViewGLSurface failed";
return false;
}
scoped_refptr<gl::GLContext> context =
gl::init::CreateGLContext(nullptr, surface.get(), gl::GLContextAttribs());
if (!context.get()) {
DLOG(ERROR) << "gl::init::CreateGLContext failed";
return false;
}
if (!context->MakeCurrent(surface.get())) {
DLOG(ERROR) << "gl::GLContext::MakeCurrent() failed";
return false;
}
surface_ = std::move(surface);
context_ = std::move(context);
return true;
}
void CardboardRenderLoop::OnBindingDisconnect() {
DVLOG(1) << __func__;
CloseBindingsIfOpen();
pending_shutdown_ = true;
if (session_shutdown_callback_) {
std::move(session_shutdown_callback_).Run();
}
}
void CardboardRenderLoop::CloseBindingsIfOpen() {
DVLOG(1) << __func__;
frame_data_receiver_.reset();
session_controller_receiver_.reset();
presentation_receiver_.reset();
submit_client_.reset();
}
void CardboardRenderLoop::OnCardboardImageTransportReady(bool success) {
DCHECK(task_runner()->BelongsToCurrentThread());
DVLOG(1) << __func__ << ": success=" << success;
if (!success) {
std::move(session_request_callback_).Run(nullptr);
return;
}
webxr_->NotifyMailboxBridgeReady();
CloseBindingsIfOpen();
device::mojom::XRPresentationTransportOptionsPtr transport_options =
device::mojom::XRPresentationTransportOptions::New();
transport_options->wait_for_gpu_fence = true;
if (CardboardImageTransport::UseSharedBuffer()) {
DVLOG(2) << __func__
<< ": UseSharedBuffer()=true, DRAW_INTO_TEXTURE_MAILBOX";
transport_options->transport_method =
device::mojom::XRPresentationTransportMethod::DRAW_INTO_TEXTURE_MAILBOX;
} else {
DVLOG(2) << __func__
<< ": UseSharedBuffer()=false, SUBMIT_AS_MAILBOX_HOLDER";
transport_options->transport_method =
device::mojom::XRPresentationTransportMethod::SUBMIT_AS_MAILBOX_HOLDER;
transport_options->wait_for_transfer_notification = true;
cardboard_image_transport_->SetFrameAvailableCallback(base::BindRepeating(
&CardboardRenderLoop::RenderFrame, weak_ptr_factory_.GetWeakPtr()));
}
mojom::XRRuntimeSessionResultPtr result =
device::mojom::XRRuntimeSessionResult::New();
result->controller = session_controller_receiver_.BindNewPipeAndPassRemote();
result->session = mojom::XRSession::New();
auto* session = result->session.get();
session->data_provider = frame_data_receiver_.BindNewPipeAndPassRemote();
session->submit_frame_sink = device::mojom::XRPresentationConnection::New();
auto* submit_frame_sink = session->submit_frame_sink.get();
submit_frame_sink->client_receiver =
submit_client_.BindNewPipeAndPassReceiver();
submit_frame_sink->provider =
presentation_receiver_.BindNewPipeAndPassRemote();
submit_frame_sink->transport_options = std::move(transport_options);
session->enabled_features.assign(enabled_features_.begin(),
enabled_features_.end());
session->device_config = device::mojom::XRSessionDeviceConfig::New();
auto* config = session->device_config.get();
config->supports_viewport_scaling = false;
config->views.push_back(left_eye_.Clone());
config->views.push_back(right_eye_.Clone());
session->enviroment_blend_mode =
device::mojom::XREnvironmentBlendMode::kOpaque;
session->interaction_mode = device::mojom::XRInteractionMode::kWorldSpace;
std::move(session_request_callback_).Run(std::move(result));
frame_data_receiver_.set_disconnect_handler(
base::BindOnce(&CardboardRenderLoop::OnBindingDisconnect,
weak_ptr_factory_.GetWeakPtr()));
session_controller_receiver_.set_disconnect_handler(
base::BindOnce(&CardboardRenderLoop::OnBindingDisconnect,
weak_ptr_factory_.GetWeakPtr()));
presentation_receiver_.set_disconnect_handler(
base::BindOnce(&CardboardRenderLoop::OnBindingDisconnect,
weak_ptr_factory_.GetWeakPtr()));
submit_client_.set_disconnect_handler(
base::BindOnce(&CardboardRenderLoop::OnBindingDisconnect,
weak_ptr_factory_.GetWeakPtr()));
}
void CardboardRenderLoop::CleanUp() {
weak_ptr_factory_.InvalidateWeakPtrs();
cardboard_image_transport_->DestroySharedBuffers(webxr_.get());
cardboard_image_transport_.reset();
OnBindingDisconnect();
context_.reset();
surface_.reset();
}
void CardboardRenderLoop::GetFrameData(
mojom::XRFrameDataRequestOptionsPtr options,
mojom::XRFrameDataProvider::GetFrameDataCallback callback) {
TRACE_EVENT1("gpu", __func__, "frame", webxr_->PeekNextFrameIndex());
DCHECK(task_runner()->BelongsToCurrentThread());
CHECK(!texture_size_.IsEmpty());
if (!CanStartNewAnimatingFrame()) {
pending_getframedata_ = base::BindPostTask(
task_runner(), base::BindOnce(&CardboardRenderLoop::GetFrameData,
weak_ptr_factory_.GetWeakPtr(),
std::move(options), std::move(callback)));
return;
}
if (restrict_frame_data_) {
DVLOG(2) << __func__ << ": frame data restricted, returning nullptr.";
std::move(callback).Run(nullptr);
return;
}
if (is_paused_) {
DVLOG(2) << __func__ << ": paused but frame data not restricted. Resuming.";
Resume();
}
base::TimeTicks now = base::TimeTicks::Now();
mojom::XRFrameDataPtr frame_data = mojom::XRFrameData::New();
frame_data->frame_id = webxr_->StartFrameAnimating();
WebXrFrame* xr_frame = webxr_->GetAnimatingFrame();
xr_frame->time_pose = now;
xr_frame->bounds_left = left_bounds_;
xr_frame->bounds_right = right_bounds_;
if (CardboardImageTransport::UseSharedBuffer()) {
frame_data->buffer_holder = cardboard_image_transport_->TransferFrame(
webxr_.get(), texture_size_, gfx::Transform());
}
frame_data->views.push_back(left_eye_.Clone());
frame_data->views.push_back(right_eye_.Clone());
frame_data->mojo_from_viewer = mojom::VRPose::New();
frame_data->time_delta = now - base::TimeTicks();
std::move(callback).Run(std::move(frame_data));
}
bool CardboardRenderLoop::IsSubmitFrameExpected(int16_t frame_index) {
if (!submit_client_.get() || !webxr_->HaveAnimatingFrame()) {
return false;
}
if (pending_shutdown_) {
return false;
}
WebXrFrame* animating_frame = webxr_->GetAnimatingFrame();
animating_frame->time_js_submit = base::TimeTicks::Now();
if (animating_frame->index != frame_index) {
DVLOG(1) << __func__ << ": wrong frame index, got " << frame_index
<< ", expected " << animating_frame->index;
presentation_receiver_.ReportBadMessage(
"SubmitFrame called with wrong frame index");
OnBindingDisconnect();
return false;
}
return true;
}
void CardboardRenderLoop::SubmitFrameMissing(int16_t frame_index,
const gpu::SyncToken& sync_token) {
TRACE_EVENT1("gpu", __func__, "frame", frame_index);
DVLOG(2) << __func__;
if (!IsSubmitFrameExpected(frame_index)) {
return;
}
webxr_->RecycleUnusedAnimatingFrame();
cardboard_image_transport_->WaitSyncToken(sync_token);
FinishFrame(frame_index);
}
void CardboardRenderLoop::SubmitFrame(int16_t frame_index,
const gpu::MailboxHolder& mailbox,
base::TimeDelta time_waited) {
TRACE_EVENT1("gpu", __func__, "frame", frame_index);
DVLOG(2) << __func__ << ": frame=" << frame_index;
CHECK(!CardboardImageTransport::UseSharedBuffer());
if (!IsSubmitFrameExpected(frame_index)) {
return;
}
webxr_->ProcessOrDefer(
base::BindOnce(&CardboardRenderLoop::ProcessFrameFromMailbox,
weak_ptr_factory_.GetWeakPtr(), frame_index, mailbox));
}
void CardboardRenderLoop::ProcessFrameFromMailbox(
int16_t frame_index,
const gpu::MailboxHolder& mailbox) {
TRACE_EVENT1("gpu", __func__, "frame", frame_index);
DVLOG(2) << __func__ << ": frame=" << frame_index;
CHECK(webxr_->HaveProcessingFrame());
CHECK(!CardboardImageTransport::UseSharedBuffer());
cardboard_image_transport_->CopyMailboxToSurfaceAndSwap(
texture_size_, mailbox, gfx::Transform());
submit_client_->OnSubmitFrameTransferred(true);
}
void CardboardRenderLoop::SubmitFrameDrawnIntoTexture(
int16_t frame_index,
const gpu::SyncToken& sync_token,
base::TimeDelta time_waited) {
TRACE_EVENT1("gpu", __func__, "frame", frame_index);
DVLOG(2) << __func__ << ": frame=" << frame_index;
CHECK(CardboardImageTransport::UseSharedBuffer());
if (!IsSubmitFrameExpected(frame_index)) {
return;
}
webxr_->ProcessOrDefer(
base::BindOnce(&CardboardRenderLoop::ProcessFrameDrawnIntoTexture,
weak_ptr_factory_.GetWeakPtr(), sync_token));
}
void CardboardRenderLoop::ProcessFrameDrawnIntoTexture(
const gpu::SyncToken& sync_token) {
cardboard_image_transport_->CreateGpuFenceForSyncToken(
sync_token,
base::BindOnce(&CardboardRenderLoop::OnWebXrTokenSignaled, GetWeakPtr()));
}
void CardboardRenderLoop::OnWebXrTokenSignaled(
std::unique_ptr<gfx::GpuFence> gpu_fence) {
cardboard_image_transport_->ServerWaitForGpuFence(std::move(gpu_fence));
RenderFrame(gfx::Transform());
}
void CardboardRenderLoop::TransitionProcessingFrameToRendering() {
if (webxr_->HaveRenderingFrame()) {
DVLOG(3) << __func__ << ": wait for previous rendering frame to complete";
FinishRenderingFrame();
}
CHECK(!webxr_->HaveRenderingFrame());
CHECK(webxr_->HaveProcessingFrame());
auto* frame = webxr_->GetProcessingFrame();
frame->time_copied = base::TimeTicks::Now();
frame->render_completion_fence = nullptr;
webxr_->TransitionFrameProcessingToRendering();
webxr_->TryDeferredProcessing();
}
void CardboardRenderLoop::RenderFrame(const gfx::Transform& uv_transform) {
DVLOG(2) << __func__;
CHECK(webxr_->HaveProcessingFrame());
int16_t frame_index = webxr_->GetProcessingFrame()->index;
TRACE_EVENT1("gpu", __func__, "frame", frame_index);
TransitionProcessingFrameToRendering();
cardboard_image_transport_->Render(webxr_.get(), 0,
texture_size_);
FinishFrame(frame_index);
if (submit_client_) {
std::unique_ptr<gl::GLFence> gl_fence = gl::GLFence::CreateForGpuFence();
std::unique_ptr<gfx::GpuFence> gpu_fence2 = gl_fence->GetGpuFence();
submit_client_->OnSubmitFrameGpuFence(
gpu_fence2->GetGpuFenceHandle().Clone());
}
if (pending_getframedata_) {
std::move(pending_getframedata_).Run();
}
}
void CardboardRenderLoop::FinishRenderingFrame(WebXrFrame* frame) {
CHECK(frame || webxr_->HaveRenderingFrame());
if (!frame) {
frame = webxr_->GetRenderingFrame();
}
if (!frame->render_completion_fence) {
frame->render_completion_fence = gl::GLFence::CreateForGpuFence();
}
ClearRenderingFrame(frame);
}
void CardboardRenderLoop::ClearRenderingFrame(WebXrFrame* frame) {
TRACE_EVENT1("gpu", __func__, "frame", frame->index);
DVLOG(3) << __func__ << ": frame=" << frame->index;
DVLOG(3) << __func__ << ": client wait start";
frame->render_completion_fence->ClientWait();
DVLOG(3) << __func__ << ": client wait done";
webxr_->EndFrameRendering(frame);
}
void CardboardRenderLoop::FinishFrame(int16_t frame_index) {
TRACE_EVENT1("gpu", __func__, "frame", frame_index);
DVLOG(3) << __func__;
surface_->SwapBuffers(base::DoNothing(), gfx::FrameData());
if (!webxr_->HaveRenderingFrame()) {
return;
}
WebXrFrame* frame = webxr_->GetRenderingFrame();
frame->render_completion_fence = gl::GLFence::CreateForGpuFence();
}
bool CardboardRenderLoop::CanStartNewAnimatingFrame() {
if (pending_shutdown_) {
return false;
}
if (webxr_->HaveAnimatingFrame()) {
DVLOG(3) << __func__ << ": deferring, HaveAnimatingFrame";
return false;
}
if (!webxr_->CanStartFrameAnimating()) {
DVLOG(3) << __func__ << ": deferring, no available frames in swapchain";
return false;
}
if (webxr_->HaveProcessingFrame() && webxr_->HaveRenderingFrame()) {
DVLOG(2) << __func__ << ": wait, have processing&rendering frames";
FinishRenderingFrame();
}
if (webxr_->HaveRenderingFrame()) {
auto* frame = webxr_->GetRenderingFrame();
if (frame->render_completion_fence &&
frame->render_completion_fence->HasCompleted()) {
FinishRenderingFrame();
}
}
return true;
}
void CardboardRenderLoop::UpdateLayerBounds(int16_t frame_index,
const gfx::RectF& left_bounds,
const gfx::RectF& right_bounds,
const gfx::Size& source_size) {
DCHECK(task_runner()->BelongsToCurrentThread());
DVLOG(2) << __func__ << " source_size=" << source_size.ToString()
<< " left_bounds=" << left_bounds.ToString()
<< " right_bounds=" << right_bounds.ToString();
if (webxr_->HaveAnimatingFrame()) {
webxr_->GetAnimatingFrame()->bounds_left = left_bounds;
webxr_->GetAnimatingFrame()->bounds_right = right_bounds;
}
left_bounds_ = left_bounds;
right_bounds_ = right_bounds;
texture_size_ = source_size;
}
void CardboardRenderLoop::SetFrameDataRestricted(bool frame_data_restricted) {
DCHECK(task_runner()->BelongsToCurrentThread());
DVLOG(3) << __func__ << ": frame_data_restricted=" << frame_data_restricted;
restrict_frame_data_ = frame_data_restricted;
if (restrict_frame_data_) {
Pause();
} else {
Resume();
}
}
void CardboardRenderLoop::Pause() {
DCHECK(task_runner()->BelongsToCurrentThread());
DVLOG(1) << __func__;
is_paused_ = true;
}
void CardboardRenderLoop::Resume() {
DCHECK(task_runner()->BelongsToCurrentThread());
DVLOG(1) << __func__;
is_paused_ = false;
}
}