#include "fuchsia_web/webengine/renderer/web_engine_audio_output_device.h"
#include "base/fuchsia/fuchsia_logging.h"
#include "base/logging.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/shared_memory_mapping.h"
#include "base/memory/writable_shared_memory_region.h"
#include "base/no_destructor.h"
#include "base/task/single_thread_task_runner.h"
#include "base/threading/thread.h"
#include "base/types/pass_key.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_glitch_info.h"
#include "media/base/audio_sample_types.h"
#include "media/base/audio_timestamp_helper.h"
namespace {
constexpr size_t kNumBuffers = 4;
constexpr base::TimeDelta kLeadTimeExtra = base::Milliseconds(20);
class DefaultAudioThread {
public:
DefaultAudioThread() : thread_("WebEngineAudioOutputDevice") {
base::Thread::Options options(base::MessagePumpType::IO, 0);
options.thread_type = base::ThreadType::kRealtimeAudio;
thread_.StartWithOptions(std::move(options));
}
~DefaultAudioThread() = default;
scoped_refptr<base::SingleThreadTaskRunner> task_runner() {
return thread_.task_runner();
}
private:
base::Thread thread_;
};
scoped_refptr<base::SingleThreadTaskRunner> GetDefaultAudioTaskRunner() {
static base::NoDestructor<DefaultAudioThread> default_audio_thread;
return default_audio_thread->task_runner();
}
}
scoped_refptr<WebEngineAudioOutputDevice> WebEngineAudioOutputDevice::Create(
fidl::InterfaceHandle<fuchsia::media::AudioConsumer> audio_consumer_handle,
scoped_refptr<base::SingleThreadTaskRunner> task_runner) {
auto result = base::MakeRefCounted<WebEngineAudioOutputDevice>(
base::PassKey<WebEngineAudioOutputDevice>(), task_runner);
task_runner->PostTask(
FROM_HERE,
base::BindOnce(
&WebEngineAudioOutputDevice::BindAudioConsumerOnAudioThread, result,
std::move(audio_consumer_handle)));
return result;
}
scoped_refptr<WebEngineAudioOutputDevice>
WebEngineAudioOutputDevice::CreateOnDefaultThread(
fidl::InterfaceHandle<fuchsia::media::AudioConsumer>
audio_consumer_handle) {
return Create(std::move(audio_consumer_handle), GetDefaultAudioTaskRunner());
}
WebEngineAudioOutputDevice::WebEngineAudioOutputDevice(
base::PassKey<WebEngineAudioOutputDevice>,
scoped_refptr<base::SingleThreadTaskRunner> task_runner)
: task_runner_(std::move(task_runner)) {}
WebEngineAudioOutputDevice::~WebEngineAudioOutputDevice() = default;
void WebEngineAudioOutputDevice::Initialize(
const media::AudioParameters& params,
RenderCallback* callback) {
DCHECK(callback);
{
base::AutoLock auto_lock(callback_lock_);
DCHECK(!callback_);
callback_ = callback;
}
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::InitializeOnAudioThread, this,
params));
}
void WebEngineAudioOutputDevice::Start() {
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::StartOnAudioThread, this));
}
void WebEngineAudioOutputDevice::Stop() {
{
base::AutoLock auto_lock(callback_lock_);
callback_ = nullptr;
}
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::StopOnAudioThread, this));
}
void WebEngineAudioOutputDevice::Pause() {
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::PauseOnAudioThread, this));
}
void WebEngineAudioOutputDevice::Play() {
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::PlayOnAudioThread, this));
}
void WebEngineAudioOutputDevice::Flush() {
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::FlushOnAudioThread, this));
}
bool WebEngineAudioOutputDevice::SetVolume(double volume) {
task_runner_->PostTask(
FROM_HERE,
base::BindOnce(&WebEngineAudioOutputDevice::SetVolumeOnAudioThread, this,
volume));
return true;
}
media::OutputDeviceInfo WebEngineAudioOutputDevice::GetOutputDeviceInfo() {
return media::OutputDeviceInfo(
std::string(), media::OUTPUT_DEVICE_STATUS_OK,
media::AudioParameters(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
media::ChannelLayoutConfig::Stereo(), 48000, 480));
}
void WebEngineAudioOutputDevice::GetOutputDeviceInfoAsync(
OutputDeviceInfoCB info_cb) {
std::move(info_cb).Run(GetOutputDeviceInfo());
}
bool WebEngineAudioOutputDevice::IsOptimizedForHardwareParameters() {
return false;
}
bool WebEngineAudioOutputDevice::CurrentThreadIsRenderingThread() {
return task_runner_->BelongsToCurrentThread();
}
void WebEngineAudioOutputDevice::BindAudioConsumerOnAudioThread(
fidl::InterfaceHandle<fuchsia::media::AudioConsumer>
audio_consumer_handle) {
DCHECK(CurrentThreadIsRenderingThread());
DCHECK(!audio_consumer_);
audio_consumer_.Bind(std::move(audio_consumer_handle));
audio_consumer_.set_error_handler([this](zx_status_t status) {
ZX_LOG(ERROR, status) << "AudioConsumer disconnected.";
ReportError();
});
}
void WebEngineAudioOutputDevice::InitializeOnAudioThread(
const media::AudioParameters& params) {
DCHECK(CurrentThreadIsRenderingThread());
params_ = params;
audio_bus_ = media::AudioBus::Create(params_);
UpdateVolume();
WatchAudioConsumerStatus();
}
void WebEngineAudioOutputDevice::StartOnAudioThread() {
DCHECK(CurrentThreadIsRenderingThread());
if (!audio_consumer_)
return;
CreateStreamSink();
media_pos_frames_ = 0;
audio_consumer_->Start(fuchsia::media::AudioConsumerStartFlags::LOW_LATENCY,
fuchsia::media::NO_TIMESTAMP, 0);
}
void WebEngineAudioOutputDevice::StopOnAudioThread() {
DCHECK(CurrentThreadIsRenderingThread());
if (!audio_consumer_)
return;
audio_consumer_->Stop();
pump_samples_timer_.Stop();
audio_consumer_.Unbind();
stream_sink_.Unbind();
volume_control_.Unbind();
}
void WebEngineAudioOutputDevice::PauseOnAudioThread() {
DCHECK(CurrentThreadIsRenderingThread());
if (!audio_consumer_)
return;
paused_ = true;
audio_consumer_->SetRate(0.0);
pump_samples_timer_.Stop();
}
void WebEngineAudioOutputDevice::PlayOnAudioThread() {
DCHECK(CurrentThreadIsRenderingThread());
if (!audio_consumer_)
return;
paused_ = false;
audio_consumer_->SetRate(1.0);
}
void WebEngineAudioOutputDevice::FlushOnAudioThread() {
DCHECK(CurrentThreadIsRenderingThread());
if (!stream_sink_)
return;
stream_sink_->DiscardAllPacketsNoReply();
}
void WebEngineAudioOutputDevice::SetVolumeOnAudioThread(double volume) {
DCHECK(CurrentThreadIsRenderingThread());
volume_ = volume;
if (audio_consumer_)
UpdateVolume();
}
void WebEngineAudioOutputDevice::CreateStreamSink() {
DCHECK(CurrentThreadIsRenderingThread());
DCHECK(audio_consumer_);
size_t buffer_size = params_.GetBytesPerBuffer(media::kSampleFormatF32);
stream_sink_buffers_.reserve(kNumBuffers);
available_buffers_indices_.clear();
std::vector<zx::vmo> vmos_for_stream_sink;
vmos_for_stream_sink.reserve(kNumBuffers);
for (size_t i = 0; i < kNumBuffers; ++i) {
auto region = base::WritableSharedMemoryRegion::Create(buffer_size);
auto mapping = region.Map();
if (!mapping.IsValid()) {
LOG(WARNING) << "Failed to allocate VMO of size " << buffer_size;
ReportError();
return;
}
stream_sink_buffers_.push_back(std::move(mapping));
available_buffers_indices_.push_back(i);
auto read_only_region =
base::WritableSharedMemoryRegion::ConvertToReadOnly(std::move(region));
vmos_for_stream_sink.push_back(
base::ReadOnlySharedMemoryRegion::TakeHandleForSerialization(
std::move(read_only_region))
.PassPlatformHandle());
}
fuchsia::media::AudioStreamType stream_type;
stream_type.channels = params_.channels();
stream_type.frames_per_second = params_.sample_rate();
stream_type.sample_format = fuchsia::media::AudioSampleFormat::FLOAT;
audio_consumer_->CreateStreamSink(std::move(vmos_for_stream_sink),
std::move(stream_type), nullptr,
stream_sink_.NewRequest());
stream_sink_.set_error_handler([this](zx_status_t status) {
ZX_LOG(ERROR, status) << "StreamSink disconnected.";
ReportError();
});
}
void WebEngineAudioOutputDevice::UpdateVolume() {
DCHECK(CurrentThreadIsRenderingThread());
DCHECK(audio_consumer_);
if (!volume_control_) {
audio_consumer_->BindVolumeControl(volume_control_.NewRequest());
volume_control_.set_error_handler([](zx_status_t status) {
ZX_LOG(ERROR, status) << "VolumeControl disconnected.";
});
}
volume_control_->SetVolume(volume_);
}
void WebEngineAudioOutputDevice::WatchAudioConsumerStatus() {
DCHECK(CurrentThreadIsRenderingThread());
audio_consumer_->WatchStatus(fit::bind_member(
this, &WebEngineAudioOutputDevice::OnAudioConsumerStatusChanged));
}
void WebEngineAudioOutputDevice::OnAudioConsumerStatusChanged(
fuchsia::media::AudioConsumerStatus status) {
DCHECK(CurrentThreadIsRenderingThread());
if (!status.has_min_lead_time()) {
DLOG(ERROR) << "AudioConsumerStatus.min_lead_time isn't set.";
ReportError();
return;
}
min_lead_time_ = base::Nanoseconds(status.min_lead_time());
if (status.has_presentation_timeline()) {
timeline_reference_time_ = base::TimeTicks::FromZxTime(
status.presentation_timeline().reference_time);
timeline_subject_time_ =
base::Nanoseconds(status.presentation_timeline().subject_time);
timeline_reference_delta_ = status.presentation_timeline().reference_delta;
timeline_subject_delta_ = status.presentation_timeline().subject_delta;
} else {
timeline_reference_time_ = base::TimeTicks();
}
pump_samples_timer_.Stop();
SchedulePumpSamples();
WatchAudioConsumerStatus();
}
void WebEngineAudioOutputDevice::SchedulePumpSamples() {
DCHECK(CurrentThreadIsRenderingThread());
if (paused_ || timeline_reference_time_.is_null() ||
pump_samples_timer_.IsRunning() || available_buffers_indices_.empty()) {
return;
}
auto media_pos = media::AudioTimestampHelper::FramesToTime(
media_pos_frames_, params_.sample_rate());
auto playback_time = timeline_reference_time_ +
(media_pos - timeline_subject_time_) *
timeline_reference_delta_ / timeline_subject_delta_;
base::TimeTicks now = base::TimeTicks::Now();
base::TimeTicks target_time = playback_time - min_lead_time_ - kLeadTimeExtra;
base::TimeDelta delay = target_time - now;
pump_samples_timer_.Start(
FROM_HERE, delay,
base::BindOnce(&WebEngineAudioOutputDevice::PumpSamples, this,
playback_time));
}
void WebEngineAudioOutputDevice::PumpSamples(base::TimeTicks playback_time) {
DCHECK(CurrentThreadIsRenderingThread());
auto now = base::TimeTicks::Now();
auto lead_time = playback_time - now;
if (lead_time < min_lead_time_) {
auto new_playback_time = now + min_lead_time_;
auto skipped_time = new_playback_time - playback_time;
media_pos_frames_ += media::AudioTimestampHelper::TimeToFrames(
skipped_time, params_.sample_rate());
playback_time += skipped_time;
}
int frames_filled;
{
base::AutoLock auto_lock(callback_lock_);
if (!callback_)
return;
frames_filled =
callback_->Render(playback_time - now, now, {}, audio_bus_.get());
}
if (frames_filled) {
DCHECK(!available_buffers_indices_.empty());
int buffer_index = available_buffers_indices_.back();
available_buffers_indices_.pop_back();
audio_bus_->ToInterleaved<media::Float32SampleTypeTraitsNoClip>(
frames_filled,
static_cast<float*>(stream_sink_buffers_[buffer_index].memory()));
fuchsia::media::StreamPacket packet;
packet.payload_buffer_id = buffer_index;
packet.pts = media::AudioTimestampHelper::FramesToTime(
media_pos_frames_, params_.sample_rate())
.InNanoseconds();
packet.payload_offset = 0;
packet.payload_size = frames_filled * sizeof(float) * params_.channels();
stream_sink_->SendPacket(std::move(packet), [this, buffer_index]() {
OnStreamSendDone(buffer_index);
});
media_pos_frames_ += frames_filled;
}
SchedulePumpSamples();
}
void WebEngineAudioOutputDevice::OnStreamSendDone(size_t buffer_index) {
DCHECK(CurrentThreadIsRenderingThread());
available_buffers_indices_.push_back(buffer_index);
SchedulePumpSamples();
}
void WebEngineAudioOutputDevice::ReportError() {
DCHECK(CurrentThreadIsRenderingThread());
audio_consumer_.Unbind();
stream_sink_.Unbind();
volume_control_.Unbind();
pump_samples_timer_.Stop();
{
base::AutoLock auto_lock(callback_lock_);
if (callback_)
callback_->OnRenderError();
}
}