#include "media/audio/win/audio_low_latency_input_win.h"
#include <objbase.h>
#include <mmdeviceapi.h>
#include <audioclient.h>
#include <audioclientactivationparams.h>
#include <combaseapi.h>
#include <ksmedia.h>
#include <propkey.h>
#include <stddef.h>
#include <algorithm>
#include <cmath>
#include <memory>
#include <utility>
#include "base/check_deref.h"
#include "base/check_op.h"
#include "base/compiler_specific.h"
#include "base/containers/span.h"
#include "base/feature_list.h"
#include "base/functional/callback.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram_functions.h"
#include "base/metrics/histogram_macros.h"
#include "base/numerics/checked_math.h"
#include "base/process/process.h"
#include "base/strings/strcat.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "base/synchronization/waitable_event.h"
#include "base/timer/elapsed_timer.h"
#include "base/trace_event/common/trace_event_common.h"
#include "base/trace_event/trace_event.h"
#include "base/win/core_winrt_util.h"
#include "base/win/scoped_propvariant.h"
#include "base/win/scoped_variant.h"
#include "base/win/vector.h"
#include "base/win/windows_version.h"
#include "media/audio/application_loopback_device_helper.h"
#include "media/audio/audio_device_description.h"
#include "media/audio/audio_device_name.h"
#include "media/audio/audio_features.h"
#include "media/audio/win/avrt_wrapper_win.h"
#include "media/audio/win/core_audio_util_win.h"
#include "media/base/audio_block_fifo.h"
#include "media/base/audio_bus.h"
#include "media/base/audio_timestamp_helper.h"
#include "media/base/channel_layout.h"
#include "media/base/limits.h"
#include "media/base/media_switches.h"
#include "media/base/sample_format.h"
#include "media/base/timestamp_constants.h"
using base::win::ScopedCoMem;
using base::win::ScopedCOMInitializer;
using Microsoft::WRL::ComPtr;
namespace media {
namespace {
constexpr uint32_t KSAUDIO_SPEAKER_UNSUPPORTED = 0;
BASE_FEATURE(kWasapiInputUseDeviceSampleFormat,
base::FEATURE_DISABLED_BY_DEFAULT);
constexpr base::TimeDelta kMaxAbsTimeDiffBeforeSwithingToFakeTimestamps =
base::Milliseconds(500);
constexpr uint32_t kWindowsSystemProcessId = 4;
constexpr HRESULT kActivationTimeoutHr = HRESULT_FROM_WIN32(WAIT_TIMEOUT);
std::string ErrorToString(HRESULT hresult) {
return CoreAudioUtil::ErrorToString(hresult);
}
enum FormatRelatedInitError {
kUnsupportedFormat = 0,
kUnsupportedFormatWithFormatConversion = 1,
kInvalidArgument = 2,
kInvalidArgumentWithFormatConversion = 3,
kCount
};
bool IsSupportedFormatForConversion(WAVEFORMATEXTENSIBLE* format_ex) {
WAVEFORMATEX* format = &format_ex->Format;
if (format->nSamplesPerSec < limits::kMinSampleRate ||
format->nSamplesPerSec > limits::kMaxSampleRate) {
return false;
}
switch (format->wBitsPerSample) {
case 8:
case 16:
case 32:
break;
default:
return false;
}
if (GuessChannelLayout(format->nChannels) == CHANNEL_LAYOUT_UNSUPPORTED) {
LOG(ERROR) << "Hardware configuration not supported for audio conversion";
return false;
}
return true;
}
ChannelConfig ChannelLayoutToChannelConfig(ChannelLayout layout) {
switch (layout) {
case CHANNEL_LAYOUT_DISCRETE:
return KSAUDIO_SPEAKER_DIRECTOUT;
case CHANNEL_LAYOUT_MONO:
return KSAUDIO_SPEAKER_MONO;
case CHANNEL_LAYOUT_STEREO:
return KSAUDIO_SPEAKER_STEREO;
default:
LOG(WARNING) << "Unsupported channel layout: " << layout;
return KSAUDIO_SPEAKER_UNSUPPORTED;
}
}
const char* StreamOpenResultToString(
WASAPIAudioInputStream::StreamOpenResult result) {
switch (result) {
case WASAPIAudioInputStream::OPEN_RESULT_OK:
return "OK";
case WASAPIAudioInputStream::OPEN_RESULT_CREATE_INSTANCE:
return "CREATE_INSTANCE";
case WASAPIAudioInputStream::OPEN_RESULT_NO_ENDPOINT:
return "NO_ENDPOINT";
case WASAPIAudioInputStream::OPEN_RESULT_NO_STATE:
return "NO_STATE";
case WASAPIAudioInputStream::OPEN_RESULT_DEVICE_NOT_ACTIVE:
return "DEVICE_NOT_ACTIVE";
case WASAPIAudioInputStream::OPEN_RESULT_ACTIVATION_FAILED:
return "ACTIVATION_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_FORMAT_NOT_SUPPORTED:
return "FORMAT_NOT_SUPPORTED";
case WASAPIAudioInputStream::OPEN_RESULT_AUDIO_CLIENT_INIT_FAILED:
return "AUDIO_CLIENT_INIT_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_GET_BUFFER_SIZE_FAILED:
return "GET_BUFFER_SIZE_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_LOOPBACK_ACTIVATE_FAILED:
return "LOOPBACK_ACTIVATE_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_LOOPBACK_INIT_FAILED:
return "LOOPBACK_INIT_FAILED";
case WASAPIAudioInputStream::OPEN_RESULT_SET_EVENT_HANDLE:
return "SET_EVENT_HANDLE";
case WASAPIAudioInputStream::OPEN_RESULT_NO_CAPTURE_CLIENT:
return "NO_CAPTURE_CLIENT";
case WASAPIAudioInputStream::OPEN_RESULT_NO_AUDIO_VOLUME:
return "NO_AUDIO_VOLUME";
case WASAPIAudioInputStream::OPEN_RESULT_OK_WITH_RESAMPLING:
return "OK_WITH_RESAMPLING";
}
return "UNKNOWN";
}
bool VariantBoolToBool(VARIANT_BOOL var_bool) {
switch (var_bool) {
case VARIANT_TRUE:
return true;
case VARIANT_FALSE:
return false;
}
LOG(ERROR) << "Invalid VARIANT_BOOL type";
return false;
}
std::string GetOpenLogString(WASAPIAudioInputStream::StreamOpenResult result,
HRESULT hr,
WAVEFORMATEXTENSIBLE input_format,
WAVEFORMATEX output_format) {
return base::StringPrintf(
"WAIS::Open => (ERROR: result=%s, hresult=%#lx, input_format=[%s], "
"output_format=[%s])",
StreamOpenResultToString(result), hr,
CoreAudioUtil::WaveFormatToString(&input_format).c_str(),
CoreAudioUtil::WaveFormatToString(&output_format).c_str());
}
void LogFakeAudioCaptureTimestamps(bool use_fake_audio_capture_timestamps,
base::TimeDelta abs_delta_time) {
TRACE_EVENT_INSTANT2(
"audio", "AudioCaptureWinTimestamps", TRACE_EVENT_SCOPE_THREAD,
"use_fake_audio_capture_timestamps", use_fake_audio_capture_timestamps,
"abs_timestamp_diff_ms", abs_delta_time.InMilliseconds());
base::UmaHistogramBoolean("Media.Audio.Capture.Win.FakeTimestamps",
use_fake_audio_capture_timestamps);
base::UmaHistogramLongTimes("Media.Audio.Capture.Win.AbsTimestampDiffMs",
abs_delta_time);
}
WASAPIAudioInputStream::ActivateAudioInterfaceAsyncCallback&
GetActivateAudioInterfaceAsyncCallback() {
static base::NoDestructor<
WASAPIAudioInputStream::ActivateAudioInterfaceAsyncCallback>
activate_audio_interface_async_callback{
base::BindRepeating(&ActivateAudioInterfaceAsync)};
return *activate_audio_interface_async_callback;
}
bool IsProcessLoopbackDevice(std::string_view device_id) {
return device_id == AudioDeviceDescription::kLoopbackWithoutChromeId ||
device_id == AudioDeviceDescription::kLoopbackAllDevicesId ||
AudioDeviceDescription::IsApplicationLoopbackDevice(device_id);
}
uint32_t GetCurrentProcessId() {
return static_cast<uint32_t>(base::Process::Current().Pid());
}
uint32_t GetTargetProcessId(std::string_view device_id) {
if (AudioDeviceDescription::IsApplicationLoopbackDevice(device_id)) {
return GetApplicationIdFromApplicationLoopbackDeviceId(device_id);
}
if (device_id == AudioDeviceDescription::kLoopbackWithoutChromeId) {
return GetCurrentProcessId();
}
CHECK(AudioDeviceDescription::IsLoopbackDevice(device_id));
return kWindowsSystemProcessId;
}
PROCESS_LOOPBACK_MODE GetProcessLoopbackMode(std::string_view device_id) {
if (AudioDeviceDescription::IsApplicationLoopbackDevice(device_id)) {
return PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE;
}
CHECK(AudioDeviceDescription::IsLoopbackDevice(device_id));
return PROCESS_LOOPBACK_MODE_EXCLUDE_TARGET_PROCESS_TREE;
}
bool IsEndpointLoopbackCapture(std::string_view device_id,
bool is_process_loopback) {
return AudioDeviceDescription::IsLoopbackDevice(device_id) &&
!is_process_loopback;
}
SampleFormat GetSampleFormatFromWaveFormat(
const WAVEFORMATEXTENSIBLE& wave_format) {
switch (wave_format.Format.wBitsPerSample) {
case 8:
return kSampleFormatU8;
case 16:
return kSampleFormatS16;
case 24:
return kSampleFormatS24;
case 32:
if (IsEqualGUID(wave_format.SubFormat, KSDATAFORMAT_SUBTYPE_PCM)) {
return kSampleFormatS32;
} else if (IsEqualGUID(wave_format.SubFormat,
KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)) {
return kSampleFormatF32;
}
}
return kUnknownSampleFormat;
}
}
class WASAPIAudioInputStream::DataDiscontinuityReporter {
public:
constexpr static int kCallbacksPerLogPeriod = 1000;
DataDiscontinuityReporter() {}
int GetLongTermDiscontinuityCountAndReset() {
int long_term_count = data_discontinuity_long_term_count_;
callback_count_ = 0;
data_discontinuity_short_term_count_ = 0;
data_discontinuity_long_term_count_ = 0;
return long_term_count;
}
void Log(bool observed_data_discontinuity) {
++callback_count_;
if (observed_data_discontinuity) {
++data_discontinuity_short_term_count_;
++data_discontinuity_long_term_count_;
}
if (callback_count_ % kCallbacksPerLogPeriod)
return;
base::UmaHistogramCounts1000("Media.Audio.Capture.Win.Glitches2",
data_discontinuity_short_term_count_);
data_discontinuity_short_term_count_ = 0;
}
private:
int callback_count_ = 0;
int data_discontinuity_short_term_count_ = 0;
int data_discontinuity_long_term_count_ = 0;
};
class WASAPIAudioInputStream::EchoCancellationConfig {
public:
using LogCallback = base::RepeatingCallback<void(std::string)>;
static std::unique_ptr<EchoCancellationConfig> Create(
const AudioParameters& params,
const std::string& device_id,
LogCallback log_callback) {
if (!(params.effects() & AudioParameters::ECHO_CANCELLER)) {
return nullptr;
}
return base::WrapUnique(
new EchoCancellationConfig(params, device_id, std::move(log_callback)));
}
static std::string GetSupportedEffectsString(int mask) {
std::vector<std::string> effects;
if (mask & AudioParameters::ECHO_CANCELLER) {
effects.push_back("ECHO_CANCELLER");
}
if (mask & AudioParameters::NOISE_SUPPRESSION) {
effects.push_back("NOISE_SUPPRESSION");
}
if (mask & AudioParameters::AUTOMATIC_GAIN_CONTROL) {
effects.push_back("AUTOMATIC_GAIN_CONTROL");
}
std::string result;
base::StringAppendF(&result, "%s => #effects=%zu (", __func__,
effects.size());
for (size_t i = 0; i < effects.size(); ++i) {
if (i > 0) {
result += " | ";
}
result += effects[i];
}
result += ")";
return result;
}
void LogMessage(std::string message) {
if (log_callback_.is_null()) {
return;
}
message.insert(0, "AEC::");
log_callback_.Run(std::move(message));
}
PRINTF_FORMAT(2, 3) void LogMessage(const char* format, ...) {
if (log_callback_.is_null()) {
return;
}
va_list args;
va_start(args, format);
std::string msg(base::StrCat({"AEC::", base::StringPrintV(format, args)}));
va_end(args);
log_callback_.Run(std::move(msg));
}
bool SetAudioClientAndLogEffects(ComPtr<IAudioClient> audio_client) {
CHECK(!AudioDeviceDescription::IsLoopbackDevice(device_id_));
CHECK(CoreAudioUtil::IsClientInitialized(audio_client.Get()));
audio_client_ = audio_client;
CHECK(audio_client_);
auto [effects, echo_cancellation_is_available] =
CoreAudioUtil::GetVoiceProcessingEffectsAndCheckForAEC(
audio_client_.Get());
if (effects != params_.effects()) {
LogMessage(
"%s => (WARNING: supported effects do not match requested effects)",
__func__);
}
if (echo_cancellation_is_available) {
UpdateEchoCancellationRenderEndpoint();
} else {
LogMessage("%s => (ERROR: system AEC is not supported)", __func__);
}
LogMessage(GetSupportedEffectsString(effects));
return echo_cancellation_is_available;
}
void UpdateEchoCancellationRenderEndpoint() {
CHECK(audio_client_);
ERole role = eConsole;
if (AudioDeviceDescription::IsCommunicationsDevice(
output_device_id_for_aec_)) {
role = eCommunications;
}
ComPtr<IMMDevice> audio_device =
CoreAudioUtil::CreateDevice(output_device_id_for_aec_, eRender, role);
if (!audio_device.Get()) {
LogMessage("%s => (ERROR: CoreAudioUtil::CreateDevice failed)", __func__);
return;
}
AudioDeviceName device_name;
CoreAudioUtil::GetDeviceName(audio_device.Get(), &device_name);
LogMessage("%s => (AEC output device=[name: %s, id: %s])", __func__,
device_name.device_name.c_str(), device_name.unique_id.c_str());
ComPtr<IAcousticEchoCancellationControl> aec_control;
HRESULT hr = audio_client_->GetService(IID_PPV_ARGS(&aec_control));
if (FAILED(hr)) {
LogMessage("%s => (ERROR: IAudioClient::GetService=[%s])", __func__,
ErrorToString(hr).c_str());
return;
}
std::wstring endpoint_id_wide = base::UTF8ToWide(device_name.unique_id);
LPCWSTR endpoint_id = endpoint_id_wide.c_str();
hr = aec_control->SetEchoCancellationRenderEndpoint(endpoint_id);
if (FAILED(hr)) {
LogMessage(
"%s => (ERROR: "
"IAcousticEchoCancellationControl::SetEchoCancellationRenderEndpoint="
"[%s])",
__func__, ErrorToString(hr).c_str());
}
}
void SetOutputDeviceForAec(const std::string& output_device_id) {
std::string new_output_device_id =
output_device_id.empty() ? AudioDeviceDescription::kDefaultDeviceId
: output_device_id;
if (new_output_device_id == output_device_id_for_aec_) {
return;
}
output_device_id_for_aec_ = new_output_device_id;
if (!audio_client_) {
return;
}
UpdateEchoCancellationRenderEndpoint();
}
private:
explicit EchoCancellationConfig(const AudioParameters& params,
const std::string& device_id,
const LogCallback log_callback)
: params_(params),
device_id_(device_id),
log_callback_(std::move(log_callback)) {}
const AudioParameters params_;
const std::string device_id_;
const LogCallback log_callback_;
ComPtr<IAudioClient> audio_client_;
std::string output_device_id_for_aec_ =
AudioDeviceDescription::kDefaultDeviceId;
};
class WASAPIAudioInputStream::AudioClientActivationHandler
: public Microsoft::WRL::RuntimeClass<
Microsoft::WRL::RuntimeClassFlags<Microsoft::WRL::ClassicCom>,
Microsoft::WRL::FtmBase,
IActivateAudioInterfaceCompletionHandler> {
public:
friend class FakeWinWASAPIEnvironment;
AudioClientActivationHandler() = default;
~AudioClientActivationHandler() override = default;
HRESULT WaitAndGetAudioClient(ComPtr<IAudioClient>* audio_client,
base::TimeDelta async_activation_timeout_ms) {
if (!wait_event_.TimedWait(async_activation_timeout_ms)) {
return kActivationTimeoutHr;
}
if (SUCCEEDED(activation_result_)) {
*audio_client = std::move(audio_client_);
}
return activation_result_;
}
private:
IFACEMETHODIMP ActivateCompleted(
IActivateAudioInterfaceAsyncOperation* activate_operation) override {
HRESULT hr_activate = S_OK;
ComPtr<IAudioClient> audio_client = nullptr;
activation_result_ =
activate_operation->GetActivateResult(&hr_activate, &audio_client);
if (FAILED(activation_result_)) {
return activation_result_;
}
activation_result_ = hr_activate;
if (SUCCEEDED(activation_result_)) {
audio_client_ = std::move(audio_client);
}
wait_event_.Signal();
return activation_result_;
}
ComPtr<IAudioClient> audio_client_ = nullptr;
HRESULT activation_result_ = E_FAIL;
base::WaitableEvent wait_event_{
base::WaitableEvent::ResetPolicy::AUTOMATIC,
base::WaitableEvent::InitialState::NOT_SIGNALED};
};
WASAPIAudioInputStream::WASAPIAudioInputStream(
AudioManagerWin* manager,
const AudioParameters& params,
const std::string& device_id,
AudioManager::LogCallback log_callback)
: manager_(manager),
params_(params),
peak_detector_(base::BindRepeating(&AudioManager::TraceAmplitudePeak,
base::Unretained(manager_),
true)),
data_discontinuity_reporter_(
std::make_unique<DataDiscontinuityReporter>()),
device_id_(device_id),
log_callback_(std::move(log_callback)),
aec_config_(EchoCancellationConfig::Create(
params,
device_id,
base::BindRepeating(
static_cast<void (WASAPIAudioInputStream::*)(std::string)>(
&WASAPIAudioInputStream::SendLogMessage),
base::Unretained(this)))),
is_loopback_capture_(AudioDeviceDescription::IsLoopbackDevice(device_id)),
is_process_loopback_capture_(IsProcessLoopbackDevice(device_id)),
glitch_reporter_(is_loopback_capture_
? SystemGlitchReporter::StreamType::kLoopback
: SystemGlitchReporter::StreamType::kCapture) {
DCHECK(manager_);
DCHECK(!device_id_.empty());
DCHECK(!log_callback_.is_null());
DCHECK_LE(params.channels(), 2);
DCHECK(params.channel_layout() == CHANNEL_LAYOUT_MONO ||
params.channel_layout() == CHANNEL_LAYOUT_STEREO ||
params.channel_layout() == CHANNEL_LAYOUT_DISCRETE);
SendLogMessage("%s({device_id=%s}, {params=[%s]})", __func__,
device_id.c_str(), params.AsHumanReadableString().c_str());
if (AudioDeviceDescription::IsLoopbackDevice(device_id_)) {
SendLogMessage("%s => (audio loopback device is of type: %s)", __func__,
is_process_loopback_capture_ ? "PROCESS" : "ENDPOINT");
}
SendLogMessage("%s => (AEC is requested=[%s])", __func__,
aec_config_ ? "true" : "false");
bool avrt_init = avrt::Initialize();
if (!avrt_init)
SendLogMessage("%s => (WARNING: failed to load Avrt.dll)", __func__);
audio_samples_ready_event_.Set(CreateEvent(NULL, FALSE, FALSE, NULL));
DCHECK(audio_samples_ready_event_.is_valid());
stop_capture_event_.Set(CreateEvent(NULL, FALSE, FALSE, NULL));
DCHECK(stop_capture_event_.is_valid());
use_device_sample_format_ =
base::FeatureList::IsEnabled(kWasapiInputUseDeviceSampleFormat);
}
WASAPIAudioInputStream::~WASAPIAudioInputStream() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
}
bool WASAPIAudioInputStream::UpdateFormats() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
sample_format_ = kSampleFormatS16;
input_format_.SubFormat = KSDATAFORMAT_SUBTYPE_PCM;
output_format_.wFormatTag = WAVE_FORMAT_PCM;
if (use_device_sample_format_) {
WAVEFORMATEXTENSIBLE mix_format;
HRESULT hr =
CoreAudioUtil::GetSharedModeMixFormat(audio_client_.Get(), &mix_format);
if (FAILED(hr)) {
ReportOpenResult(hr);
return false;
}
auto mix_sample_format = GetSampleFormatFromWaveFormat(mix_format);
base::UmaHistogramEnumeration("Media.Audio.Capture.Win.AudioEngineFormat",
mix_sample_format);
if (mix_sample_format != kUnknownSampleFormat) {
sample_format_ = mix_sample_format;
CHECK_NE(sample_format_, kSampleFormatS24, base::NotFatalUntil::M148);
if (sample_format_ == kSampleFormatS24) {
sample_format_ = kSampleFormatS32;
}
input_format_.SubFormat = mix_format.SubFormat;
if (IsEqualGUID(mix_format.SubFormat, KSDATAFORMAT_SUBTYPE_PCM)) {
CHECK_NE(sample_format_, kSampleFormatF32);
output_format_.wFormatTag = WAVE_FORMAT_PCM;
} else if (IsEqualGUID(mix_format.SubFormat,
KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)) {
CHECK_EQ(sample_format_, kSampleFormatF32);
output_format_.wFormatTag = WAVE_FORMAT_IEEE_FLOAT;
} else {
return false;
}
} else {
const uint32_t format_tag =
EXTRACT_WAVEFORMATEX_ID(&mix_format.SubFormat);
base::UmaHistogramSparse(
"Media.Audio.Capture.Win.AudioEngineFormat.Unknown", format_tag);
use_device_sample_format_ = false;
}
}
WAVEFORMATEX* format = &input_format_.Format;
format->wFormatTag = WAVE_FORMAT_EXTENSIBLE;
format->nChannels = params_.channels();
format->nSamplesPerSec = params_.sample_rate();
format->wBitsPerSample = SampleFormatToBitsPerChannel(sample_format_);
format->nBlockAlign = (format->wBitsPerSample / 8) * format->nChannels;
format->nAvgBytesPerSec = format->nSamplesPerSec * format->nBlockAlign;
format->cbSize = sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX);
input_format_.Samples.wValidBitsPerSample = format->wBitsPerSample;
input_format_.dwChannelMask =
ChannelLayoutToChannelConfig(params_.channel_layout());
SendLogMessage("%s => (audio engine format=[%s])", __func__,
CoreAudioUtil::WaveFormatToString(&input_format_).c_str());
output_format_.nChannels = format->nChannels;
output_format_.nSamplesPerSec = format->nSamplesPerSec;
output_format_.wBitsPerSample = format->wBitsPerSample;
output_format_.nBlockAlign = format->nBlockAlign;
output_format_.nAvgBytesPerSec = format->nAvgBytesPerSec;
output_format_.cbSize = 0;
SendLogMessage("%s => (audio sink format=[%s])", __func__,
CoreAudioUtil::WaveFormatToString(&output_format_).c_str());
frame_size_bytes_ = format->nBlockAlign;
packet_size_bytes_ = params_.GetBytesPerBuffer(sample_format_);
packet_size_frames_ = packet_size_bytes_ / format->nBlockAlign;
SendLogMessage(
"%s => (packet size=[%zu bytes/%zu audio frames/%.3f milliseconds])",
__func__, packet_size_bytes_, packet_size_frames_,
params_.GetBufferDuration().InMillisecondsF());
return true;
}
AudioInputStream::OpenOutcome WASAPIAudioInputStream::Open() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
SendLogMessage("%s([opened=%s])", __func__, opened_ ? "true" : "false");
if (opened_) {
return OpenOutcome::kAlreadyOpen;
}
HRESULT hr = S_OK;
if (!is_process_loopback_capture_) {
hr = SetCaptureDevice();
if (FAILED(hr)) {
ReportOpenResult(hr);
return OpenOutcome::kFailed;
}
}
hr = ActivateAudioClientInterface();
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_ACTIVATION_FAILED;
ReportOpenResult(hr);
return OpenOutcome::kFailed;
}
if (!UpdateFormats()) {
return OpenOutcome::kFailed;
}
if (!is_process_loopback_capture_) {
raw_processing_supported_ = RawProcessingSupported();
}
WORD audio_engine_channels = 0;
if (!AudioDeviceDescription::IsLoopbackDevice(device_id_)) {
hr = GetAudioEngineNumChannels(&audio_engine_channels);
}
if (raw_processing_supported_ &&
!AudioDeviceDescription::IsLoopbackDevice(device_id_) && SUCCEEDED(hr)) {
SetCommunicationsCategoryAndMaybeRawCaptureMode(audio_engine_channels);
}
hr = S_OK;
if (!DesiredFormatIsSupported(&hr)) {
open_result_ = OPEN_RESULT_FORMAT_NOT_SUPPORTED;
ReportOpenResult(hr);
return OpenOutcome::kFailed;
}
hr = InitializeAudioEngine();
if (SUCCEEDED(hr) && converter_)
open_result_ = OPEN_RESULT_OK_WITH_RESAMPLING;
ReportOpenResult(hr);
opened_ = SUCCEEDED(hr);
if (aec_config_) {
if (!aec_config_->SetAudioClientAndLogEffects(audio_client_)) {
aec_config_.reset();
}
}
if (opened_) {
return OpenOutcome::kSuccess;
}
switch (hr) {
case E_ACCESSDENIED:
return OpenOutcome::kFailedSystemPermissions;
case AUDCLNT_E_DEVICE_IN_USE:
return OpenOutcome::kFailedInUse;
default:
return OpenOutcome::kFailed;
}
}
void WASAPIAudioInputStream::Start(AudioInputCallback* callback) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(callback);
SendLogMessage("%s([opened=%s, started=%s])", __func__,
opened_ ? "true" : "false", started_ ? "true" : "false");
if (!opened_)
return;
if (started_)
return;
if (device_id_ == AudioDeviceDescription::kLoopbackWithMuteDeviceId &&
system_audio_volume_) {
BOOL muted = false;
system_audio_volume_->GetMute(&muted);
if (!muted) {
system_audio_volume_->SetMute(true, nullptr);
mute_done_ = true;
}
}
DCHECK(!sink_);
sink_ = callback;
StartAgc();
DCHECK(!capture_thread_.get());
capture_thread_ = std::make_unique<base::DelegateSimpleThread>(
this, "wasapi_capture_thread",
base::SimpleThread::Options(base::ThreadType::kRealtimeAudio));
capture_thread_->Start();
HRESULT hr = audio_client_->Start();
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient::Start=[%s])", __func__,
ErrorToString(hr).c_str());
}
if (SUCCEEDED(hr) && audio_render_client_for_loopback_.Get()) {
hr = audio_render_client_for_loopback_->Start();
if (FAILED(hr)) {
SendLogMessage(
"%s => (ERROR: IAudioClient::Start=[%s] (endpoint loopback))",
__func__, ErrorToString(hr).c_str());
}
}
started_ = SUCCEEDED(hr);
}
void WASAPIAudioInputStream::Stop() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
SendLogMessage("%s([started=%s])", __func__, started_ ? "true" : "false");
if (!started_)
return;
if (device_id_ == AudioDeviceDescription::kLoopbackWithMuteDeviceId &&
mute_done_) {
DCHECK(system_audio_volume_);
if (system_audio_volume_) {
system_audio_volume_->SetMute(false, nullptr);
mute_done_ = false;
}
}
StopAgc();
if (stop_capture_event_.is_valid()) {
SetEvent(stop_capture_event_.Get());
}
HRESULT hr = audio_client_->Stop();
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient::Stop=[%s])", __func__,
ErrorToString(hr).c_str());
}
if (capture_thread_) {
SetEvent(stop_capture_event_.Get());
capture_thread_->Join();
capture_thread_.reset();
}
SendLogMessage(
"%s => (timestamp(n)-timestamp(n-1)=[min: %.3f msec, max: %.3f msec])",
__func__, min_timestamp_diff_.InMillisecondsF(),
max_timestamp_diff_.InMillisecondsF());
started_ = false;
sink_ = nullptr;
}
void WASAPIAudioInputStream::Close() {
SendLogMessage("%s()", __func__);
Stop();
if (GetAutomaticGainControl()) {
base::UmaHistogramBoolean("Media.Audio.RawProcessingSupportedWin",
raw_processing_supported_);
}
if (converter_)
converter_->RemoveInput(this);
ReportAndResetGlitchStats();
manager_->ReleaseInputStream(this);
}
double WASAPIAudioInputStream::GetMaxVolume() {
DLOG_IF(ERROR, !opened_) << "Open() has not been called successfully";
if (!opened_)
return 0.0;
return 1.0;
}
void WASAPIAudioInputStream::SetVolume(double volume) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK_GE(volume, 0.0);
DCHECK_LE(volume, 1.0);
SendLogMessage("%s({volume=%.2f} [opened=%s])", __func__, volume,
opened_ ? "true" : "false");
if (!opened_ || !simple_audio_volume_) {
return;
}
HRESULT hr = simple_audio_volume_->SetMasterVolume(static_cast<float>(volume),
nullptr);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: ISimpleAudioVolume::SetMasterVolume=[%s])",
__func__, ErrorToString(hr).c_str());
}
UpdateAgcVolume();
}
double WASAPIAudioInputStream::GetVolume() {
DCHECK(opened_) << "Open() has not been called successfully";
if (!simple_audio_volume_) {
return 0.0;
}
float level = 0.0f;
HRESULT hr = simple_audio_volume_->GetMasterVolume(&level);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: ISimpleAudioVolume::GetMasterVolume=[%s])",
__func__, ErrorToString(hr).c_str());
}
return static_cast<double>(level);
}
bool WASAPIAudioInputStream::IsMuted() {
DCHECK(opened_) << "Open() has not been called successfully";
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!simple_audio_volume_) {
return false;
}
BOOL is_muted = FALSE;
HRESULT hr = simple_audio_volume_->GetMute(&is_muted);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: ISimpleAudioVolume::GetMute=[%s])", __func__,
ErrorToString(hr).c_str());
}
return is_muted != FALSE;
}
void WASAPIAudioInputStream::SetOutputDeviceForAec(
const std::string& output_device_id) {
SendLogMessage("%s({output_device_id=%s})", __func__,
output_device_id.c_str());
if (aec_config_) {
aec_config_->SetOutputDeviceForAec(output_device_id);
}
}
void WASAPIAudioInputStream::SendLogMessage(std::string message) {
if (log_callback_.is_null()) {
return;
}
message.insert(0, "WAIS::");
log_callback_.Run(std::move(message));
}
void WASAPIAudioInputStream::SendLogMessage(const char* format, ...) {
if (log_callback_.is_null())
return;
va_list args;
va_start(args, format);
std::string msg(base::StrCat({"WAIS::", base::StringPrintV(format, args)}));
va_end(args);
log_callback_.Run(std::move(msg));
}
void WASAPIAudioInputStream::
OverrideActivateAudioInterfaceAsyncCallbackForTesting(
ActivateAudioInterfaceAsyncCallback callback) {
GetActivateAudioInterfaceAsyncCallback() = callback;
}
void WASAPIAudioInputStream::SimulateErrorForTesting() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
CHECK(capture_thread_);
simulate_error_for_testing_ = true;
}
HRESULT WASAPIAudioInputStream::CreateFifoIfNeeded() {
if (fifo_) {
return S_OK;
}
uint32_t endpoint_buffer_size_frames = 0;
HRESULT hr = audio_client_->GetBufferSize(&endpoint_buffer_size_frames);
if (FAILED(hr)) {
return hr;
}
size_t capture_buffer_size =
std::max(2 * endpoint_buffer_size_frames * frame_size_bytes_,
2 * packet_size_frames_ * frame_size_bytes_);
int buffers_required = capture_buffer_size / packet_size_bytes_;
if (converter_ && imperfect_buffer_size_conversion_)
++buffers_required;
DCHECK(!fifo_);
fifo_ = std::make_unique<AudioBlockFifo>(
input_format_.Format.nChannels, packet_size_frames_, buffers_required);
DVLOG(1) << "AudioBlockFifo buffer count: " << buffers_required;
return S_OK;
}
void WASAPIAudioInputStream::Run() {
ScopedCOMInitializer com_init(ScopedCOMInitializer::kMTA);
DWORD task_index = 0;
HANDLE mm_task =
avrt::AvSetMmThreadCharacteristics(L"Pro Audio", &task_index);
bool mmcss_is_ok =
(mm_task && avrt::AvSetMmThreadPriority(mm_task, AVRT_PRIORITY_CRITICAL));
if (!mmcss_is_ok) {
DWORD err = GetLastError();
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: Failed to enable MMCSS (error code=" << err
<< "))";
}
bool recording = true;
bool error = false;
HANDLE wait_array[2] = {stop_capture_event_.Get(),
audio_samples_ready_event_.Get()};
record_start_time_ = base::TimeTicks::Now();
last_capture_time_ = base::TimeTicks();
max_timestamp_diff_ = base::TimeDelta::Min();
min_timestamp_diff_ = base::TimeDelta::Max();
while (recording && !error) {
DWORD wait_result = WaitForMultipleObjects(2, wait_array, FALSE, INFINITE);
if (simulate_error_for_testing_) {
wait_result = WAIT_FAILED;
simulate_error_for_testing_ = false;
}
switch (wait_result) {
case WAIT_OBJECT_0 + 0:
recording = false;
break;
case WAIT_OBJECT_0 + 1:
CreateFifoIfNeeded();
if (!fifo_) {
error = true;
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: failed to create FIFO)";
break;
}
PullCaptureDataAndPushToSink();
break;
case WAIT_FAILED:
default:
error = true;
break;
}
}
if (recording && error) {
auto saved_last_error = GetLastError();
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: capturing failed with error code: "
<< saved_last_error << ")";
audio_client_->Stop();
sink_->OnError();
}
if (mm_task && !avrt::AvRevertMmThreadCharacteristics(mm_task)) {
PLOG(WARNING) << "Failed to disable MMCSS";
}
fifo_.reset();
}
void WASAPIAudioInputStream::PullCaptureDataAndPushToSink() {
TRACE_EVENT1("audio", "WASAPIAudioInputStream::PullCaptureDataAndPushToSink",
"sample rate", input_format_.Format.nSamplesPerSec);
UINT64 last_device_position = 0;
UINT32 num_frames_in_next_packet = 0;
HRESULT hr =
audio_capture_client_->GetNextPacketSize(&num_frames_in_next_packet);
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: 1-IAudioCaptureClient::GetNextPacketSize=["
<< ErrorToString(hr).c_str() << "])";
return;
}
while (num_frames_in_next_packet > 0) {
BYTE* data_ptr = nullptr;
UINT32 num_frames_to_read = 0;
DWORD flags = 0;
UINT64 device_position = 0;
UINT64 capture_time_100ns = 0;
hr =
audio_capture_client_->GetBuffer(&data_ptr, &num_frames_to_read, &flags,
&device_position, &capture_time_100ns);
if (hr == AUDCLNT_S_BUFFER_EMPTY) {
DCHECK_EQ(num_frames_to_read, 0u);
return;
}
if (hr == AUDCLNT_E_OUT_OF_ORDER) {
audio_capture_client_->ReleaseBuffer(num_frames_to_read);
}
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: IAudioCaptureClient::GetBuffer=["
<< ErrorToString(hr).c_str() << "])";
return;
}
if (!use_fake_audio_capture_timestamps_.has_value()) {
base::TimeDelta delta_time =
base::TimeTicks::Now() -
base::TimeTicks::FromQPCValue(capture_time_100ns);
if (delta_time.magnitude() >
kMaxAbsTimeDiffBeforeSwithingToFakeTimestamps) {
use_fake_audio_capture_timestamps_ = true;
LOG(WARNING) << "WAIS::" << __func__
<< " => (WARNING: capture timestamps will be fake)";
} else {
use_fake_audio_capture_timestamps_ = false;
}
LogFakeAudioCaptureTimestamps(use_fake_audio_capture_timestamps_.value(),
delta_time.magnitude());
}
const bool observed_data_discontinuity =
(device_position > 0 && flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY);
if (observed_data_discontinuity) {
LOG(WARNING) << "WAIS::" << __func__
<< " => (WARNING: AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY)";
}
data_discontinuity_reporter_->Log(observed_data_discontinuity);
bool timestamp_error_was_detected = false;
if (flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) {
LOG(WARNING) << "WAIS::" << __func__
<< " => (WARNING: AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR)";
if (num_timestamp_errors_ == 0) {
time_until_first_timestamp_error_ =
base::TimeTicks::Now() - record_start_time_;
}
++num_timestamp_errors_;
timestamp_error_was_detected = true;
}
if (device_position != last_device_position) {
if (expected_next_device_position_ != 0) {
base::TimeDelta glitch_duration;
if (device_position > expected_next_device_position_) {
glitch_duration = AudioTimestampHelper::FramesToTime(
device_position - expected_next_device_position_,
input_format_.Format.nSamplesPerSec);
}
glitch_reporter_.UpdateStats(glitch_duration);
if (glitch_duration.is_positive()) {
glitch_accumulator_.Add(AudioGlitchInfo::SingleBoundedSystemGlitch(
glitch_duration, is_loopback_capture_
? AudioGlitchInfo::Direction::kLoopback
: AudioGlitchInfo::Direction::kCapture));
}
}
last_device_position = device_position;
expected_next_device_position_ = device_position + num_frames_to_read;
} else {
expected_next_device_position_ += num_frames_to_read;
}
base::TimeTicks capture_time;
if (use_fake_audio_capture_timestamps_.has_value() &&
*use_fake_audio_capture_timestamps_) {
capture_time = base::TimeTicks::Now();
} else if (!timestamp_error_was_detected) {
capture_time += base::Microseconds(capture_time_100ns / 10.0);
}
if (capture_time <= last_capture_time_) {
capture_time = last_capture_time_ + base::Microseconds(1);
}
if (!last_capture_time_.is_null()) {
const auto delta_ts = capture_time - last_capture_time_;
if (is_process_loopback_capture_) {
DCHECK_EQ(device_position, 0u);
} else {
DCHECK_GT(device_position, 0u);
}
DCHECK_GT(delta_ts, base::TimeDelta::Min());
if (delta_ts > max_timestamp_diff_) {
max_timestamp_diff_ = delta_ts;
} else if (delta_ts < min_timestamp_diff_) {
min_timestamp_diff_ = delta_ts;
}
}
last_capture_time_ = capture_time;
capture_time -= AudioTimestampHelper::FramesToTime(
fifo_->GetAvailableFrames(), input_format_.Format.nSamplesPerSec);
if (flags & AUDCLNT_BUFFERFLAGS_SILENT) {
fifo_->PushSilence(num_frames_to_read);
} else {
const int bytes_per_sample = input_format_.Format.wBitsPerSample / 8;
CHECK_EQ(input_format_.Format.nBlockAlign,
bytes_per_sample * input_format_.Format.nChannels);
UNSAFE_BUFFERS(base::span<const uint8_t> audio_frames(
reinterpret_cast<const uint8_t*>(data_ptr),
base::CheckMul<size_t>(num_frames_to_read,
input_format_.Format.nBlockAlign)
.ValueOrDie()));
peak_detector_.FindPeak(audio_frames, sample_format_);
fifo_->Push(audio_frames, num_frames_to_read, sample_format_);
}
hr = audio_capture_client_->ReleaseBuffer(num_frames_to_read);
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: IAudioCaptureClient::ReleaseBuffer=["
<< ErrorToString(hr).c_str() << "])";
return;
}
TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("audio"),
"AudioInputCallback::OnData", "capture_time",
capture_time - base::TimeTicks(), "time_ticks_now",
base::TimeTicks::Now() - base::TimeTicks());
double volume = 0.0;
GetAgcVolume(&volume);
while (fifo_->available_blocks()) {
if (converter_) {
if (imperfect_buffer_size_conversion_ &&
fifo_->available_blocks() == 1) {
return;
}
converter_->Convert(convert_bus_.get());
sink_->OnData(convert_bus_.get(), capture_time, volume,
glitch_accumulator_.GetAndReset());
capture_time += AudioTimestampHelper::FramesToTime(
convert_bus_->frames(), output_format_.nSamplesPerSec);
} else {
sink_->OnData(fifo_->Consume(), capture_time, volume,
glitch_accumulator_.GetAndReset());
capture_time += AudioTimestampHelper::FramesToTime(
packet_size_frames_, input_format_.Format.nSamplesPerSec);
}
}
hr = audio_capture_client_->GetNextPacketSize(&num_frames_in_next_packet);
if (FAILED(hr)) {
LOG(ERROR) << "WAIS::" << __func__
<< " => (ERROR: 2-IAudioCaptureClient::GetNextPacketSize=["
<< ErrorToString(hr).c_str() << "])";
return;
}
}
}
void WASAPIAudioInputStream::HandleError(HRESULT err) {
NOTREACHED() << "Error code: " << err;
}
HRESULT WASAPIAudioInputStream::SetCaptureDevice() {
DCHECK(!is_process_loopback_capture_);
DCHECK_EQ(OPEN_RESULT_OK, open_result_);
DCHECK(!endpoint_device_.Get());
SendLogMessage("%s()", __func__);
Microsoft::WRL::ComPtr<IMMDeviceEnumerator> enumerator;
HRESULT hr = ::CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr,
CLSCTX_ALL, IID_PPV_ARGS(&enumerator));
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_CREATE_INSTANCE;
return hr;
}
const EDataFlow data_flow =
AudioDeviceDescription::IsLoopbackDevice(device_id_) ? eRender : eCapture;
const ERole role = AudioDeviceDescription::IsCommunicationsDevice(device_id_)
? eCommunications
: eConsole;
if (AudioDeviceDescription::IsDefaultDevice(device_id_) ||
AudioDeviceDescription::IsCommunicationsDevice(device_id_) ||
AudioDeviceDescription::IsLoopbackDevice(device_id_)) {
hr =
enumerator->GetDefaultAudioEndpoint(data_flow, role, &endpoint_device_);
} else {
hr = enumerator->GetDevice(base::UTF8ToWide(device_id_).c_str(),
&endpoint_device_);
}
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_NO_ENDPOINT;
return hr;
}
hr = endpoint_device_->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL,
nullptr, &system_audio_volume_);
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_ACTIVATION_FAILED;
return hr;
}
DWORD state = DEVICE_STATE_DISABLED;
hr = endpoint_device_->GetState(&state);
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_NO_STATE;
return hr;
}
if (!(state & DEVICE_STATE_ACTIVE)) {
DLOG(ERROR) << "Selected capture device is not active.";
open_result_ = OPEN_RESULT_DEVICE_NOT_ACTIVE;
hr = E_ACCESSDENIED;
}
return hr;
}
HRESULT WASAPIAudioInputStream::ActivateAudioClientInterface() {
if (!is_process_loopback_capture_) {
return endpoint_device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL,
nullptr, &audio_client_);
}
CHECK(is_process_loopback_capture_);
AUDIOCLIENT_ACTIVATION_PARAMS params = {
.ActivationType = AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK,
.ProcessLoopbackParams =
{
.TargetProcessId = GetTargetProcessId(device_id_),
.ProcessLoopbackMode = GetProcessLoopbackMode(device_id_),
},
};
PROPVARIANT propvariant = {
.vt = VT_BLOB,
.blob =
{
.cbSize = sizeof(params),
.pBlobData = reinterpret_cast<BYTE*>(¶ms),
},
};
TRACE_EVENT("audio", "AudioClientActivation");
base::ElapsedTimer timer;
ComPtr<AudioClientActivationHandler> completion_handler =
Microsoft::WRL::Make<AudioClientActivationHandler>();
ComPtr<IActivateAudioInterfaceAsyncOperation> async_op;
HRESULT hr = GetActivateAudioInterfaceAsyncCallback().Run(
VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, __uuidof(IAudioClient),
&propvariant, completion_handler.Get(), &async_op);
if (FAILED(hr)) {
TRACE_EVENT_INSTANT0("audio", "ActivateAudioInterfaceAsync failed",
TRACE_EVENT_SCOPE_THREAD);
return hr;
}
hr = completion_handler->WaitAndGetAudioClient(&audio_client_,
async_activation_timeout_ms_);
const bool timed_out = (hr == kActivationTimeoutHr);
base::UmaHistogramBoolean("Media.Audio.Capture.Win.GetAudioClientTimedOut",
timed_out);
if (!timed_out) {
base::UmaHistogramTimes("Media.Audio.Capture.Win.TimeToGetAudioClient",
timer.Elapsed());
} else {
TRACE_EVENT_INSTANT0("audio", "GetAudioClient timed out",
TRACE_EVENT_SCOPE_THREAD);
}
return hr;
}
bool WASAPIAudioInputStream::RawProcessingSupported() {
DCHECK(!is_process_loopback_capture_);
DCHECK(endpoint_device_.Get());
bool raw_processing_supported = false;
Microsoft::WRL::ComPtr<IPropertyStore> properties;
base::win::ScopedPropVariant raw_processing;
if (FAILED(endpoint_device_->OpenPropertyStore(STGM_READ, &properties)) ||
FAILED(
properties->GetValue(PKEY_Devices_AudioDevice_RawProcessingSupported,
raw_processing.Receive())) ||
raw_processing.get().vt != VT_BOOL) {
SendLogMessage(
"%s => (WARNING: failed to access "
"System.Devices.AudioDevice.RawProcessingSupported)",
__func__);
} else {
raw_processing_supported = VariantBoolToBool(raw_processing.get().boolVal);
SendLogMessage(
"%s => (System.Devices.AudioDevice.RawProcessingSupported=%s)",
__func__, raw_processing_supported ? "true" : "false");
}
return raw_processing_supported;
}
HRESULT WASAPIAudioInputStream::GetAudioEngineNumChannels(WORD* channels) {
DCHECK(audio_client_.Get());
SendLogMessage("%s()", __func__);
WAVEFORMATEXTENSIBLE mix_format;
HRESULT hr =
CoreAudioUtil::GetSharedModeMixFormat(audio_client_.Get(), &mix_format);
if (SUCCEEDED(hr)) {
CoreAudioUtil::WaveFormatWrapper wformat(&mix_format);
*channels = wformat->nChannels;
SendLogMessage("%s => (native channels=[%d])", __func__, *channels);
}
return hr;
}
HRESULT
WASAPIAudioInputStream::SetCommunicationsCategoryAndMaybeRawCaptureMode(
WORD channels) {
DCHECK(audio_client_.Get());
DCHECK(!AudioDeviceDescription::IsLoopbackDevice(device_id_));
DCHECK(raw_processing_supported_);
SendLogMessage("%s({channels=%d})", __func__, channels);
Microsoft::WRL::ComPtr<IAudioClient2> audio_client2;
HRESULT hr = audio_client_.As(&audio_client2);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient2 is not supported)", __func__);
return hr;
}
if (audio_client2.Get()) {
AudioClientProperties audio_props = {0};
audio_props.cbSize = sizeof(AudioClientProperties);
audio_props.bIsOffload = false;
audio_props.eCategory = AudioCategory_Communications;
if (channels > 0 && channels <= media::kMaxConcurrentChannels) {
audio_props.Options = AUDCLNT_STREAMOPTIONS_RAW;
}
if (aec_config_) {
audio_props.Options = AUDCLNT_STREAMOPTIONS_NONE;
SendLogMessage("%s => (WARNING: attempting to enable system AEC)",
__func__);
}
hr = audio_client2->SetClientProperties(&audio_props);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient2::SetClientProperties=[%s])",
__func__, ErrorToString(hr).c_str());
}
}
return hr;
}
bool WASAPIAudioInputStream::DesiredFormatIsSupported(HRESULT* hr) {
SendLogMessage("%s()", __func__);
if (is_process_loopback_capture_) {
return true;
}
base::win::ScopedCoMem<WAVEFORMATEX> closest_match;
HRESULT hresult = audio_client_->IsFormatSupported(
AUDCLNT_SHAREMODE_SHARED,
reinterpret_cast<const WAVEFORMATEX*>(&input_format_), &closest_match);
if (FAILED(hresult)) {
SendLogMessage("%s => (ERROR: IAudioClient::IsFormatSupported=[%s])",
__func__, ErrorToString(hresult).c_str());
}
if (hresult == S_FALSE) {
SendLogMessage(
"%s => (WARNING: Format is not supported but a closest match exists)",
__func__);
WAVEFORMATEX* input_format = &input_format_.Format;
input_format->nChannels = closest_match->nChannels;
input_format->nSamplesPerSec = closest_match->nSamplesPerSec;
if (CoreAudioUtil::WaveFormatWrapper(closest_match.get()).IsPcm() &&
input_format->wBitsPerSample != closest_match->wBitsPerSample) {
CHECK(!use_device_sample_format_, base::NotFatalUntil::M148);
input_format->wBitsPerSample = closest_match->wBitsPerSample;
}
input_format->nBlockAlign =
(input_format->wBitsPerSample / 8) * input_format->nChannels;
input_format->nAvgBytesPerSec =
input_format->nSamplesPerSec * input_format->nBlockAlign;
if (IsSupportedFormatForConversion(&input_format_)) {
SendLogMessage(
"%s => (WARNING: Captured audio will be converted: [%s] ==> [%s])",
__func__, CoreAudioUtil::WaveFormatToString(&input_format_).c_str(),
CoreAudioUtil::WaveFormatToString(&output_format_).c_str());
SetupConverterAndStoreFormatInfo();
hresult = S_OK;
}
}
*hr = hresult;
return (hresult == S_OK);
}
void WASAPIAudioInputStream::SetupConverterAndStoreFormatInfo() {
const double buffer_ratio =
output_format_.nSamplesPerSec / static_cast<double>(packet_size_frames_);
double new_frames_per_buffer =
input_format_.Format.nSamplesPerSec / buffer_ratio;
const auto input_layout =
ChannelLayoutConfig::Guess(input_format_.Format.nChannels);
DCHECK_NE(CHANNEL_LAYOUT_UNSUPPORTED, input_layout.channel_layout());
const auto output_layout =
ChannelLayoutConfig::Guess(output_format_.nChannels);
DCHECK_NE(CHANNEL_LAYOUT_UNSUPPORTED, output_layout.channel_layout());
const AudioParameters input(AudioParameters::AUDIO_PCM_LOW_LATENCY,
input_layout, input_format_.Format.nSamplesPerSec,
static_cast<int>(new_frames_per_buffer));
const AudioParameters output(AudioParameters::AUDIO_PCM_LOW_LATENCY,
output_layout, output_format_.nSamplesPerSec,
packet_size_frames_);
converter_ = std::make_unique<AudioConverter>(input, output, false);
converter_->AddInput(this);
converter_->PrimeWithSilence();
convert_bus_ = AudioBus::Create(output);
const auto new_bytes_per_buffer = static_cast<int>(new_frames_per_buffer) *
input_format_.Format.nBlockAlign;
packet_size_frames_ = new_bytes_per_buffer / input_format_.Format.nBlockAlign;
packet_size_bytes_ = new_bytes_per_buffer;
frame_size_bytes_ = input_format_.Format.nBlockAlign;
imperfect_buffer_size_conversion_ =
std::modf(new_frames_per_buffer, &new_frames_per_buffer) != 0.0;
if (imperfect_buffer_size_conversion_) {
SendLogMessage("%s => (WARNING: Audio capture conversion requires a FIFO)",
__func__);
}
}
HRESULT WASAPIAudioInputStream::InitializeAudioEngine() {
DCHECK_EQ(OPEN_RESULT_OK, open_result_);
SendLogMessage("%s()", __func__);
DWORD flags =
IsEndpointLoopbackCapture(device_id_, is_process_loopback_capture_)
? 0
: AUDCLNT_STREAMFLAGS_EVENTCALLBACK;
if (!is_process_loopback_capture_) {
flags |= AUDCLNT_STREAMFLAGS_NOPERSIST;
}
if (AudioDeviceDescription::IsLoopbackDevice(device_id_)) {
flags |= AUDCLNT_STREAMFLAGS_LOOPBACK;
}
HRESULT hr = audio_client_->Initialize(
AUDCLNT_SHAREMODE_SHARED, flags,
100 * 1000 * 10,
0,
reinterpret_cast<const WAVEFORMATEX*>(&input_format_),
AudioDeviceDescription::IsCommunicationsDevice(device_id_)
? &kCommunicationsSessionId
: nullptr);
if (FAILED(hr)) {
SendLogMessage("%s => (ERROR: IAudioClient::Initialize=[%s])", __func__,
ErrorToString(hr).c_str());
open_result_ = OPEN_RESULT_AUDIO_CLIENT_INIT_FAILED;
base::UmaHistogramSparse("Media.Audio.Capture.Win.InitError", hr);
MaybeReportFormatRelatedInitError(hr);
return hr;
}
if (!is_process_loopback_capture_) {
hr = CreateFifoIfNeeded();
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_GET_BUFFER_SIZE_FAILED;
return hr;
}
}
#ifndef NDEBUG
REFERENCE_TIME device_period_shared_mode = 0;
REFERENCE_TIME device_period_exclusive_mode = 0;
HRESULT hr_dbg = audio_client_->GetDevicePeriod(
&device_period_shared_mode, &device_period_exclusive_mode);
if (SUCCEEDED(hr_dbg)) {
const int device_period_ms = (device_period_shared_mode + 5000) / 10000;
DVLOG(1) << "Device period: " << device_period_ms << " ms";
}
REFERENCE_TIME latency = 0;
hr_dbg = audio_client_->GetStreamLatency(&latency);
if (SUCCEEDED(hr_dbg)) {
const int latency_ms = (latency + 5000) / 10000;
DVLOG(1) << "Stream latency: " << latency_ms << " ms";
}
#endif
if (IsEndpointLoopbackCapture(device_id_, is_process_loopback_capture_)) {
SendLogMessage("%s => (WARNING: endpoint loopback mode is selected)",
__func__);
hr = endpoint_device_->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
&audio_render_client_for_loopback_);
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_LOOPBACK_ACTIVATE_FAILED;
return hr;
}
hr = audio_render_client_for_loopback_->Initialize(
AUDCLNT_SHAREMODE_SHARED,
AUDCLNT_STREAMFLAGS_EVENTCALLBACK | AUDCLNT_STREAMFLAGS_NOPERSIST, 0, 0,
reinterpret_cast<const WAVEFORMATEX*>(&input_format_),
AudioDeviceDescription::IsCommunicationsDevice(device_id_)
? &kCommunicationsSessionId
: nullptr);
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_LOOPBACK_INIT_FAILED;
return hr;
}
hr = audio_render_client_for_loopback_->SetEventHandle(
audio_samples_ready_event_.Get());
} else {
hr = audio_client_->SetEventHandle(audio_samples_ready_event_.Get());
}
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_SET_EVENT_HANDLE;
return hr;
}
hr = audio_client_->GetService(IID_PPV_ARGS(&audio_capture_client_));
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_NO_CAPTURE_CLIENT;
return hr;
}
if (!is_process_loopback_capture_) {
hr = audio_client_->GetService(IID_PPV_ARGS(&simple_audio_volume_));
if (FAILED(hr)) {
open_result_ = OPEN_RESULT_NO_AUDIO_VOLUME;
}
}
return hr;
}
void WASAPIAudioInputStream::ReportOpenResult(HRESULT hr) {
DCHECK(!opened_);
UMA_HISTOGRAM_ENUMERATION("Media.Audio.Capture.Win.Open", open_result_,
OPEN_RESULT_MAX + 1);
if (open_result_ != OPEN_RESULT_OK &&
open_result_ != OPEN_RESULT_OK_WITH_RESAMPLING) {
SendLogMessage(
"%s", GetOpenLogString(open_result_, hr, input_format_, output_format_)
.c_str());
}
}
void WASAPIAudioInputStream::MaybeReportFormatRelatedInitError(
HRESULT hr) const {
if (hr != AUDCLNT_E_UNSUPPORTED_FORMAT && hr != E_INVALIDARG)
return;
const FormatRelatedInitError format_related_error =
hr == AUDCLNT_E_UNSUPPORTED_FORMAT
? converter_.get()
? FormatRelatedInitError::kUnsupportedFormatWithFormatConversion
: FormatRelatedInitError::kUnsupportedFormat
: converter_.get()
? FormatRelatedInitError::kInvalidArgumentWithFormatConversion
: FormatRelatedInitError::kInvalidArgument;
base::UmaHistogramEnumeration(
"Media.Audio.Capture.Win.InitError.FormatRelated", format_related_error,
FormatRelatedInitError::kCount);
}
double WASAPIAudioInputStream::ProvideInput(
AudioBus* audio_bus,
uint32_t frames_delayed,
const AudioGlitchInfo& glitch_info) {
CHECK_DEREF(fifo_.get()).Consume()->CopyTo(audio_bus);
return 1.0;
}
void WASAPIAudioInputStream::ReportAndResetGlitchStats() {
glitch_accumulator_.GetAndReset();
SystemGlitchReporter::Stats stats =
glitch_reporter_.GetLongTermStatsAndReset();
SendLogMessage(
"%s => (num_glitches_detected=[%d], cumulative_audio_lost=[%llu ms], "
"largest_glitch=[%llu ms])",
__func__, stats.glitches_detected,
stats.total_glitch_duration.InMilliseconds(),
stats.largest_glitch_duration.InMilliseconds());
int num_data_discontinuities =
data_discontinuity_reporter_->GetLongTermDiscontinuityCountAndReset();
SendLogMessage("%s => (discontinuity warnings=[%d])", __func__,
num_data_discontinuities);
SendLogMessage("%s => (timstamp errors=[%" PRIu64 "])", __func__,
num_timestamp_errors_);
if (num_timestamp_errors_ > 0) {
SendLogMessage("%s => (time until first timestamp error=[%" PRId64 " ms])",
__func__,
time_until_first_timestamp_error_.InMilliseconds());
}
expected_next_device_position_ = 0;
num_timestamp_errors_ = 0;
}
}