#include "ash/assistant/assistant_interaction_controller_impl.h"
#include <utility>
#include "ash/accessibility/accessibility_controller_impl.h"
#include "ash/assistant/assistant_controller_impl.h"
#include "ash/assistant/model/assistant_interaction_model_observer.h"
#include "ash/assistant/model/assistant_query.h"
#include "ash/assistant/model/assistant_response.h"
#include "ash/assistant/model/assistant_ui_model.h"
#include "ash/assistant/model/ui/assistant_card_element.h"
#include "ash/assistant/model/ui/assistant_error_element.h"
#include "ash/assistant/model/ui/assistant_text_element.h"
#include "ash/assistant/ui/assistant_ui_constants.h"
#include "ash/assistant/util/assistant_util.h"
#include "ash/assistant/util/deep_link_util.h"
#include "ash/assistant/util/histogram_util.h"
#include "ash/constants/ash_pref_names.h"
#include "ash/public/cpp/android_intent_helper.h"
#include "ash/public/cpp/assistant/assistant_setup.h"
#include "ash/public/cpp/assistant/assistant_state.h"
#include "ash/public/cpp/assistant/controller/assistant_suggestions_controller.h"
#include "ash/public/cpp/assistant/controller/assistant_ui_controller.h"
#include "ash/session/session_controller_impl.h"
#include "ash/shell.h"
#include "ash/strings/grit/ash_strings.h"
#include "base/functional/bind.h"
#include "base/metrics/histogram_functions.h"
#include "base/strings/string_util.h"
#include "base/strings/utf_string_conversions.h"
#include "base/task/sequenced_task_runner.h"
#include "chromeos/ash/services/assistant/public/cpp/features.h"
#include "components/prefs/pref_registry_simple.h"
#include "components/prefs/pref_service.h"
#include "net/base/url_util.h"
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "ui/base/l10n/l10n_util.h"
namespace ash {
namespace {
using assistant::features::IsWaitSchedulingEnabled;
constexpr char kAndroidIntentScheme[] = "intent://";
constexpr char kAndroidIntentPrefix[] = "#Intent";
ash::TabletModeController* GetTabletModeController() {
return Shell::Get()->tablet_mode_controller();
}
bool IsTabletMode() {
return GetTabletModeController()->InTabletMode();
}
bool launch_with_mic_open() {
return AssistantState::Get()->launch_with_mic_open().value_or(false);
}
bool IsPreferVoice() {
return launch_with_mic_open() || IsTabletMode();
}
PrefService* pref_service() {
auto* result =
Shell::Get()->session_controller()->GetPrimaryUserPrefService();
DCHECK(result);
return result;
}
}
AssistantInteractionControllerImpl::AssistantInteractionControllerImpl(
AssistantControllerImpl* assistant_controller)
: assistant_controller_(assistant_controller) {
model_.AddObserver(this);
assistant_controller_observation_.Observe(AssistantController::Get());
tablet_mode_controller_observation_.Observe(GetTabletModeController());
}
AssistantInteractionControllerImpl::~AssistantInteractionControllerImpl() {
model_.RemoveObserver(this);
if (assistant_)
assistant_->RemoveAssistantInteractionSubscriber(this);
}
void AssistantInteractionControllerImpl::RegisterProfilePrefs(
PrefRegistrySimple* registry) {
registry->RegisterTimePref(prefs::kAssistantTimeOfLastInteraction,
base::Time());
}
void AssistantInteractionControllerImpl::SetAssistant(
assistant::Assistant* assistant) {
if (assistant_)
assistant_->RemoveAssistantInteractionSubscriber(this);
assistant_ = assistant;
if (assistant_)
assistant_->AddAssistantInteractionSubscriber(this);
}
const AssistantInteractionModel* AssistantInteractionControllerImpl::GetModel()
const {
return &model_;
}
base::TimeDelta
AssistantInteractionControllerImpl::GetTimeDeltaSinceLastInteraction() const {
return base::Time::Now() -
pref_service()->GetTime(prefs::kAssistantTimeOfLastInteraction);
}
bool AssistantInteractionControllerImpl::HasHadInteraction() const {
return has_had_interaction_;
}
void AssistantInteractionControllerImpl::StartTextInteraction(
const std::string& text,
bool allow_tts,
AssistantQuerySource query_source) {
DCHECK(assistant_);
model_.SetPendingQuery(
std::make_unique<AssistantTextQuery>(text, query_source));
assistant_->StartTextInteraction(text, query_source, allow_tts);
}
void AssistantInteractionControllerImpl::OnAssistantControllerConstructed() {
AssistantUiController::Get()->GetModel()->AddObserver(this);
assistant_controller_->view_delegate()->AddObserver(this);
}
void AssistantInteractionControllerImpl::OnAssistantControllerDestroying() {
assistant_controller_->view_delegate()->RemoveObserver(this);
AssistantUiController::Get()->GetModel()->RemoveObserver(this);
}
void AssistantInteractionControllerImpl::OnDeepLinkReceived(
assistant::util::DeepLinkType type,
const std::map<std::string, std::string>& params) {
using assistant::util::DeepLinkParam;
using assistant::util::DeepLinkType;
if (type == DeepLinkType::kReminders) {
using ReminderAction = assistant::util::ReminderAction;
const absl::optional<ReminderAction>& action =
GetDeepLinkParamAsRemindersAction(params, DeepLinkParam::kAction);
if (!action)
return;
switch (action.value()) {
case ReminderAction::kCreate:
StartTextInteraction(
l10n_util::GetStringUTF8(IDS_ASSISTANT_CREATE_REMINDER_QUERY),
model_.response() && model_.response()->has_tts(),
AssistantQuerySource::kDeepLink);
break;
case ReminderAction::kEdit:
const absl::optional<std::string>& client_id =
GetDeepLinkParam(params, DeepLinkParam::kClientId);
if (client_id && !client_id.value().empty()) {
model_.SetPendingQuery(std::make_unique<AssistantTextQuery>(
l10n_util::GetStringUTF8(IDS_ASSISTANT_EDIT_REMINDER_QUERY),
AssistantQuerySource::kDeepLink));
assistant_->StartEditReminderInteraction(client_id.value());
}
break;
}
return;
}
if (type != DeepLinkType::kQuery)
return;
const absl::optional<std::string>& query =
GetDeepLinkParam(params, DeepLinkParam::kQuery);
if (!query.has_value())
return;
if (query.value().empty()) {
LOG(ERROR) << "Ignoring deep link containing empty query.";
return;
}
const AssistantEntryPoint entry_point =
GetDeepLinkParamAsEntryPoint(params, DeepLinkParam::kEntryPoint)
.value_or(AssistantEntryPoint::kDeepLink);
AssistantUiController::Get()->ShowUi(entry_point);
const AssistantQuerySource query_source =
GetDeepLinkParamAsQuerySource(params, DeepLinkParam::kQuerySource)
.value_or(AssistantQuerySource::kDeepLink);
StartTextInteraction(query.value(), model_.response() &&
model_.response()->has_tts(),
query_source);
}
void AssistantInteractionControllerImpl::OnUiVisibilityChanged(
AssistantVisibility new_visibility,
AssistantVisibility old_visibility,
absl::optional<AssistantEntryPoint> entry_point,
absl::optional<AssistantExitPoint> exit_point) {
switch (new_visibility) {
case AssistantVisibility::kClosed:
StopActiveInteraction(true);
model_.ClearInteraction();
model_.SetInputModality(GetDefaultInputModality());
break;
case AssistantVisibility::kVisible:
OnUiVisible(entry_point.value());
break;
case AssistantVisibility::kClosing:
break;
}
}
void AssistantInteractionControllerImpl::OnInputModalityChanged(
InputModality input_modality) {
if (!IsVisible())
return;
if (input_modality == InputModality::kVoice)
return;
StopActiveInteraction(false);
}
void AssistantInteractionControllerImpl::OnMicStateChanged(MicState mic_state) {
if (mic_state == MicState::kOpen)
Shell::Get()->accessibility_controller()->SilenceSpokenFeedback();
}
void AssistantInteractionControllerImpl::OnCommittedQueryChanged(
const AssistantQuery& assistant_query) {
pref_service()->SetTime(prefs::kAssistantTimeOfLastInteraction,
base::Time::Now());
has_had_interaction_ = true;
std::string query;
switch (assistant_query.type()) {
case AssistantQueryType::kText: {
const auto* assistant_text_query =
static_cast<const AssistantTextQuery*>(&assistant_query);
query = assistant_text_query->text();
break;
}
case AssistantQueryType::kVoice: {
const auto* assistant_voice_query =
static_cast<const AssistantVoiceQuery*>(&assistant_query);
query = assistant_voice_query->high_confidence_speech();
break;
}
case AssistantQueryType::kNull:
break;
}
model_.query_history().Add(query);
assistant::util::IncrementAssistantQueryCountForEntryPoint(
AssistantUiController::Get()->GetModel()->entry_point());
assistant::util::RecordAssistantQuerySource(assistant_query.source());
}
void AssistantInteractionControllerImpl::OnInteractionStarted(
const AssistantInteractionMetadata& metadata) {
VLOG(1) << __func__;
auto* assistant_setup = AssistantSetup::GetInstance();
if (assistant_setup && assistant_setup->BounceOptInWindowIfActive()) {
StopActiveInteraction(true);
return;
}
const bool is_voice_interaction =
assistant::AssistantInteractionType::kVoice == metadata.type;
if (is_voice_interaction) {
AssistantUiController::Get()->ShowUi(AssistantEntryPoint::kHotword);
}
model_.SetInteractionState(InteractionState::kActive);
if (is_voice_interaction) {
model_.SetInputModality(InputModality::kVoice);
model_.SetMicState(MicState::kOpen);
if (model_.pending_query().type() == AssistantQueryType::kNull) {
model_.SetPendingQuery(std::make_unique<AssistantVoiceQuery>());
}
} else {
if (model_.pending_query().type() == AssistantQueryType::kNull) {
model_.SetPendingQuery(std::make_unique<AssistantTextQuery>(
metadata.query, metadata.source));
}
model_.CommitPendingQuery();
model_.SetMicState(MicState::kClosed);
}
model_.SetPendingResponse(base::MakeRefCounted<AssistantResponse>());
}
void AssistantInteractionControllerImpl::OnInteractionFinished(
AssistantInteractionResolution resolution) {
VLOG(1) << __func__;
base::UmaHistogramEnumeration("Assistant.Interaction.Resolution", resolution);
model_.SetMicState(MicState::kClosed);
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
model_.SetInteractionState(InteractionState::kInactive);
const bool is_mic_timeout =
resolution == AssistantInteractionResolution::kMicTimeout ||
(resolution == AssistantInteractionResolution::kNormal &&
model_.pending_query().type() == AssistantQueryType::kVoice &&
model_.pending_query().Empty());
if (is_mic_timeout) {
model_.ClearPendingQuery();
model_.ClearPendingResponse();
return;
}
if (model_.pending_query().type() != AssistantQueryType::kNull)
model_.CommitPendingQuery();
AssistantResponse* response = GetResponseForActiveInteraction();
switch (resolution) {
case AssistantInteractionResolution::kError: {
auto err = std::make_unique<AssistantErrorElement>(
IDS_ASH_ASSISTANT_ERROR_GENERIC);
if (!response->ContainsUiElement(err.get()))
response->AddUiElement(std::move(err));
break;
}
case AssistantInteractionResolution::kMultiDeviceHotwordLoss:
response->AddUiElement(
std::make_unique<AssistantTextElement>(l10n_util::GetStringUTF8(
IDS_ASH_ASSISTANT_MULTI_DEVICE_HOTWORD_LOSS)));
break;
case AssistantInteractionResolution::kMicTimeout:
NOTREACHED();
break;
case AssistantInteractionResolution::kInterruption:
case AssistantInteractionResolution::kNormal:
break;
}
if (response == model_.pending_response())
model_.CommitPendingResponse();
}
void AssistantInteractionControllerImpl::OnHtmlResponse(
const std::string& html,
const std::string& fallback) {
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
DCHECK(AssistantUiController::Get());
AssistantResponse* response = GetResponseForActiveInteraction();
response->AddUiElement(std::make_unique<AssistantCardElement>(
html, fallback,
AssistantUiController::Get()->GetModel()->AppListBubbleWidth()));
if (response == model_.pending_response())
model_.CommitPendingResponse();
}
void AssistantInteractionControllerImpl::OnSuggestionPressed(
const base::UnguessableToken& suggestion_id) {
auto* suggestion =
AssistantSuggestionsController::Get()->GetModel()->GetSuggestionById(
suggestion_id);
if (!suggestion && model_.response())
suggestion = model_.response()->GetSuggestionById(suggestion_id);
DCHECK(suggestion);
if (!suggestion->action_url.is_empty()) {
base::SequencedTaskRunner::GetCurrentDefault()->PostTask(
FROM_HERE,
base::BindOnce(&AssistantController::OpenUrl,
AssistantController::Get()->GetWeakPtr(),
suggestion->action_url, false,
false));
return;
}
AssistantQuerySource query_source;
switch (suggestion->type) {
case AssistantSuggestionType::kBetterOnboarding:
query_source = AssistantQuerySource::kBetterOnboarding;
base::UmaHistogramEnumeration("Assistant.BetterOnboarding.Click",
suggestion->better_onboarding_type);
break;
case AssistantSuggestionType::kConversationStarter:
query_source = AssistantQuerySource::kConversationStarter;
break;
case AssistantSuggestionType::kUnspecified:
query_source = AssistantQuerySource::kSuggestionChip;
break;
}
StartTextInteraction(
suggestion->text,
model_.response() && model_.response()->has_tts(),
query_source);
}
void AssistantInteractionControllerImpl::OnTabletModeStarted() {
OnTabletModeChanged();
}
void AssistantInteractionControllerImpl::OnTabletModeEnded() {
OnTabletModeChanged();
}
void AssistantInteractionControllerImpl::OnTabletModeChanged() {
if (!HasActiveInteraction() && !IsVisible())
model_.SetInputModality(GetDefaultInputModality());
}
void AssistantInteractionControllerImpl::OnSuggestionsResponse(
const std::vector<AssistantSuggestion>& suggestions) {
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
AssistantResponse* response = GetResponseForActiveInteraction();
response->AddSuggestions(suggestions);
if (response == model_.pending_response())
model_.CommitPendingResponse();
}
void AssistantInteractionControllerImpl::OnTextResponse(
const std::string& text) {
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
AssistantResponse* response = GetResponseForActiveInteraction();
response->AddUiElement(std::make_unique<AssistantTextElement>(text));
if (response == model_.pending_response())
model_.CommitPendingResponse();
}
void AssistantInteractionControllerImpl::OnSpeechRecognitionStarted() {}
void AssistantInteractionControllerImpl::OnSpeechRecognitionIntermediateResult(
const std::string& high_confidence_text,
const std::string& low_confidence_text) {
model_.SetPendingQuery(std::make_unique<AssistantVoiceQuery>(
high_confidence_text, low_confidence_text));
}
void AssistantInteractionControllerImpl::OnSpeechRecognitionEndOfUtterance() {
model_.SetMicState(MicState::kClosed);
}
void AssistantInteractionControllerImpl::OnSpeechRecognitionFinalResult(
const std::string& final_result) {
if (final_result.empty())
return;
model_.SetPendingQuery(std::make_unique<AssistantVoiceQuery>(final_result));
model_.CommitPendingQuery();
}
void AssistantInteractionControllerImpl::OnSpeechLevelUpdated(
float speech_level) {
model_.SetSpeechLevel(speech_level);
}
void AssistantInteractionControllerImpl::OnTtsStarted(bool due_to_error) {
Shell::Get()->accessibility_controller()->SilenceSpokenFeedback();
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
if (model_.pending_query().type() != AssistantQueryType::kNull)
model_.CommitPendingQuery();
AssistantResponse* response = GetResponseForActiveInteraction();
if (due_to_error) {
model_.SetMicState(MicState::kClosed);
auto err = std::make_unique<AssistantErrorElement>(
IDS_ASH_ASSISTANT_ERROR_GENERIC);
if (!response->ContainsUiElement(err.get()))
response->AddUiElement(std::move(err));
}
response->set_has_tts(true);
if (response == model_.pending_response())
model_.CommitPendingResponse();
}
void AssistantInteractionControllerImpl::OnWaitStarted() {
DCHECK(IsWaitSchedulingEnabled());
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
if (model_.pending_query().type() != AssistantQueryType::kNull)
model_.CommitPendingQuery();
if (model_.pending_response())
model_.CommitPendingResponse();
}
void AssistantInteractionControllerImpl::OnOpenUrlResponse(const GURL& url,
bool in_background) {
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
AssistantController::Get()->OpenUrl(url, in_background, true);
}
void AssistantInteractionControllerImpl::OnOpenAppResponse(
const assistant::AndroidAppInfo& app_info) {
if (!HasActiveInteraction()) {
DVLOG(1) << "Assistant: Dropping response outside of active interaction";
return;
}
auto* android_helper = AndroidIntentHelper::GetInstance();
if (!android_helper)
return;
auto intent = android_helper->GetAndroidAppLaunchIntent(app_info);
if (!intent.has_value())
return;
auto intent_str = intent.value();
if (base::StartsWith(intent_str, kAndroidIntentPrefix,
base::CompareCase::SENSITIVE)) {
intent_str = kAndroidIntentScheme + intent_str;
}
AssistantController::Get()->OpenUrl(GURL(intent_str), false,
true);
}
void AssistantInteractionControllerImpl::OnDialogPlateButtonPressed(
AssistantButtonId id) {
if (id == AssistantButtonId::kKeyboardInputToggle) {
model_.SetInputModality(InputModality::kKeyboard);
return;
}
if (id != AssistantButtonId::kVoiceInputToggle)
return;
switch (model_.mic_state()) {
case MicState::kClosed:
StartVoiceInteraction();
break;
case MicState::kOpen:
StopActiveInteraction(false);
break;
}
}
void AssistantInteractionControllerImpl::OnDialogPlateContentsCommitted(
const std::string& text) {
DCHECK(!text.empty());
StartTextInteraction(
text, false,
AssistantQuerySource::kDialogPlateTextField);
}
bool AssistantInteractionControllerImpl::HasActiveInteraction() const {
return model_.interaction_state() == InteractionState::kActive;
}
void AssistantInteractionControllerImpl::OnUiVisible(
AssistantEntryPoint entry_point) {
DCHECK(IsVisible());
if (assistant::util::IsVoiceEntryPoint(entry_point, IsPreferVoice()) &&
entry_point != AssistantEntryPoint::kHotword) {
StartVoiceInteraction();
return;
}
}
void AssistantInteractionControllerImpl::StartVoiceInteraction() {
model_.SetPendingQuery(std::make_unique<AssistantVoiceQuery>());
assistant_->StartVoiceInteraction();
}
void AssistantInteractionControllerImpl::StopActiveInteraction(
bool cancel_conversation) {
model_.SetInteractionState(InteractionState::kInactive);
model_.ClearPendingQuery();
if (AssistantState::Get()->assistant_status() ==
assistant::AssistantStatus::READY) {
assistant_->StopActiveInteraction(cancel_conversation);
}
model_.ClearPendingResponse();
}
InputModality AssistantInteractionControllerImpl::GetDefaultInputModality()
const {
return IsPreferVoice() ? InputModality::kVoice : InputModality::kKeyboard;
}
AssistantResponse*
AssistantInteractionControllerImpl::GetResponseForActiveInteraction() {
return model_.pending_response() ? model_.pending_response()
: model_.response();
}
AssistantVisibility AssistantInteractionControllerImpl::GetVisibility() const {
return AssistantUiController::Get()->GetModel()->visibility();
}
bool AssistantInteractionControllerImpl::IsVisible() const {
return GetVisibility() == AssistantVisibility::kVisible;
}
}