#include "chrome/renderer/accessibility/read_anything/read_anything_app_model.h"
#include <cstddef>
#include <string>
#include <utility>
#include "base/check.h"
#include "base/containers/contains.h"
#include "base/metrics/histogram_functions.h"
#include "base/strings/stringprintf.h"
#include "chrome/common/read_anything/read_anything_util.h"
#include "chrome/renderer/accessibility/read_anything/read_aloud_traversal_utils.h"
#include "chrome/renderer/accessibility/read_anything/read_anything_node_utils.h"
#include "content/public/renderer/render_thread.h"
#include "services/strings/grit/services_strings.h"
#include "ui/accessibility/accessibility_features.h"
#include "ui/accessibility/ax_enum_util.h"
#include "ui/accessibility/ax_enums.mojom-shared.h"
#include "ui/accessibility/ax_node.h"
#include "ui/accessibility/ax_node_id_forward.h"
#include "ui/accessibility/ax_role_properties.h"
#include "ui/accessibility/ax_selection.h"
#include "ui/accessibility/ax_serializable_tree.h"
#include "ui/accessibility/ax_text_utils.h"
#include "ui/accessibility/ax_tree_id.h"
#include "ui/accessibility/ax_tree_observer.h"
#include "ui/accessibility/ax_tree_update_util.h"
#include "ui/base/l10n/l10n_util.h"
#include "url/gurl.h"
namespace {
base::TimeDelta kTimeElapsedSincePageLoadForDataCollection = base::Seconds(30);
base::TimeDelta kTimeElapsedSinceTreeChangedForDataCollection =
base::Seconds(30);
const ui::AXNode* GetUnignoredParentForSelection(const ui::AXNode* node) {
const ui::AXNode* parent = node;
while (const ui::AXNode* ancestor =
parent->GetUnignoredParentCrossingTreeBoundary()) {
static constexpr auto should_skip = [](const ui::AXNode* node) {
const std::string_view display =
node->GetStringAttribute(ax::mojom::StringAttribute::kDisplay);
return base::Contains(display, "inline") ||
base::Contains(display, "list-item");
};
if (!should_skip(ancestor)) {
return ancestor;
}
parent = ancestor;
}
return parent == node ? nullptr : parent;
}
enum class ReadAnythingHeuristics {
kNone = 0,
kNodeNotFound = 1,
kInvisibleOrIgnored = 2,
kNotExpanded = 3,
kNoDeepsetLastDecendent = 4,
kMaxValue = kNoDeepsetLastDecendent
};
void RecordHeuristicMetric(ReadAnythingHeuristics heuristic) {
base::UmaHistogramEnumeration("Accessibility.ReadAnything.Heuristics",
heuristic);
}
}
ReadAnythingAppModel::AXTreeInfo::AXTreeInfo(
std::unique_ptr<ui::AXTreeManager> manager)
: manager(std::move(manager)) {}
ReadAnythingAppModel::AXTreeInfo::~AXTreeInfo() = default;
ReadAnythingAppModel::SelectionEndpoint::SelectionEndpoint(
const ui::AXSelection& selection,
Source source)
: id(source == Source::kAnchor ? selection.anchor_object_id
: selection.focus_object_id),
offset(source == Source::kAnchor ? selection.anchor_offset
: selection.focus_offset) {}
ReadAnythingAppModel::ReadAnythingAppModel() {
ResetTextSize();
}
ReadAnythingAppModel::~ReadAnythingAppModel() = default;
void ReadAnythingAppModel::InsertIdIfNotIgnored(
ui::AXNodeID id,
std::set<ui::AXNodeID>& non_ignored_ids) {
const ui::AXNode* const ax_node = GetAXNode(id);
if (!ax_node) {
return;
}
if (!a11y::IsIgnored(ax_node, is_pdf_)) {
non_ignored_ids.insert(id);
}
}
void ReadAnythingAppModel::OnSettingsRestoredFromPrefs(
read_anything::mojom::LineSpacing line_spacing,
read_anything::mojom::LetterSpacing letter_spacing,
std::string font_name,
double font_size,
bool links_enabled,
bool images_enabled,
read_anything::mojom::Colors color) {
line_spacing_ = line_spacing;
letter_spacing_ = letter_spacing;
font_name_ = std::move(font_name);
SetFontSize(font_size);
links_enabled_ = links_enabled;
images_enabled_ = images_enabled;
color_theme_ = color;
}
void ReadAnythingAppModel::Reset(std::vector<ui::AXNodeID> content_node_ids) {
content_node_ids_ = std::move(content_node_ids);
display_node_ids_.clear();
distillation_in_progress_ = false;
requires_post_process_selection_ = false;
selections_from_reading_mode_ = 0;
ResetSelection();
}
void ReadAnythingAppModel::ResetSelection() {
selection_node_ids_.clear();
start_ = SelectionEndpoint();
end_ = SelectionEndpoint();
}
bool ReadAnythingAppModel::PostProcessSelection() {
CHECK_NE(active_tree_id_, ui::AXTreeIDUnknown());
const auto it = tree_infos_.find(active_tree_id_);
CHECK(it != tree_infos_.end());
requires_post_process_selection_ = false;
const auto selection_in_distilled_content = [&] {
return display_node_ids_.contains(start_.id) &&
display_node_ids_.contains(end_.id);
};
const bool need_to_draw = (selections_from_reading_mode_ == 0) &&
has_selection() &&
!selection_in_distilled_content();
const bool was_empty = is_empty();
ResetSelection();
if (const ui::AXSelection selection =
GetTreeFromId(active_tree_id_)->GetUnignoredSelection();
selection.anchor_object_id != ui::kInvalidAXNodeID &&
selection.focus_object_id != ui::kInvalidAXNodeID &&
!selection.IsCollapsed()) {
auto source_start = SelectionEndpoint::Source::kAnchor,
source_end = SelectionEndpoint::Source::kFocus;
if (selection.is_backward) {
std::swap(source_start, source_end);
}
start_ = SelectionEndpoint(selection, source_start);
end_ = SelectionEndpoint(selection, source_end);
}
if (!has_selection()) {
return need_to_draw;
}
if (was_empty) {
base::UmaHistogramEnumeration(kEmptyStateHistogramName,
EmptyState::kShownWithSelectionAfter);
++it->second->num_selections;
}
if (selection_in_distilled_content()) {
return need_to_draw;
}
const ui::AXNode* node = GetAXNode(start_.id);
const ui::AXNode* end = GetAXNode(end_.id);
DUMP_WILL_BE_CHECK(node && end);
if (!node || !end) {
return false;
}
if (!node->IsInvisibleOrIgnored() && !end->IsInvisibleOrIgnored()) {
for (base::queue<ui::AXNode*> ancestors =
node->GetAncestorsCrossingTreeBoundaryAsQueue();
!ancestors.empty(); ancestors.pop()) {
InsertIdIfNotIgnored(ancestors.front()->id(), selection_node_ids_);
}
node = GetUnignoredParentForSelection(node);
end = GetUnignoredParentForSelection(end);
if (end) {
end = end->GetDeepestLastUnignoredDescendantCrossingTreeBoundary();
if (node && end) {
for (node = node->GetFirstUnignoredChildCrossingTreeBoundary();
node && node->CompareTo(*end).value_or(1) <= 0;
node = node->GetNextUnignoredInTreeOrder()) {
InsertIdIfNotIgnored(node->id(), selection_node_ids_);
}
}
}
}
return true;
}
bool ReadAnythingAppModel::ContentNodesOnlyContainHeadings() {
for (ui::AXNodeID node_id : content_node_ids_) {
ui::AXNode* node = GetAXNode(node_id);
if (!node || node->IsInvisibleOrIgnored() ||
node->GetRole() == ax::mojom::Role::kHeading) {
continue;
}
base::queue<ui::AXNode*> ancestors =
node->GetAncestorsCrossingTreeBoundaryAsQueue();
bool found_heading = false;
while (!ancestors.empty()) {
if (ancestors.front()->GetRole() == ax::mojom::Role::kHeading) {
found_heading = true;
break;
}
ancestors.pop();
}
if (!found_heading) {
return false;
}
}
return true;
}
void ReadAnythingAppModel::ComputeDisplayNodeIdsForDistilledTree() {
DCHECK(!content_node_ids_.empty());
if (ContentNodesOnlyContainHeadings()) {
return;
}
for (auto content_node_id : content_node_ids_) {
ui::AXNode* content_node = GetAXNode(content_node_id);
if (!content_node) {
RecordHeuristicMetric(ReadAnythingHeuristics::kNodeNotFound);
continue;
}
if (content_node->IsInvisibleOrIgnored()) {
RecordHeuristicMetric(ReadAnythingHeuristics::kInvisibleOrIgnored);
continue;
}
if (content_node->data().SupportsExpandCollapse() &&
!content_node->HasState(ax::mojom::State::kRichlyEditable)) {
if (!content_node->HasState(ax::mojom::State::kExpanded)) {
RecordHeuristicMetric(ReadAnythingHeuristics::kNotExpanded);
continue;
}
}
base::queue<ui::AXNode*> ancestors =
content_node->GetAncestorsCrossingTreeBoundaryAsQueue();
while (!ancestors.empty()) {
ui::AXNodeID ancestor_id = ancestors.front()->id();
if (base::Contains(display_node_ids_, ancestor_id)) {
break;
}
ancestors.pop();
InsertIdIfNotIgnored(ancestor_id, display_node_ids_);
}
ui::AXNode* next_node = content_node;
ui::AXNode* deepest_last_descendant =
content_node->GetDeepestLastUnignoredDescendant();
if (!deepest_last_descendant) {
RecordHeuristicMetric(ReadAnythingHeuristics::kNoDeepsetLastDecendent);
continue;
}
while (next_node != deepest_last_descendant) {
next_node = next_node->GetNextUnignoredInTreeOrder();
InsertIdIfNotIgnored(next_node->id(), display_node_ids_);
}
RecordHeuristicMetric(ReadAnythingHeuristics::kNone);
}
}
ui::AXSerializableTree* ReadAnythingAppModel::GetActiveTree() const {
return GetTreeFromId(active_tree_id_);
}
ui::AXSerializableTree* ReadAnythingAppModel::GetTreeFromId(
const ui::AXTreeID& tree_id) const {
DUMP_WILL_BE_CHECK(ContainsTree(tree_id));
DUMP_WILL_BE_CHECK(tree_id != ui::AXTreeIDUnknown());
if (!ContainsTree(tree_id) || tree_id == ui::AXTreeIDUnknown()) {
return nullptr;
}
return static_cast<ui::AXSerializableTree*>(
tree_infos_.at(tree_id)->manager->ax_tree());
}
bool ReadAnythingAppModel::ContainsTree(const ui::AXTreeID& tree_id) const {
return base::Contains(tree_infos_, tree_id);
}
bool ReadAnythingAppModel::ContainsActiveTree() const {
return ContainsTree(active_tree_id_);
}
void ReadAnythingAppModel::SetUrlInformationCallback(
base::OnceCallback<void()> callback) {
if (tree_infos_.contains(active_tree_id_) &&
tree_infos_.at(active_tree_id_)->is_url_information_set) {
std::move(callback).Run();
return;
}
set_url_information_callback_ = std::move(callback);
}
void ReadAnythingAppModel::SetTreeInfoUrlInformation(
ReadAnythingAppModel::AXTreeInfo& tree_info) {
if (tree_info.is_url_information_set) {
return;
}
CHECK(tree_info.manager);
if (!tree_info.manager->IsRoot()) {
return;
}
const ui::AXNode* const root = tree_info.manager->GetRoot();
if (!root || !root->HasStringAttribute(ax::mojom::StringAttribute::kUrl)) {
return;
}
const GURL url(root->GetStringAttribute(ax::mojom::StringAttribute::kUrl));
tree_info.is_reload =
!previous_tree_url_.empty() && (previous_tree_url_ == url.GetContent());
tree_info.is_docs = url.SchemeIsHTTPOrHTTPS() &&
(url.DomainIs("docs.google.com") ||
url.DomainIs("docs.sandbox.google.com")) &&
url.GetPath().starts_with("/document") &&
!url.ExtractFileName().empty();
tree_info.is_url_information_set = true;
previous_tree_url_ = url.GetContent();
if (!set_url_information_callback_.is_null()) {
std::move(set_url_information_callback_).Run();
}
}
bool ReadAnythingAppModel::IsDocs() const {
if (!tree_infos_.contains(active_tree_id_)) {
return false;
}
return tree_infos_.at(active_tree_id_)->is_docs;
}
bool ReadAnythingAppModel::IsReload() const {
if (!tree_infos_.contains(active_tree_id_)) {
return false;
}
return tree_infos_.at(active_tree_id_)->is_reload;
}
void ReadAnythingAppModel::AddPendingUpdates(const ui::AXTreeID& tree_id,
Updates& updates) {
pending_updates_[tree_id].emplace_back(std::move(updates));
}
void ReadAnythingAppModel::ClearPendingUpdates() {
pending_updates_.clear();
}
void ReadAnythingAppModel::UnserializePendingUpdates(
const ui::AXTreeID& tree_id) {
if (!pending_updates_.contains(tree_id)) {
VLOG(1) << "Returning early in UnserializePendingUpdates because it "
"doesn't contain tree id "
<< tree_id;
return;
}
std::vector<Updates> updates = pending_updates_.extract(tree_id).mapped();
for (const Updates& update : updates) {
DCHECK(update.empty() || tree_id == active_tree_id_);
UnserializeUpdates(update, tree_id);
}
}
void ReadAnythingAppModel::UnserializeUpdates(const Updates& updates,
const ui::AXTreeID& tree_id) {
VLOG(1) << "Unserializing updates for " << tree_id;
if (updates.empty()) {
VLOG(1) << "Unable to unserialize updates for " << tree_id
<< " because the updates are empty";
return;
}
DCHECK_NE(tree_id, ui::AXTreeIDUnknown());
const auto it = tree_infos_.find(tree_id);
DCHECK(it != tree_infos_.end());
auto* const tree =
static_cast<ui::AXSerializableTree*>(it->second->manager->ax_tree());
CHECK(tree);
ui::AXEventGenerator event_generator(tree);
const size_t prev_tree_size = tree->size();
for (const ui::AXTreeUpdate& update : updates) {
DUMP_WILL_BE_CHECK(tree->root() || update.root_id != ui::kInvalidAXNodeID);
if (!tree->root() && update.root_id == ui::kInvalidAXNodeID) {
VLOG(1) << "Skipping unserialize because the tree has no root and the "
"update has an invalid root";
return;
}
if (update.tree_data.tree_id == ui::AXTreeIDUnknown()) {
VLOG(1) << "unserializing an update with an unknown tree ID";
} else {
VLOG(1) << "Unserializing an update with a known tree ID: "
<< update.tree_data.tree_id;
}
tree->Unserialize(update);
}
SetTreeInfoUrlInformation(*it->second);
ProcessGeneratedEvents(event_generator, prev_tree_size, tree->size());
}
void ReadAnythingAppModel::AccessibilityEventReceived(
const ui::AXTreeID& tree_id,
Updates& updates,
std::vector<ui::AXEvent>& events,
bool speech_playing) {
DCHECK_NE(tree_id, ui::AXTreeIDUnknown());
VLOG(1) << "AccessibilityEventReceived for " << tree_id;
if (!ContainsTree(tree_id)) {
auto new_tree = std::make_unique<ui::AXSerializableTree>();
for (auto& observer : observers_) {
observer.OnTreeAdded(new_tree.get());
}
tree_infos_.emplace(
tree_id, std::make_unique<AXTreeInfo>(
std::make_unique<ui::AXTreeManager>(std::move(new_tree))));
if (tree_id == active_tree_id_ && pending_ukm_sources_.count(tree_id) > 0) {
ukm::SourceId ukm_source_id = pending_ukm_sources_[tree_id];
pending_ukm_sources_.erase(tree_id);
SetUkmSourceId(ukm_source_id);
}
}
if (may_use_child_for_active_tree_) {
if (root_tree_id_ == tree_id) {
SetRootTreeId(root_tree_id_);
} else if (active_tree_id_ != ui::AXTreeIDUnknown() &&
active_tree_id_ != tree_id &&
child_tree_ids_.find(tree_id) != child_tree_ids_.end()) {
VLOG(1) << "Using child to set active tree id to " << tree_id;
SetActiveTreeId(tree_id);
requires_distillation_ = true;
}
}
if (tree_id == active_tree_id_) {
if (distillation_in_progress_ || speech_playing) {
AddPendingUpdates(tree_id, updates);
ProcessNonGeneratedEvents(events);
if (timer_since_tree_changed_for_data_collection_.IsRunning()) {
CHECK(features::IsDataCollectionModeForScreen2xEnabled());
timer_since_tree_changed_for_data_collection_.Reset();
}
VLOG(1) << "Returning early in AccessibilityEventReceived because "
"distillation is in progress";
return;
}
VLOG(1) << "AccessibilityEventReceived- tree ID is the active tree";
UnserializePendingUpdates(tree_id);
UnserializeUpdates(updates, tree_id);
ProcessNonGeneratedEvents(events);
} else {
VLOG(1) << "AccessibilityEventReceived- tree ID is not the active tree";
UnserializeUpdates(updates, tree_id);
}
if (features::IsDataCollectionModeForScreen2xEnabled() && updates.size()) {
waiting_for_tree_change_timer_trigger_ = true;
timer_since_tree_changed_for_data_collection_.Start(
FROM_HERE, kTimeElapsedSinceTreeChangedForDataCollection,
base::BindRepeating(&ReadAnythingAppModel::OnTreeChangeTimerTriggered,
weak_ptr_factory_.GetWeakPtr()));
}
}
void ReadAnythingAppModel::OnAXTreeDestroyed(const ui::AXTreeID& tree_id) {
VLOG(1) << "OnAXTreeDestroyed for " << tree_id;
const auto it = tree_infos_.find(tree_id);
if (it == tree_infos_.end()) {
return;
}
if (active_tree_id_ == tree_id) {
active_tree_id_ = ui::AXTreeIDUnknown();
}
for (ui::AXTree* const ax_tree = it->second->manager->ax_tree();
auto& observer : observers_) {
observer.OnTreeRemoved(ax_tree);
}
tree_infos_.erase(it);
pending_updates_.erase(tree_id);
}
ukm::SourceId ReadAnythingAppModel::GetUkmSourceId() const {
if (base::Contains(tree_infos_, active_tree_id_)) {
ReadAnythingAppModel::AXTreeInfo* tree_info =
tree_infos_.at(active_tree_id_).get();
if (tree_info) {
return tree_info->ukm_source_id;
}
}
return ukm::kInvalidSourceId;
}
void ReadAnythingAppModel::SetUkmSourceIdForTree(const ui::AXTreeID& tree,
ukm::SourceId ukm_source_id) {
if (!base::Contains(tree_infos_, active_tree_id_)) {
pending_ukm_sources_[tree] = ukm_source_id;
return;
}
SetUkmSourceId(ukm_source_id);
}
void ReadAnythingAppModel::SetUkmSourceId(ukm::SourceId ukm_source_id) {
if (!base::Contains(tree_infos_, active_tree_id_)) {
return;
}
ReadAnythingAppModel::AXTreeInfo* tree_info =
tree_infos_.at(active_tree_id_).get();
if (!tree_info) {
return;
}
if (tree_info->ukm_source_id == ukm::kInvalidSourceId) {
tree_info->ukm_source_id = ukm_source_id;
} else {
DCHECK_EQ(tree_info->ukm_source_id, ukm_source_id);
}
}
int ReadAnythingAppModel::GetNumSelections() const {
if (base::Contains(tree_infos_, active_tree_id_)) {
ReadAnythingAppModel::AXTreeInfo* tree_info =
tree_infos_.at(active_tree_id_).get();
if (tree_info) {
return tree_info->num_selections;
}
}
return 0;
}
void ReadAnythingAppModel::SetNumSelections(int num_selections) {
if (!base::Contains(tree_infos_, active_tree_id_)) {
return;
}
ReadAnythingAppModel::AXTreeInfo* tree_info =
tree_infos_.at(active_tree_id_).get();
if (!tree_info) {
return;
}
tree_info->num_selections = num_selections;
}
ui::AXNode* ReadAnythingAppModel::GetAXNode(
const ui::AXNodeID& ax_node_id) const {
ui::AXSerializableTree* tree = GetTreeFromId(active_tree_id_);
if (!tree) {
return nullptr;
}
return tree->GetFromId(ax_node_id);
}
bool ReadAnythingAppModel::NodeIsContentNode(ui::AXNodeID ax_node_id) const {
return base::Contains(content_node_ids_, ax_node_id);
}
void ReadAnythingAppModel::AdjustTextSize(int increment) {
SetFontSize(font_size_, increment);
}
void ReadAnythingAppModel::ResetTextSize() {
SetFontSize(2.0f);
}
void ReadAnythingAppModel::OnScroll(bool on_selection,
bool from_reading_mode) const {
enum class ReadAnythingScrollEvent {
kSelectedSidePanel = 0,
kSelectedMainPanel = 1,
kScrolledSidePanel = 2,
kScrolledMainPanel = 3,
kMaxValue = kScrolledMainPanel,
};
using enum ReadAnythingScrollEvent;
ReadAnythingScrollEvent event;
if (on_selection) {
event = from_reading_mode ? kSelectedMainPanel : kSelectedSidePanel;
} else {
event = from_reading_mode ? kScrolledSidePanel : kScrolledMainPanel;
}
base::UmaHistogramEnumeration("Accessibility.ReadAnything.ScrollEvent",
event);
}
void ReadAnythingAppModel::SetRootTreeId(ui::AXTreeID root_tree_id) {
root_tree_id_ = root_tree_id;
SetActiveTreeId(root_tree_id);
may_use_child_for_active_tree_ = false;
child_tree_ids_.clear();
}
void ReadAnythingAppModel::SetActiveTreeId(ui::AXTreeID active_tree_id) {
if (active_tree_id_ != active_tree_id &&
active_tree_id_ != ui::AXTreeIDUnknown() && ContainsActiveTree()) {
UnserializePendingUpdates(active_tree_id_);
}
active_tree_id_ = std::move(active_tree_id);
if (features::IsDataCollectionModeForScreen2xEnabled()) {
timer_since_page_load_for_data_collection_.Start(
FROM_HERE, kTimeElapsedSincePageLoadForDataCollection,
base::BindOnce(&ReadAnythingAppModel::OnPageLoadTimerTriggered,
weak_ptr_factory_.GetWeakPtr()));
if (timer_since_tree_changed_for_data_collection_.IsRunning()) {
timer_since_tree_changed_for_data_collection_.Stop();
}
waiting_for_tree_change_timer_trigger_ = false;
}
}
void ReadAnythingAppModel::ProcessNonGeneratedEvents(
const std::vector<ui::AXEvent>& events) {
bool delay_screen2x_training_data_collection_ = false;
for (auto& event : events) {
#if BUILDFLAG(IS_MAC)
VLOG(2) << "Non-generated event type: " << event.event_type;
#endif
switch (event.event_type) {
case ax::mojom::Event::kLoadComplete:
requires_distillation_ = true;
page_finished_loading_ = true;
delay_screen2x_training_data_collection_ = true;
break;
case ax::mojom::Event::kLocationChanged:
delay_screen2x_training_data_collection_ = true;
break;
case ax::mojom::Event::kBlur:
if (features::IsReadAnythingReadAloudEnabled() &&
content_node_ids_.size() == 0) {
requires_distillation_ = true;
}
break;
case ax::mojom::Event::kActiveDescendantChanged:
case ax::mojom::Event::kCheckedStateChanged:
case ax::mojom::Event::kChildrenChanged:
case ax::mojom::Event::kDocumentSelectionChanged:
case ax::mojom::Event::kDocumentTitleChanged:
case ax::mojom::Event::kExpandedChanged:
case ax::mojom::Event::kRowCollapsed:
case ax::mojom::Event::kRowCountChanged:
case ax::mojom::Event::kRowExpanded:
case ax::mojom::Event::kSelectedChildrenChanged:
case ax::mojom::Event::kNone:
case ax::mojom::Event::kAlert:
case ax::mojom::Event::kAutocorrectionOccured:
case ax::mojom::Event::kClicked:
case ax::mojom::Event::kControlsChanged:
case ax::mojom::Event::kEndOfTest:
case ax::mojom::Event::kFocus:
case ax::mojom::Event::kFocusAfterMenuClose:
case ax::mojom::Event::kFocusContext:
case ax::mojom::Event::kHide:
case ax::mojom::Event::kHitTestResult:
case ax::mojom::Event::kHover:
case ax::mojom::Event::kImageFrameUpdated:
case ax::mojom::Event::kLayoutComplete:
case ax::mojom::Event::kLiveRegionCreated:
case ax::mojom::Event::kLiveRegionChanged:
case ax::mojom::Event::kLoadStart:
case ax::mojom::Event::kMediaStartedPlaying:
case ax::mojom::Event::kMediaStoppedPlaying:
case ax::mojom::Event::kMenuEnd:
case ax::mojom::Event::kMenuPopupEnd:
case ax::mojom::Event::kMenuPopupStart:
case ax::mojom::Event::kMenuStart:
case ax::mojom::Event::kMouseCanceled:
case ax::mojom::Event::kMouseDragged:
case ax::mojom::Event::kMouseMoved:
case ax::mojom::Event::kMousePressed:
case ax::mojom::Event::kMouseReleased:
case ax::mojom::Event::kScrolledToAnchor:
case ax::mojom::Event::kScrollPositionChanged:
case ax::mojom::Event::kSelection:
case ax::mojom::Event::kSelectionAdd:
case ax::mojom::Event::kSelectionRemove:
case ax::mojom::Event::kShow:
case ax::mojom::Event::kStateChanged:
case ax::mojom::Event::kTextChanged:
case ax::mojom::Event::kWindowActivated:
case ax::mojom::Event::kWindowDeactivated:
case ax::mojom::Event::kWindowVisibilityChanged:
case ax::mojom::Event::kTextSelectionChanged:
case ax::mojom::Event::kTooltipClosed:
case ax::mojom::Event::kTooltipOpened:
case ax::mojom::Event::kTreeChanged:
break;
case ax::mojom::Event::kValueChanged:
#if BUILDFLAG(IS_MAC)
if (ui::AXNode* node = GetAXNode(event.id);
node && ui::IsTextField(node->GetRole())) {
VLOG(1) << "kValueChanged on a text field";
}
#endif
if (event.event_from == ax::mojom::EventFrom::kUser &&
event.event_intents.size() > 0) {
reset_draw_timer_ = true;
#if BUILDFLAG(IS_MAC)
VLOG(1) << "kValueChanged on a user event triggering redraw timer";
#endif
} else {
#if BUILDFLAG(IS_MAC)
VLOG(1) << "kValueChanged without a redraw timer trigger";
#endif
}
break;
case ax::mojom::Event::kAriaAttributeChangedDeprecated:
case ax::mojom::Event::kMenuListValueChangedDeprecated:
NOTREACHED();
}
}
if (features::IsDataCollectionModeForScreen2xEnabled() &&
delay_screen2x_training_data_collection_) {
waiting_for_tree_change_timer_trigger_ = true;
timer_since_tree_changed_for_data_collection_.Start(
FROM_HERE, kTimeElapsedSinceTreeChangedForDataCollection,
base::BindRepeating(&ReadAnythingAppModel::OnTreeChangeTimerTriggered,
weak_ptr_factory_.GetWeakPtr()));
}
}
void ReadAnythingAppModel::ProcessGeneratedEvents(
const ui::AXEventGenerator& event_generator,
size_t prev_tree_size,
size_t tree_size) {
for (const auto& event : event_generator) {
#if BUILDFLAG(IS_MAC)
VLOG(2) << "Generated event type: " << event.event_params->event;
#endif
switch (event.event_params->event) {
case ui::AXEventGenerator::Event::DOCUMENT_SELECTION_CHANGED:
requires_post_process_selection_ = true;
break;
case ui::AXEventGenerator::Event::DOCUMENT_TITLE_CHANGED:
if (!features::IsReadAnythingReadAloudEnabled() ||
event.event_params->event_from == ax::mojom::EventFrom::kUser) {
requires_distillation_ = true;
}
break;
case ui::AXEventGenerator::Event::ALERT:
requires_distillation_ = true;
break;
case ui::AXEventGenerator::Event::SCROLL_VERTICAL_POSITION_CHANGED:
OnScroll(event.event_params->event_from_action ==
ax::mojom::Action::kSetSelection,
false);
break;
case ui::AXEventGenerator::Event::SUBTREE_CREATED:
if (is_pdf_ && prev_tree_size < tree_size) {
requires_distillation_ = true;
}
break;
case ui::AXEventGenerator::Event::COLLAPSED:
if (features::IsReadAnythingReadAloudEnabled()) {
ResetSelection();
requires_post_process_selection_ = false;
redraw_required_ = true;
}
break;
case ui::AXEventGenerator::Event::EXPANDED:
if (features::IsReadAnythingReadAloudEnabled()) {
if (base::Contains(content_node_ids_, event.node_id)) {
redraw_required_ = true;
} else {
requires_distillation_ = true;
}
}
break;
case ui::AXEventGenerator::Event::EDITABLE_TEXT_CHANGED:
case ui::AXEventGenerator::Event::NONE:
case ui::AXEventGenerator::Event::ACCESS_KEY_CHANGED:
case ui::AXEventGenerator::Event::ACTIVE_DESCENDANT_CHANGED:
case ui::AXEventGenerator::Event::ARIA_CURRENT_CHANGED:
case ui::AXEventGenerator::Event::ARIA_NOTIFICATIONS_POSTED:
case ui::AXEventGenerator::Event::ATK_TEXT_OBJECT_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::ATOMIC_CHANGED:
case ui::AXEventGenerator::Event::AUTO_COMPLETE_CHANGED:
case ui::AXEventGenerator::Event::AUTOFILL_AVAILABILITY_CHANGED:
case ui::AXEventGenerator::Event::BUSY_CHANGED:
case ui::AXEventGenerator::Event::CARET_BOUNDS_CHANGED:
case ui::AXEventGenerator::Event::CHECKED_STATE_CHANGED:
case ui::AXEventGenerator::Event::CHECKED_STATE_DESCRIPTION_CHANGED:
case ui::AXEventGenerator::Event::CHILDREN_CHANGED:
case ui::AXEventGenerator::Event::CONTROLS_CHANGED:
case ui::AXEventGenerator::Event::DEFAULT_ACTION_VERB_CHANGED:
case ui::AXEventGenerator::Event::DETAILS_CHANGED:
case ui::AXEventGenerator::Event::DESCRIBED_BY_CHANGED:
case ui::AXEventGenerator::Event::DESCRIPTION_CHANGED:
case ui::AXEventGenerator::Event::ENABLED_CHANGED:
case ui::AXEventGenerator::Event::FOCUS_CHANGED:
case ui::AXEventGenerator::Event::FLOW_FROM_CHANGED:
case ui::AXEventGenerator::Event::FLOW_TO_CHANGED:
case ui::AXEventGenerator::Event::HASPOPUP_CHANGED:
case ui::AXEventGenerator::Event::HIERARCHICAL_LEVEL_CHANGED:
case ui::AXEventGenerator::Event::IGNORED_CHANGED:
case ui::AXEventGenerator::Event::IMAGE_ANNOTATION_CHANGED:
case ui::AXEventGenerator::Event::INVALID_STATUS_CHANGED:
case ui::AXEventGenerator::Event::KEY_SHORTCUTS_CHANGED:
case ui::AXEventGenerator::Event::LABELED_BY_CHANGED:
case ui::AXEventGenerator::Event::LANGUAGE_CHANGED:
case ui::AXEventGenerator::Event::LAYOUT_INVALIDATED:
case ui::AXEventGenerator::Event::LIVE_REGION_CHANGED:
case ui::AXEventGenerator::Event::LIVE_REGION_CREATED:
case ui::AXEventGenerator::Event::LIVE_REGION_NODE_CHANGED:
case ui::AXEventGenerator::Event::LIVE_RELEVANT_CHANGED:
case ui::AXEventGenerator::Event::LIVE_STATUS_CHANGED:
case ui::AXEventGenerator::Event::MENU_ITEM_SELECTED:
case ui::AXEventGenerator::Event::MENU_POPUP_END:
case ui::AXEventGenerator::Event::MENU_POPUP_START:
case ui::AXEventGenerator::Event::MULTILINE_STATE_CHANGED:
case ui::AXEventGenerator::Event::MULTISELECTABLE_STATE_CHANGED:
break;
case ui::AXEventGenerator::Event::NAME_CHANGED:
if (!features::IsReadAnythingReadAloudEnabled() &&
last_expanded_node_id_ == event.node_id) {
ResetSelection();
requires_post_process_selection_ = false;
reset_last_expanded_node_id();
redraw_required_ = true;
}
break;
case ui::AXEventGenerator::Event::OBJECT_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::ORIENTATION_CHANGED:
case ui::AXEventGenerator::Event::PARENT_CHANGED:
case ui::AXEventGenerator::Event::PLACEHOLDER_CHANGED:
case ui::AXEventGenerator::Event::POSITION_IN_SET_CHANGED:
case ui::AXEventGenerator::Event::RANGE_VALUE_CHANGED:
case ui::AXEventGenerator::Event::RANGE_VALUE_MAX_CHANGED:
case ui::AXEventGenerator::Event::RANGE_VALUE_MIN_CHANGED:
case ui::AXEventGenerator::Event::RANGE_VALUE_STEP_CHANGED:
case ui::AXEventGenerator::Event::READONLY_CHANGED:
case ui::AXEventGenerator::Event::RELATED_NODE_CHANGED:
case ui::AXEventGenerator::Event::REQUIRED_STATE_CHANGED:
case ui::AXEventGenerator::Event::ROLE_CHANGED:
case ui::AXEventGenerator::Event::ROW_COUNT_CHANGED:
case ui::AXEventGenerator::Event::SCROLL_HORIZONTAL_POSITION_CHANGED:
case ui::AXEventGenerator::Event::SELECTED_CHANGED:
case ui::AXEventGenerator::Event::SELECTED_CHILDREN_CHANGED:
case ui::AXEventGenerator::Event::SELECTED_VALUE_CHANGED:
case ui::AXEventGenerator::Event::SET_SIZE_CHANGED:
case ui::AXEventGenerator::Event::SORT_CHANGED:
case ui::AXEventGenerator::Event::STATE_CHANGED:
case ui::AXEventGenerator::Event::TEXT_ATTRIBUTE_CHANGED:
case ui::AXEventGenerator::Event::TEXT_SELECTION_CHANGED:
case ui::AXEventGenerator::Event::VALUE_IN_TEXT_FIELD_CHANGED:
case ui::AXEventGenerator::Event::WIN_IACCESSIBLE_STATE_CHANGED:
break;
}
}
}
bool ReadAnythingAppModel::ScreenAIServiceReadyForDataCollection() const {
CHECK(features::IsDataCollectionModeForScreen2xEnabled());
return screen_ai_service_ready_for_data_collection_;
}
void ReadAnythingAppModel::SetScreenAIServiceReadyForDataCollection() {
screen_ai_service_ready_for_data_collection_ = true;
MaybeRunDataCollectionForScreen2xCallback();
}
bool ReadAnythingAppModel::PageFinishedLoadingForDataCollection() const {
CHECK(features::IsDataCollectionModeForScreen2xEnabled());
return !waiting_for_page_load_completion_timer_trigger_ &&
!waiting_for_tree_change_timer_trigger_;
}
void ReadAnythingAppModel::OnPageLoadTimerTriggered() {
CHECK(waiting_for_page_load_completion_timer_trigger_);
waiting_for_page_load_completion_timer_trigger_ = false;
MaybeRunDataCollectionForScreen2xCallback();
}
void ReadAnythingAppModel::OnTreeChangeTimerTriggered() {
CHECK(waiting_for_tree_change_timer_trigger_);
waiting_for_tree_change_timer_trigger_ = false;
MaybeRunDataCollectionForScreen2xCallback();
}
void ReadAnythingAppModel::SetDataCollectionForScreen2xCallback(
base::OnceCallback<void()> callback) {
CHECK(features::IsDataCollectionModeForScreen2xEnabled());
data_collection_for_screen2x_callback_ = std::move(callback);
}
void ReadAnythingAppModel::MaybeRunDataCollectionForScreen2xCallback() {
CHECK(features::IsDataCollectionModeForScreen2xEnabled());
if (!PageFinishedLoadingForDataCollection() ||
!ScreenAIServiceReadyForDataCollection()) {
return;
}
if (data_collection_for_screen2x_callback_.is_null()) {
LOG(ERROR) << "Callback not set or triggered more than once.";
return;
}
std::move(data_collection_for_screen2x_callback_).Run();
}
void ReadAnythingAppModel::SetBaseLanguageCode(std::string base_language_code) {
DCHECK(!base_language_code.empty());
base_language_code_ = std::move(base_language_code);
supported_fonts_ = GetSupportedFonts(base_language_code_);
}
void ReadAnythingAppModel::AddObserver(ModelObserver* observer) {
observers_.AddObserver(observer);
}
void ReadAnythingAppModel::RemoveObserver(ModelObserver* observer) {
observers_.RemoveObserver(observer);
}
void ReadAnythingAppModel::SetFontSize(double font_size, int increment) {
font_size_ = AdjustFontScale(font_size, increment);
}
const std::set<ui::AXNodeID>* ReadAnythingAppModel::GetCurrentlyVisibleNodes()
const {
return selection_node_ids_.empty() ? &display_node_ids()
: &selection_node_ids_;
}
void ReadAnythingAppModel::AllowChildTreeForActiveTree(bool use_child_tree) {
may_use_child_for_active_tree_ = use_child_tree;
if (!may_use_child_for_active_tree_) {
child_tree_ids_.clear();
}
ui::AXSerializableTree* active_tree = GetTreeFromId(active_tree_id_);
if (!active_tree) {
VLOG(1) << "Not allowing child tree for active tree because active tree is "
"null";
return;
}
std::set<ui::AXTreeID> child_ids = active_tree->GetAllChildTreeIds();
if (!child_ids.size()) {
VLOG(1) << "Not allowing child tree for active tree because active tree "
"has no child trees";
return;
}
VLOG(1) << "Allow child tree for active tree";
child_tree_ids_.insert(child_ids.begin(), child_ids.end());
}
bool ReadAnythingAppModel::SelectionNodesContainedInDistilledContent() const {
std::vector<ui::AXNodeID> sorted_content_ids = content_node_ids_;
std::sort(sorted_content_ids.begin(), sorted_content_ids.end());
return std::includes(sorted_content_ids.begin(), sorted_content_ids.end(),
selection_node_ids_.begin(), selection_node_ids_.end());
}