// Copyright 2022 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "chrome/renderer/accessibility/read_anything_app_controller.h" #include <climits> #include <memory> #include <queue> #include <string> #include <utility> #include <vector> #include "base/debug/stack_trace.h" #include "base/metrics/histogram_functions.h" #include "base/metrics/metrics_hashes.h" #include "base/notreached.h" #include "base/strings/string_util.h" #include "base/strings/to_string.h" #include "base/strings/utf_string_conversions.h" #include "chrome/common/accessibility/read_anything_constants.h" #include "chrome/renderer/accessibility/ax_tree_distiller.h" #include "chrome/renderer/accessibility/read_aloud_traversal_utils.h" #include "chrome/renderer/accessibility/read_anything_node_utils.h" #include "components/language/core/common/locale_util.h" #include "components/translate/core/common/translate_constants.h" #include "content/public/renderer/chrome_object_extensions_utils.h" #include "content/public/renderer/render_frame.h" #include "content/public/renderer/render_thread.h" #include "gin/converter.h" #include "gin/dictionary.h" #include "gin/handle.h" #include "gin/object_template_builder.h" #include "read_anything_app_controller.h" #include "services/metrics/public/cpp/mojo_ukm_recorder.h" #include "services/metrics/public/cpp/ukm_builders.h" #include "third_party/blink/public/platform/browser_interface_broker_proxy.h" #include "third_party/blink/public/platform/scheduler/web_agent_group_scheduler.h" #include "third_party/blink/public/web/web_local_frame.h" #include "third_party/blink/public/web/web_script_source.h" #include "third_party/re2/src/re2/re2.h" #include "third_party/skia/include/core/SkColorSpace.h" #include "third_party/skia/include/core/SkColorType.h" #include "third_party/skia/include/core/SkData.h" #include "third_party/skia/include/core/SkImageInfo.h" #include "ui/accessibility/accessibility_features.h" #include "ui/accessibility/ax_enums.mojom-shared.h" #include "ui/accessibility/ax_node.h" #include "ui/accessibility/ax_node_id_forward.h" #include "ui/accessibility/ax_role_properties.h" #include "ui/accessibility/ax_selection.h" #include "ui/accessibility/ax_serializable_tree.h" #include "ui/accessibility/ax_text_utils.h" #include "ui/accessibility/ax_tree.h" #include "ui/accessibility/ax_tree_id.h" #include "ui/accessibility/ax_tree_serializer.h" #include "ui/accessibility/ax_tree_update.h" #include "ui/accessibility/mojom/ax_event.mojom.h" #include "ui/base/l10n/l10n_util.h" #include "ui/gfx/geometry/size.h" #include "url/url_util.h" #include "v8/include/v8-context.h" #include "v8/include/v8-microtask-queue.h" #include "v8/include/v8-typed-array.h" namespace { constexpr char kUndeterminedLocale[] = …; // The number of seconds to wait before distilling after a user has stopped // entering text into a richly editable text field. const double kPostInputDistillSeconds = …; // The following methods convert v8::Value types to an AXTreeUpdate. This is not // a complete conversion (thus way gin::Converter<ui::AXTreeUpdate> is not used // or implemented) but just converting the bare minimum data types needed for // the ReadAnythingAppTest. void SetAXNodeDataChildIds(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataId(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataLanguage(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataName(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataRole(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataHtmlTag(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataDisplay(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataTextDirection(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataTextStyle(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetAXNodeDataUrl(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXNodeData* ax_node_data) { … } void SetSelectionAnchorObjectId(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXTreeData* ax_tree_data) { … } void SetSelectionFocusObjectId(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXTreeData* ax_tree_data) { … } void SetSelectionAnchorOffset(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXTreeData* ax_tree_data) { … } void SetSelectionFocusOffset(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXTreeData* ax_tree_data) { … } void SetSelectionIsBackward(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXTreeData* ax_tree_data) { … } void SetAXTreeUpdateRootId(v8::Isolate* isolate, gin::Dictionary* v8_dict, ui::AXTreeUpdate* snapshot) { … } ui::AXTreeUpdate GetSnapshotFromV8SnapshotLite( v8::Isolate* isolate, v8::Local<v8::Value> v8_snapshot_lite) { … } SkBitmap CorrectColorOfBitMap(SkBitmap& originalBitmap) { … } } // namespace // static gin::WrapperInfo ReadAnythingAppController::kWrapperInfo = …; // static ReadAnythingAppController* ReadAnythingAppController::Install( content::RenderFrame* render_frame) { … } ReadAnythingAppController::ReadAnythingAppController( content::RenderFrame* render_frame) : … { … } ReadAnythingAppController::~ReadAnythingAppController() { … } void ReadAnythingAppController::OnDestruct() { … } void ReadAnythingAppController::OnNodeDataChanged( ui::AXTree* tree, const ui::AXNodeData& old_node_data, const ui::AXNodeData& new_node_data) { … } void ReadAnythingAppController::OnNodeWillBeDeleted(ui::AXTree* tree, ui::AXNode* node) { … } void ReadAnythingAppController::OnNodeDeleted(ui::AXTree* tree, ui::AXNodeID node_id) { … } void ReadAnythingAppController::AccessibilityEventReceived( const ui::AXTreeID& tree_id, const std::vector<ui::AXTreeUpdate>& updates, const std::vector<ui::AXEvent>& events) { … } void ReadAnythingAppController::ExecuteJavaScript(const std::string& script) { … } void ReadAnythingAppController::OnActiveAXTreeIDChanged( const ui::AXTreeID& tree_id, ukm::SourceId ukm_source_id, bool is_pdf) { … } void ReadAnythingAppController::RecordNumSelections() { … } void ReadAnythingAppController::OnAXTreeDestroyed(const ui::AXTreeID& tree_id) { … } void ReadAnythingAppController::Distill() { … } void ReadAnythingAppController::OnAXTreeDistilled( const ui::AXTreeID& tree_id, const std::vector<ui::AXNodeID>& content_node_ids) { … } bool ReadAnythingAppController::PostProcessSelection() { … } void ReadAnythingAppController::Draw(bool recompute_display_nodes) { … } void ReadAnythingAppController::DrawSelection() { … } void ReadAnythingAppController::OnSettingsRestoredFromPrefs( read_anything::mojom::LineSpacing line_spacing, read_anything::mojom::LetterSpacing letter_spacing, const std::string& font, double font_size, bool links_enabled, bool images_enabled, read_anything::mojom::Colors color, double speech_rate, base::Value::Dict voices, base::Value::List languages_enabled_in_pref, read_anything::mojom::HighlightGranularity granularity) { … } void ReadAnythingAppController::ScreenAIServiceReady() { … } gin::ObjectTemplateBuilder ReadAnythingAppController::GetObjectTemplateBuilder( v8::Isolate* isolate) { … } ui::AXNodeID ReadAnythingAppController::RootId() const { … } ui::AXNodeID ReadAnythingAppController::StartNodeId() const { … } int ReadAnythingAppController::StartOffset() const { … } ui::AXNodeID ReadAnythingAppController::EndNodeId() const { … } int ReadAnythingAppController::EndOffset() const { … } std::string ReadAnythingAppController::FontName() const { … } float ReadAnythingAppController::FontSize() const { … } bool ReadAnythingAppController::LinksEnabled() const { … } bool ReadAnythingAppController::ImagesEnabled() const { … } bool ReadAnythingAppController::ImagesFeatureEnabled() const { … } int ReadAnythingAppController::LetterSpacing() const { … } int ReadAnythingAppController::LineSpacing() const { … } int ReadAnythingAppController::ColorTheme() const { … } double ReadAnythingAppController::SpeechRate() const { … } std::string ReadAnythingAppController::GetStoredVoice() const { … } std::vector<std::string> ReadAnythingAppController::GetLanguagesEnabledInPref() const { … } int ReadAnythingAppController::HighlightGranularity() const { … } int ReadAnythingAppController::StandardLineSpacing() const { … } int ReadAnythingAppController::LooseLineSpacing() const { … } int ReadAnythingAppController::VeryLooseLineSpacing() const { … } int ReadAnythingAppController::StandardLetterSpacing() const { … } int ReadAnythingAppController::WideLetterSpacing() const { … } int ReadAnythingAppController::VeryWideLetterSpacing() const { … } int ReadAnythingAppController::DefaultTheme() const { … } int ReadAnythingAppController::LightTheme() const { … } int ReadAnythingAppController::DarkTheme() const { … } int ReadAnythingAppController::YellowTheme() const { … } int ReadAnythingAppController::BlueTheme() const { … } bool ReadAnythingAppController::IsHighlightOn() { … } std::vector<ui::AXNodeID> ReadAnythingAppController::GetChildren( ui::AXNodeID ax_node_id) const { … } std::string ReadAnythingAppController::GetDataFontCss( ui::AXNodeID ax_node_id) const { … } std::string ReadAnythingAppController::GetHtmlTag( ui::AXNodeID ax_node_id) const { … } std::string ReadAnythingAppController::GetLanguage( ui::AXNodeID ax_node_id) const { … } std::u16string ReadAnythingAppController::GetTextContent( ui::AXNodeID ax_node_id) const { … } std::string ReadAnythingAppController::GetTextDirection( ui::AXNodeID ax_node_id) const { … } std::string ReadAnythingAppController::GetUrl(ui::AXNodeID ax_node_id) const { … } void ReadAnythingAppController::SendGetVoicePackInfoRequest( const std::string& language) const { … } void ReadAnythingAppController::OnGetVoicePackInfoResponse( read_anything::mojom::VoicePackInfoPtr voice_pack_info) { … } void ReadAnythingAppController::SendInstallVoicePackRequest( const std::string& language) const { … } void ReadAnythingAppController::OnInstallVoicePackResponse( read_anything::mojom::VoicePackInfoPtr voice_pack_info) { … } std::string ReadAnythingAppController::GetAltText( ui::AXNodeID ax_node_id) const { … } bool ReadAnythingAppController::ShouldBold(ui::AXNodeID ax_node_id) const { … } bool ReadAnythingAppController::IsOverline(ui::AXNodeID ax_node_id) const { … } bool ReadAnythingAppController::IsLeafNode(ui::AXNodeID ax_node_id) const { … } bool ReadAnythingAppController::IsReadAloudEnabled() const { … } bool ReadAnythingAppController::IsChromeOsAsh() const { … } bool ReadAnythingAppController::IsAutoVoiceSwitchingEnabled() const { … } bool ReadAnythingAppController::IsLanguagePackDownloadingEnabled() const { … } bool ReadAnythingAppController::IsAutomaticWordHighlightingEnabled() const { … } bool ReadAnythingAppController::IsGoogleDocs() const { … } std::vector<std::string> ReadAnythingAppController::GetSupportedFonts() { … } std::string ReadAnythingAppController::GetValidatedFontName( const std::string& font) const { … } std::vector<std::string> ReadAnythingAppController::GetAllFonts() { … } void ReadAnythingAppController::RequestImageDataUrl( ui::AXNodeID node_id) const { … } void ReadAnythingAppController::OnImageDataDownloaded( const ui::AXTreeID& tree_id, ui::AXNodeID node_id, const SkBitmap& image) { … } v8::Local<v8::Value> ReadAnythingAppController::GetImageBitmap( ui::AXNodeID node_id) { … } std::string ReadAnythingAppController::GetImageDataUrl( ui::AXNodeID node_id) const { … } const std::string ReadAnythingAppController::GetDisplayNameForLocale( const std::string& locale, const std::string& display_locale) const { … } const std::string& ReadAnythingAppController::GetLanguageCodeForSpeech() const { … } bool ReadAnythingAppController::RequiresDistillation() { … } const std::string& ReadAnythingAppController::GetDefaultLanguageCodeForSpeech() const { … } void ReadAnythingAppController::OnConnected() { … } void ReadAnythingAppController::OnCopy() const { … } void ReadAnythingAppController::OnFontSizeChanged(bool increase) { … } void ReadAnythingAppController::OnFontSizeReset() { … } void ReadAnythingAppController::OnLinksEnabledToggled() { … } void ReadAnythingAppController::OnImagesEnabledToggled() { … } void ReadAnythingAppController::OnScroll(bool on_selection) const { … } void ReadAnythingAppController::OnLinkClicked(ui::AXNodeID ax_node_id) const { … } void ReadAnythingAppController::OnLetterSpacingChange(int value) { … } void ReadAnythingAppController::OnLineSpacingChange(int value) { … } void ReadAnythingAppController::OnThemeChange(int value) { … } void ReadAnythingAppController::OnFontChange(const std::string& font) { … } void ReadAnythingAppController::OnSpeechRateChange(double rate) { … } void ReadAnythingAppController::OnVoiceChange(const std::string& voice, const std::string& lang) { … } void ReadAnythingAppController::OnLanguagePrefChange(const std::string& lang, bool enabled) { … } void ReadAnythingAppController::TurnedHighlightOn() { … } void ReadAnythingAppController::TurnedHighlightOff() { … } double ReadAnythingAppController::GetLineSpacingValue(int line_spacing) const { … } double ReadAnythingAppController::GetLetterSpacingValue( int letter_spacing) const { … } void ReadAnythingAppController::OnSelectionChange(ui::AXNodeID anchor_node_id, int anchor_offset, ui::AXNodeID focus_node_id, int focus_offset) const { … } void ReadAnythingAppController::OnCollapseSelection() const { … } void ReadAnythingAppController::ResetGranularityIndex() { … } void ReadAnythingAppController::InitAXPositionWithNode( const ui::AXNodeID& starting_node_id) { … } std::vector<ui::AXNodeID> ReadAnythingAppController::GetCurrentText() { … } void ReadAnythingAppController::PreprocessTextForSpeech() { … } void ReadAnythingAppController::MovePositionToNextGranularity() { … } void ReadAnythingAppController::MovePositionToPreviousGranularity() { … } int ReadAnythingAppController::GetCurrentTextStartIndex(ui::AXNodeID node_id) { … } int ReadAnythingAppController::GetCurrentTextEndIndex(ui::AXNodeID node_id) { … } void ReadAnythingAppController::SetLanguageForTesting( const std::string& language_code) { … } void ReadAnythingAppController::SetLanguageCode(const std::string& code) { … } #if BUILDFLAG(IS_CHROMEOS_ASH) void ReadAnythingAppController::OnDeviceLocked() { // Signal to the WebUI that the device has been locked. We'll only receive // this callback on ChromeOS. ExecuteJavaScript("chrome.readingMode.onLockScreen();"); } #endif void ReadAnythingAppController::SetDefaultLanguageCode( const std::string& code) { … } void ReadAnythingAppController::SetContentForTesting( v8::Local<v8::Value> v8_snapshot_lite, std::vector<ui::AXNodeID> content_node_ids) { … } void ReadAnythingAppController::ShouldShowUI() { … } void ReadAnythingAppController::OnSpeechPlayingStateChanged( bool is_speech_active) { … } int ReadAnythingAppController::GetAccessibleBoundary(const std::u16string& text, int max_text_length) { … } v8::Local<v8::Value> ReadAnythingAppController::GetHighlightForCurrentSegmentIndex(int index) { … } void ReadAnythingAppController::IncrementMetricCount( const std::string& metric) { … } void ReadAnythingAppController::OnScrolledToBottom() { … } bool ReadAnythingAppController::IsDocsLoadMoreButtonVisible() const { … }
Codebase Browser
chromium