[ATP] Separate AutomationClient from Automation
This matches the current separation in the extensions process and will make it easier to implement Automation in ATP. Since AutomationClient now needs to be bound from Javascript, adds automation.js as a shim in the AX service V8, and adds method getDesktop which calls AutomationClient::Enable. To support getDesktop, copies most of automation_node.js from the extensions system into automation.js in services/, so that we have a full chrome.automation.getDesktop that matches the implementation in extensions. Note that most functionality in automation.js still needs to have unittests added, and we'll also need to figure out how to run existing extension JS tests against ATP. Since this is not released, testing this functionality is left as a TODO. Bug: b/262638176 AX-Relnotes: N/A Test: New Change-Id: I1936360e292fc73fbe72521588713214894c6f03 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5112629 Reviewed-by: Anastasia Helfinstein <anastasi@google.com> Reviewed-by: Emily Stark <estark@chromium.org> Commit-Queue: Katie Dektar <katie@chromium.org> Cr-Commit-Position: refs/heads/main@{#1238108}
This commit is contained in:

committed by
Chromium LUCI CQ

parent
374486eb46
commit
01bbdfd219
chrome/browser/ash/accessibility/service
accessibility_service_client.ccaccessibility_service_client.hautomation_client_impl.ccautomation_client_impl.hfake_accessibility_service.ccfake_accessibility_service.h
extensions/renderer/resources/automation
services/accessibility
BUILD.gnassistive_technology_controller_impl.ccassistive_technology_controller_impl.hassistive_technology_controller_impl_unittest.ccautomation_impl.hfake_service_client.ccfake_service_client.h
features
atp_js_api_test.ccautomation_client_interface_binder.ccautomation_client_interface_binder.hautomation_internal_bindings.ccautomation_internal_bindings.hautomation_internal_bindings_unittest.cc
javascript
v8_manager.ccv8_manager.hpublic
@ -62,9 +62,13 @@ AccessibilityServiceClient::~AccessibilityServiceClient() {
|
||||
}
|
||||
|
||||
void AccessibilityServiceClient::BindAutomation(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation,
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation) {
|
||||
automation_client_->BindAutomation(std::move(automation));
|
||||
}
|
||||
|
||||
void AccessibilityServiceClient::BindAutomationClient(
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client) {
|
||||
automation_client_->Bind(std::move(automation), std::move(automation_client));
|
||||
automation_client_->BindAutomationClient(std::move(automation_client));
|
||||
}
|
||||
|
||||
void AccessibilityServiceClient::BindAutoclickClient(
|
||||
@ -187,6 +191,13 @@ void AccessibilityServiceClient::EnableAssistiveTechnology(
|
||||
AccessibilityManager::Get()->RemoveFocusRings(type);
|
||||
}
|
||||
|
||||
// If nothing at all is enabled, ensure that automation gets disabled,
|
||||
// which will keep the system from collecting and passing a11y trees.
|
||||
// Note it is safe to call Disable multiple times in a row.
|
||||
if (enabled_features_.empty()) {
|
||||
automation_client_->Disable();
|
||||
}
|
||||
|
||||
if (!enabled && !at_controller_.is_bound()) {
|
||||
// No need to launch the service, nothing is enabled.
|
||||
return;
|
||||
|
@ -51,9 +51,9 @@ class AccessibilityServiceClient : public ax::mojom::AccessibilityServiceClient,
|
||||
|
||||
// ax::mojom::AccessibilityServiceClient:
|
||||
void BindAutomation(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation,
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client)
|
||||
override;
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation) override;
|
||||
void BindAutomationClient(mojo::PendingReceiver<ax::mojom::AutomationClient>
|
||||
automation_client) override;
|
||||
void BindAutoclickClient(mojo::PendingReceiver<ax::mojom::AutoclickClient>
|
||||
autoclick_receiver) override;
|
||||
void BindSpeechRecognition(
|
||||
|
@ -20,9 +20,8 @@ AutomationClientImpl::~AutomationClientImpl() {
|
||||
nullptr);
|
||||
}
|
||||
|
||||
void AutomationClientImpl::Bind(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation,
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client) {
|
||||
void AutomationClientImpl::BindAutomation(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation) {
|
||||
// Launches the service if it wasn't running yet.
|
||||
// Development note (crbug.com/1355633): Using the remote router means
|
||||
// extensions don't get a11y events when AutomationClientImpl is bound, so
|
||||
@ -34,6 +33,10 @@ void AutomationClientImpl::Bind(
|
||||
this);
|
||||
}
|
||||
automation_remotes_.Add(std::move(automation));
|
||||
}
|
||||
|
||||
void AutomationClientImpl::BindAutomationClient(
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client) {
|
||||
automation_client_receivers_.Add(this, std::move(automation_client));
|
||||
}
|
||||
|
||||
|
@ -10,6 +10,7 @@
|
||||
#include "mojo/public/cpp/bindings/receiver_set.h"
|
||||
#include "mojo/public/cpp/bindings/remote_set.h"
|
||||
#include "services/accessibility/public/mojom/automation.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation_client.mojom.h"
|
||||
|
||||
namespace ash {
|
||||
|
||||
@ -23,19 +24,21 @@ class AutomationClientImpl : public ax::mojom::AutomationClient,
|
||||
AutomationClientImpl& operator=(const AutomationClientImpl&) = delete;
|
||||
~AutomationClientImpl() override;
|
||||
|
||||
void Bind(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation,
|
||||
void BindAutomation(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation);
|
||||
void BindAutomationClient(
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client);
|
||||
|
||||
void Disable();
|
||||
|
||||
private:
|
||||
friend class AccessibilityServiceClientTest;
|
||||
|
||||
// The following are called by the Accessibility service, passing information
|
||||
// back to the OS.
|
||||
// ax::mojom::AutomationClient:
|
||||
void Enable(EnableCallback callback) override;
|
||||
// TODO(crbug.com/1355633): Override from ax::mojom::AutomationClient:
|
||||
using EnableCallback = base::OnceCallback<void(const ui::AXTreeID&)>;
|
||||
void Enable(EnableCallback callback);
|
||||
void Disable();
|
||||
void EnableTree(const ui::AXTreeID& tree_id);
|
||||
void PerformAction(const ui::AXActionData& data);
|
||||
|
||||
|
@ -52,13 +52,14 @@ void FakeAccessibilityService::BindAnotherAutomation() {
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation_remote;
|
||||
automation_receivers_.Add(
|
||||
this, automation_remote.InitWithNewEndpointAndPassReceiver());
|
||||
accessibility_service_client_remote_->BindAutomation(
|
||||
std::move(automation_remote));
|
||||
}
|
||||
|
||||
void FakeAccessibilityService::BindAnotherAutomationClient() {
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client_receiver;
|
||||
automation_client_remotes_.Add(
|
||||
automation_client_receiver.InitWithNewPipeAndPassRemote());
|
||||
|
||||
accessibility_service_client_remote_->BindAutomation(
|
||||
std::move(automation_remote), std::move(automation_client_receiver));
|
||||
}
|
||||
|
||||
void FakeAccessibilityService::BindAnotherSpeechRecognition() {
|
||||
|
@ -17,8 +17,8 @@
|
||||
#include "services/accessibility/public/mojom/accessibility_service.mojom.h"
|
||||
#include "services/accessibility/public/mojom/autoclick.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation_client.mojom.h"
|
||||
#include "services/accessibility/public/mojom/file_loader.mojom.h"
|
||||
#include "services/accessibility/public/mojom/speech_recognition.mojom-forward.h"
|
||||
#include "services/accessibility/public/mojom/speech_recognition.mojom.h"
|
||||
#include "services/accessibility/public/mojom/tts.mojom.h"
|
||||
#include "services/accessibility/public/mojom/user_input.mojom.h"
|
||||
@ -101,6 +101,7 @@ class FakeAccessibilityService
|
||||
// Allows tests to bind APIs multiple times, mimicking multiple
|
||||
// V8 instances in the service.
|
||||
void BindAnotherAutomation();
|
||||
void BindAnotherAutomationClient();
|
||||
void BindAnotherAutoclickClient();
|
||||
void BindAnotherSpeechRecognition();
|
||||
void BindAnotherTts();
|
||||
|
@ -267,8 +267,7 @@ const GetUnclippedLocation = natives.GetUnclippedLocation;
|
||||
* array if this node has no text content, or undefined if the tree or node
|
||||
* was not found.
|
||||
*/
|
||||
const GetLineStartOffsets =
|
||||
requireNative('automationInternal').GetLineStartOffsets;
|
||||
const GetLineStartOffsets = natives.GetLineStartOffsets;
|
||||
|
||||
/**
|
||||
* @param {string} axTreeID The id of the accessibility tree.
|
||||
|
@ -25,6 +25,8 @@ source_set("lib") {
|
||||
"assistive_technology_controller_impl.h",
|
||||
"features/autoclick_client_interface_binder.cc",
|
||||
"features/autoclick_client_interface_binder.h",
|
||||
"features/automation_client_interface_binder.cc",
|
||||
"features/automation_client_interface_binder.h",
|
||||
"features/automation_internal_bindings.cc",
|
||||
"features/automation_internal_bindings.h",
|
||||
"features/bindings_isolate_holder.cc",
|
||||
|
@ -43,10 +43,14 @@ void AssistiveTechnologyControllerImpl::BindAccessibilityServiceClient(
|
||||
}
|
||||
|
||||
void AssistiveTechnologyControllerImpl::BindAutomation(
|
||||
mojo::PendingAssociatedRemote<mojom::Automation> automation,
|
||||
mojo::PendingAssociatedRemote<mojom::Automation> automation) {
|
||||
accessibility_service_client_remote_->BindAutomation(std::move(automation));
|
||||
}
|
||||
|
||||
void AssistiveTechnologyControllerImpl::BindAutomationClient(
|
||||
mojo::PendingReceiver<mojom::AutomationClient> automation_client) {
|
||||
accessibility_service_client_remote_->BindAutomation(
|
||||
std::move(automation), std::move(automation_client));
|
||||
accessibility_service_client_remote_->BindAutomationClient(
|
||||
std::move(automation_client));
|
||||
}
|
||||
|
||||
void AssistiveTechnologyControllerImpl::BindAutoclickClient(
|
||||
@ -151,11 +155,8 @@ void AssistiveTechnologyControllerImpl::CreateV8ManagerForTypeIfNoneExists(
|
||||
// TODO(b/262637071): Create a easy way to map AT types to APIs needed instead
|
||||
// of these large if statements.
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation;
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client;
|
||||
BindAutomation(automation.InitWithNewEndpointAndPassRemote(),
|
||||
automation_client.InitWithNewPipeAndPassReceiver());
|
||||
manager.ConfigureAutomation(std::move(automation),
|
||||
std::move(automation_client));
|
||||
BindAutomation(automation.InitWithNewEndpointAndPassRemote());
|
||||
manager.ConfigureAutomation(this, std::move(automation));
|
||||
manager.ConfigureFileLoader(&file_loader_remote_);
|
||||
if (type == mojom::AssistiveTechnologyType::kChromeVox ||
|
||||
type == mojom::AssistiveTechnologyType::kDictation) {
|
||||
|
@ -49,9 +49,9 @@ class AssistiveTechnologyControllerImpl
|
||||
// mojo interfaces in the OS.
|
||||
// mojom::AccessibilityServiceClient:
|
||||
void BindAutomation(
|
||||
mojo::PendingAssociatedRemote<mojom::Automation> automation,
|
||||
mojo::PendingReceiver<mojom::AutomationClient> automation_client)
|
||||
override;
|
||||
mojo::PendingAssociatedRemote<mojom::Automation> automation) override;
|
||||
void BindAutomationClient(mojo::PendingReceiver<mojom::AutomationClient>
|
||||
automation_client) override;
|
||||
void BindAutoclickClient(
|
||||
mojo::PendingReceiver<mojom::AutoclickClient> autoclick_client) override;
|
||||
void BindSpeechRecognition(
|
||||
|
@ -139,13 +139,13 @@ TEST_F(AssistiveTechnologyControllerTest,
|
||||
enabled_features.emplace_back(mojom::AssistiveTechnologyType::kChromeVox);
|
||||
at_controller_->EnableAssistiveTechnology(enabled_features);
|
||||
base::RunLoop script_waiter;
|
||||
// This script will not compile if chrome.automation.GetFocus() is not found
|
||||
// in V8, causing the test to crash.
|
||||
// This script will not compile if nativeAutomationInternal.GetFocus() is not
|
||||
// found in V8, causing the test to crash.
|
||||
// TODO(crbug.com/1355633): After adding mojom for automation, we can
|
||||
// start passing a11y events to V8 and then ensuring calling these methods
|
||||
// changes the underlying accessibility info.
|
||||
std::string script = R"JS(
|
||||
chrome.automation.GetFocus();
|
||||
nativeAutomationInternal.GetFocus();
|
||||
)JS";
|
||||
at_controller_->RunScriptForTest(mojom::AssistiveTechnologyType::kChromeVox,
|
||||
script, script_waiter.QuitClosure());
|
||||
|
@ -10,6 +10,7 @@
|
||||
#include "mojo/public/cpp/bindings/receiver_set.h"
|
||||
#include "mojo/public/cpp/bindings/remote_set.h"
|
||||
#include "services/accessibility/public/mojom/automation.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation_client.mojom.h"
|
||||
#include "ui/accessibility/ax_event.h"
|
||||
#include "ui/accessibility/ax_node_id_forward.h"
|
||||
#include "ui/accessibility/ax_relative_bounds.h"
|
||||
|
@ -7,23 +7,39 @@
|
||||
#include "base/functional/callback_forward.h"
|
||||
#include "base/notreached.h"
|
||||
#include "mojo/public/cpp/bindings/pending_associated_remote.h"
|
||||
#include "ui/accessibility/ax_tree_id.h"
|
||||
|
||||
namespace ax {
|
||||
FakeServiceClient::FakeServiceClient(mojom::AccessibilityService* service)
|
||||
: service_(service) {}
|
||||
: service_(service) {
|
||||
desktop_tree_id_ = ui::AXTreeID::CreateNewAXTreeID();
|
||||
}
|
||||
|
||||
FakeServiceClient::~FakeServiceClient() = default;
|
||||
|
||||
void FakeServiceClient::BindAutomation(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation,
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client) {
|
||||
automation_client_receivers_.Add(this, std::move(automation_client));
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation) {
|
||||
automation_remotes_.Add(std::move(automation));
|
||||
if (automation_bound_closure_) {
|
||||
std::move(automation_bound_closure_).Run();
|
||||
}
|
||||
}
|
||||
|
||||
void FakeServiceClient::BindAutomationClient(
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client) {
|
||||
automation_client_receivers_.Add(this, std::move(automation_client));
|
||||
}
|
||||
|
||||
void FakeServiceClient::Enable(EnableCallback callback) {
|
||||
std::move(callback).Run(desktop_tree_id_);
|
||||
}
|
||||
|
||||
void FakeServiceClient::Disable() {}
|
||||
|
||||
void FakeServiceClient::EnableTree(const ui::AXTreeID& tree_id) {}
|
||||
|
||||
void FakeServiceClient::PerformAction(const ui::AXActionData& data) {}
|
||||
|
||||
#if BUILDFLAG(SUPPORTS_OS_ACCESSIBILITY_SERVICE)
|
||||
void FakeServiceClient::BindAutoclickClient(
|
||||
mojo::PendingReceiver<ax::mojom::AutoclickClient>
|
||||
@ -335,6 +351,17 @@ FakeServiceClient::GetFocusRingsForType(
|
||||
return focus_rings_for_type_.at(type);
|
||||
}
|
||||
|
||||
void FakeServiceClient::SendAccessibilityEvents(
|
||||
const ui::AXTreeID& tree_id,
|
||||
const std::vector<ui::AXTreeUpdate>& updates,
|
||||
const gfx::Point& mouse_location,
|
||||
const std::vector<ui::AXEvent>& events) {
|
||||
for (auto& remote : automation_remotes_) {
|
||||
remote->DispatchAccessibilityEvents(tree_id, updates, mouse_location,
|
||||
events);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // BUILDFLAG(SUPPORTS_OS_ACCESSIBILITY_SERVICE)
|
||||
|
||||
bool FakeServiceClient::AccessibilityServiceClientIsBound() const {
|
||||
|
@ -16,6 +16,8 @@
|
||||
#include "services/accessibility/buildflags.h"
|
||||
#include "services/accessibility/public/mojom/accessibility_service.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation_client.mojom.h"
|
||||
#include "ui/accessibility/ax_tree_id.h"
|
||||
|
||||
#if BUILDFLAG(SUPPORTS_OS_ACCESSIBILITY_SERVICE)
|
||||
#include "services/accessibility/public/mojom/autoclick.mojom.h"
|
||||
@ -52,9 +54,16 @@ class FakeServiceClient : public mojom::AccessibilityServiceClient,
|
||||
|
||||
// ax::mojom::AccessibilityServiceClient:
|
||||
void BindAutomation(
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation,
|
||||
mojo::PendingReceiver<ax::mojom::AutomationClient> automation_client)
|
||||
override;
|
||||
mojo::PendingAssociatedRemote<ax::mojom::Automation> automation) override;
|
||||
void BindAutomationClient(mojo::PendingReceiver<ax::mojom::AutomationClient>
|
||||
automation_client) override;
|
||||
|
||||
// ax::mojom::AutomationClient:
|
||||
void Enable(EnableCallback callback) override;
|
||||
void Disable();
|
||||
void EnableTree(const ui::AXTreeID& tree_id);
|
||||
void PerformAction(const ui::AXActionData& data);
|
||||
|
||||
#if BUILDFLAG(SUPPORTS_OS_ACCESSIBILITY_SERVICE)
|
||||
void BindAccessibilityFileLoader(
|
||||
mojo::PendingReceiver<ax::mojom::AccessibilityFileLoader>
|
||||
@ -149,6 +158,13 @@ class FakeServiceClient : public mojom::AccessibilityServiceClient,
|
||||
SkColor color)> callback);
|
||||
void SetVirtualKeyboardVisibleCallback(
|
||||
base::RepeatingCallback<void(bool is_visible)> callback);
|
||||
|
||||
const ui::AXTreeID& desktop_tree_id() const { return desktop_tree_id_; }
|
||||
void SendAccessibilityEvents(const ui::AXTreeID& tree_id,
|
||||
const std::vector<ui::AXTreeUpdate>& updates,
|
||||
const gfx::Point& mouse_location,
|
||||
const std::vector<ui::AXEvent>& events);
|
||||
|
||||
#endif // BUILDFLAG(SUPPORTS_OS_ACCESSIBILITY_SERVICE)
|
||||
base::WeakPtr<FakeServiceClient> GetWeakPtr() {
|
||||
return weak_ptr_factory_.GetWeakPtr();
|
||||
@ -160,6 +176,8 @@ class FakeServiceClient : public mojom::AccessibilityServiceClient,
|
||||
|
||||
mojo::AssociatedRemoteSet<mojom::Automation> automation_remotes_;
|
||||
mojo::ReceiverSet<mojom::AutomationClient> automation_client_receivers_;
|
||||
|
||||
ui::AXTreeID desktop_tree_id_;
|
||||
#if BUILDFLAG(SUPPORTS_OS_ACCESSIBILITY_SERVICE)
|
||||
mojo::ReceiverSet<ax::mojom::AutoclickClient> autoclick_client_recievers_;
|
||||
mojo::Remote<ax::mojom::Autoclick> autoclick_remote_;
|
||||
|
@ -7,6 +7,7 @@
|
||||
#include "base/files/file_util.h"
|
||||
#include "base/memory/raw_ptr.h"
|
||||
#include "base/path_service.h"
|
||||
#include "base/strings/stringprintf.h"
|
||||
#include "base/test/bind.h"
|
||||
#include "base/test/task_environment.h"
|
||||
#include "build/build_config.h"
|
||||
@ -21,6 +22,7 @@
|
||||
#include "services/accessibility/public/mojom/user_interface.mojom-shared.h"
|
||||
#include "services/accessibility/public/mojom/user_interface.mojom.h"
|
||||
#include "testing/gtest/include/gtest/gtest.h"
|
||||
#include "ui/accessibility/ax_tree_id.h"
|
||||
#include "ui/events/event_constants.h"
|
||||
#include "ui/events/keycodes/keyboard_codes.h"
|
||||
#include "ui/events/mojom/event_constants.mojom-shared.h"
|
||||
@ -1106,4 +1108,98 @@ TEST_F(SpeechRecognitionJSApiTest, StopError) {
|
||||
WaitForJSTestComplete();
|
||||
}
|
||||
|
||||
class AutomationJSApiTest : public AtpJSApiTest {
|
||||
public:
|
||||
AutomationJSApiTest() = default;
|
||||
AutomationJSApiTest(const AutomationJSApiTest&) = delete;
|
||||
AutomationJSApiTest& operator=(const AutomationJSApiTest&) = delete;
|
||||
~AutomationJSApiTest() override = default;
|
||||
|
||||
mojom::AssistiveTechnologyType GetATTypeForTest() const override {
|
||||
return mojom::AssistiveTechnologyType::kAutoClick;
|
||||
}
|
||||
|
||||
const std::vector<std::string> GetJSFilePathsToLoad() const override {
|
||||
// TODO(b:266856702): Eventually ATP will load its own JS instead of us
|
||||
// doing it in the test. Right now the service doesn't have enough
|
||||
// permissions so we load support JS within the test.
|
||||
return std::vector<std::string>{
|
||||
"services/accessibility/features/mojo/test/mojom_test_support.js",
|
||||
"ui/gfx/geometry/mojom/geometry.mojom-lite.js",
|
||||
"mojo/public/mojom/base/unguessable_token.mojom-lite.js",
|
||||
"ui/accessibility/ax_enums.mojom-lite.js",
|
||||
"ui/accessibility/mojom/ax_tree_id.mojom-lite.js",
|
||||
"ui/accessibility/mojom/ax_action_data.mojom-lite.js",
|
||||
"services/accessibility/public/mojom/automation_client.mojom-lite.js",
|
||||
"services/accessibility/features/javascript/automation.js",
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Ensures chrome.automation.getDesktop exists and returns something.
|
||||
// Note that there are no tree updates so properties of the desktop object
|
||||
// can't yet be calculated.
|
||||
TEST_F(AutomationJSApiTest, GetDesktop) {
|
||||
ExecuteJS(R"JS(
|
||||
const remote = axtest.mojom.TestBindingInterface.getRemote();
|
||||
chrome.automation.getDesktop(desktop => {
|
||||
remote.testComplete(/*success=*/desktop !== null && desktop.isRootNode);
|
||||
});
|
||||
)JS");
|
||||
WaitForJSTestComplete();
|
||||
}
|
||||
|
||||
// Ensures chrome.automation.getFocus exists and gets the correct node.
|
||||
TEST_F(AutomationJSApiTest, GetFocus) {
|
||||
std::vector<ui::AXTreeUpdate> updates;
|
||||
updates.emplace_back();
|
||||
auto& tree_update = updates.back();
|
||||
tree_update.has_tree_data = true;
|
||||
tree_update.root_id = 1;
|
||||
auto& tree_data = tree_update.tree_data;
|
||||
tree_data.tree_id = client_->desktop_tree_id();
|
||||
tree_data.focus_id = 2;
|
||||
tree_update.nodes.emplace_back();
|
||||
auto& node_data1 = tree_update.nodes.back();
|
||||
node_data1.id = 1;
|
||||
node_data1.role = ax::mojom::Role::kDesktop;
|
||||
node_data1.child_ids.push_back(2);
|
||||
tree_update.nodes.emplace_back();
|
||||
auto& node_data2 = tree_update.nodes.back();
|
||||
node_data2.id = 2;
|
||||
node_data2.role = ax::mojom::Role::kButton;
|
||||
std::vector<ui::AXEvent> events;
|
||||
client_->SendAccessibilityEvents(tree_data.tree_id, updates, gfx::Point(),
|
||||
events);
|
||||
|
||||
ExecuteJS(R"JS(
|
||||
const remote = axtest.mojom.TestBindingInterface.getRemote();
|
||||
chrome.automation.getDesktop(desktop => {
|
||||
if (!desktop) {
|
||||
remote.testComplete(/*success=*/false);
|
||||
}
|
||||
if (desktop.children.length !== 1 ||
|
||||
desktop.firstChild !== desktop.lastChild) {
|
||||
remote.testComplete(/*success=*/false);
|
||||
}
|
||||
const button = desktop.firstChild;
|
||||
if (button.role !== 'button') {
|
||||
remote.testComplete(/*success=*/false);
|
||||
}
|
||||
// Spot check button node.
|
||||
if (button.parent !== desktop || button.root !== desktop ||
|
||||
button.indexInParent !== 0 || button.children.length !== 0) {
|
||||
remote.testComplete(/*success=*/false);
|
||||
}
|
||||
chrome.automation.getFocus(focus => {
|
||||
if (!focus) {
|
||||
remote.testComplete(/*success=*/false);
|
||||
}
|
||||
remote.testComplete(/*success=*/focus === button);
|
||||
});
|
||||
});
|
||||
)JS");
|
||||
WaitForJSTestComplete();
|
||||
}
|
||||
|
||||
} // namespace ax
|
||||
|
@ -0,0 +1,29 @@
|
||||
// Copyright 2023 The Chromium Authors
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "services/accessibility/features/automation_client_interface_binder.h"
|
||||
|
||||
#include "services/accessibility/public/mojom/accessibility_service.mojom.h"
|
||||
#include "services/accessibility/public/mojom/automation_client.mojom.h"
|
||||
|
||||
namespace ax {
|
||||
|
||||
AutomationClientInterfaceBinder::AutomationClientInterfaceBinder(
|
||||
mojom::AccessibilityServiceClient* ax_service_client)
|
||||
: ax_service_client_(ax_service_client) {}
|
||||
|
||||
AutomationClientInterfaceBinder::~AutomationClientInterfaceBinder() = default;
|
||||
|
||||
bool AutomationClientInterfaceBinder::MatchesInterface(
|
||||
const std::string& interface_name) {
|
||||
return interface_name == "ax.mojom.AutomationClient";
|
||||
}
|
||||
|
||||
void AutomationClientInterfaceBinder::BindReceiver(
|
||||
mojo::GenericPendingReceiver automation_client_receiver) {
|
||||
ax_service_client_->BindAutomationClient(
|
||||
automation_client_receiver.As<ax::mojom::AutomationClient>());
|
||||
}
|
||||
|
||||
} // namespace ax
|
@ -0,0 +1,38 @@
|
||||
// Copyright 2023 The Chromium Authors
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef SERVICES_ACCESSIBILITY_FEATURES_AUTOMATION_CLIENT_INTERFACE_BINDER_H_
|
||||
#define SERVICES_ACCESSIBILITY_FEATURES_AUTOMATION_CLIENT_INTERFACE_BINDER_H_
|
||||
|
||||
#include "services/accessibility/features/interface_binder.h"
|
||||
#include "services/accessibility/public/mojom/accessibility_service.mojom.h"
|
||||
|
||||
namespace ax {
|
||||
|
||||
// Binds one end of a mojom AutomationClient pipe hosted in Javascript to the
|
||||
// AccessibilityServiceClient that connects back to the main OS process.
|
||||
class AutomationClientInterfaceBinder : public InterfaceBinder {
|
||||
public:
|
||||
explicit AutomationClientInterfaceBinder(
|
||||
mojom::AccessibilityServiceClient* ax_service_client);
|
||||
~AutomationClientInterfaceBinder() override;
|
||||
AutomationClientInterfaceBinder(const AutomationClientInterfaceBinder&) =
|
||||
delete;
|
||||
AutomationClientInterfaceBinder& operator=(
|
||||
const AutomationClientInterfaceBinder&) = delete;
|
||||
|
||||
// InterfaceBinder:
|
||||
bool MatchesInterface(const std::string& interface_name) override;
|
||||
void BindReceiver(mojo::GenericPendingReceiver receiver) override;
|
||||
|
||||
private:
|
||||
// The caller must ensure the client outlives `this`. Here, this is guaranteed
|
||||
// because the client is always a `AssistiveTechnologyControllerImpl`, which
|
||||
// transitively owns `this` via `V8Manager`.
|
||||
raw_ptr<mojom::AccessibilityServiceClient> ax_service_client_;
|
||||
};
|
||||
|
||||
} // namespace ax
|
||||
|
||||
#endif // SERVICES_ACCESSIBILITY_FEATURES_AUTOMATION_CLIENT_INTERFACE_BINDER_H_
|
@ -5,9 +5,7 @@
|
||||
#include "services/accessibility/features/automation_internal_bindings.h"
|
||||
|
||||
#include "base/functional/bind.h"
|
||||
#include "base/functional/callback_forward.h"
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "base/task/sequenced_task_runner.h"
|
||||
#include "gin/function_template.h"
|
||||
#include "mojo/public/cpp/bindings/pending_associated_remote.h"
|
||||
#include "services/accessibility/assistive_technology_controller_impl.h"
|
||||
@ -25,13 +23,11 @@ namespace ax {
|
||||
|
||||
AutomationInternalBindings::AutomationInternalBindings(
|
||||
BindingsIsolateHolder* isolate_holder,
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation,
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client)
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation)
|
||||
: isolate_holder_(isolate_holder),
|
||||
automation_v8_bindings_(std::make_unique<ui::AutomationV8Bindings>(
|
||||
/*AutomationTreeManagerOwner=*/this,
|
||||
/*AutomationV8Router=*/this)),
|
||||
automation_client_remote_(std::move(automation_client)) {
|
||||
/*AutomationV8Router=*/this)) {
|
||||
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
|
||||
receiver_.Bind(std::move(automation));
|
||||
}
|
||||
@ -45,28 +41,6 @@ void AutomationInternalBindings::AddAutomationRoutesToTemplate(
|
||||
template_ = nullptr;
|
||||
}
|
||||
|
||||
void AutomationInternalBindings::AddAutomationInternalRoutesToTemplate(
|
||||
v8::Local<v8::ObjectTemplate>* object_template) {
|
||||
// Adds V8 route for "automationInternal.enableDesktop" and "disableDesktop".
|
||||
(*object_template)
|
||||
->Set(GetIsolate(), "enableDesktop",
|
||||
gin::CreateFunctionTemplate(
|
||||
GetIsolate(),
|
||||
base::BindRepeating(&AutomationInternalBindings::Enable,
|
||||
weak_ptr_factory_.GetWeakPtr())));
|
||||
(*object_template)
|
||||
->Set(GetIsolate(), "disableDesktop",
|
||||
gin::CreateFunctionTemplate(
|
||||
GetIsolate(),
|
||||
base::BindRepeating(&AutomationInternalBindings::Disable,
|
||||
weak_ptr_factory_.GetWeakPtr())));
|
||||
|
||||
// TODO(crbug.com/1357889): Add bindings for additional AutomationInternalAPI
|
||||
// functions, like automationInternal.performAction, etc, paralleling the
|
||||
// implementation in
|
||||
// extensions/browser/api/automation_internal/automation_internal_api.h.
|
||||
}
|
||||
|
||||
ui::AutomationV8Bindings* AutomationInternalBindings::GetAutomationV8Bindings()
|
||||
const {
|
||||
DCHECK(automation_v8_bindings_);
|
||||
@ -161,33 +135,4 @@ void AutomationInternalBindings::DispatchEvent(
|
||||
// TODO(crbug.com/1357889): Send the event to V8.
|
||||
}
|
||||
|
||||
void AutomationInternalBindings::Enable(gin::Arguments* args) {
|
||||
// TODO(crbug.com/1357889): Send to the OS AutomationClient.
|
||||
// automation_client_remote_->Enable(
|
||||
// base::BindOnce(&AutomationInternalBindings::OnTreeID,
|
||||
// weak_ptr_factory_.GetWeakPtr()));
|
||||
|
||||
// TODO(crbug.com/1357889): Need to figure out how to RespondLater to these
|
||||
// args in order to send the tree ID back to the JS caller as a callback.
|
||||
}
|
||||
|
||||
void AutomationInternalBindings::Disable(gin::Arguments* args) {
|
||||
// TODO(crbug.com/1357889): Send to the OS AutomationClient.
|
||||
// automation_client_remote_->Disable();
|
||||
// Execute the return value on the args so that the JS callback
|
||||
// will execute.
|
||||
args->Return(true);
|
||||
}
|
||||
|
||||
void AutomationInternalBindings::EnableTree(const ui::AXTreeID& tree_id) {
|
||||
// TODO(crbug.com/1357889): Send to the OS AutomationClient.
|
||||
// automation_client_remote_->EnableTree(tree_id);
|
||||
}
|
||||
|
||||
void AutomationInternalBindings::PerformAction(
|
||||
const ui::AXActionData& action_data) {
|
||||
// TODO(crbug.com/1357889): Send to the OS AutomationClient.
|
||||
// automation_client_remote_->PerformAction(action_data);
|
||||
}
|
||||
|
||||
} // namespace ax
|
||||
|
@ -10,8 +10,6 @@
|
||||
#include "base/memory/weak_ptr.h"
|
||||
#include "base/task/sequenced_task_runner.h"
|
||||
#include "mojo/public/cpp/bindings/pending_associated_receiver.h"
|
||||
#include "mojo/public/cpp/bindings/pending_remote.h"
|
||||
#include "mojo/public/cpp/bindings/remote.h"
|
||||
#include "services/accessibility/assistive_technology_controller_impl.h"
|
||||
#include "services/accessibility/public/mojom/accessibility_service.mojom.h"
|
||||
#include "ui/accessibility/ax_event.h"
|
||||
@ -29,10 +27,6 @@ class Local;
|
||||
class ObjectTemplate;
|
||||
} // namespace v8
|
||||
|
||||
namespace gin {
|
||||
class Arguments;
|
||||
} // namespace gin
|
||||
|
||||
namespace ax {
|
||||
class BindingsIsolateHolder;
|
||||
|
||||
@ -50,8 +44,7 @@ class AutomationInternalBindings : public ui::AutomationTreeManagerOwner,
|
||||
// V8Manager which implements BindingsIsolateHolder.
|
||||
explicit AutomationInternalBindings(
|
||||
BindingsIsolateHolder* isolate_holder,
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation,
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client);
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation);
|
||||
~AutomationInternalBindings() override;
|
||||
AutomationInternalBindings(const AutomationInternalBindings&) = delete;
|
||||
AutomationInternalBindings& operator=(const AutomationInternalBindings&) =
|
||||
@ -62,9 +55,6 @@ class AutomationInternalBindings : public ui::AutomationTreeManagerOwner,
|
||||
// Adds V8 bindings for the chrome.automation API.
|
||||
void AddAutomationRoutesToTemplate(
|
||||
v8::Local<v8::ObjectTemplate>* object_template);
|
||||
// Adds V8 bindings for the chrome.automationInternal API.
|
||||
void AddAutomationInternalRoutesToTemplate(
|
||||
v8::Local<v8::ObjectTemplate>* object_template);
|
||||
|
||||
// ui::AutomationTreeManagerOwner:
|
||||
ui::AutomationV8Bindings* GetAutomationV8Bindings() const override;
|
||||
@ -94,13 +84,6 @@ class AutomationInternalBindings : public ui::AutomationTreeManagerOwner,
|
||||
void DispatchEvent(const std::string& event_name,
|
||||
const base::Value::List& event_args) const override;
|
||||
|
||||
// Methods to communicate back to the OS main process. These should get bound
|
||||
// to V8 JS methods and called from there.
|
||||
void Enable(gin::Arguments* args);
|
||||
void Disable(gin::Arguments* args);
|
||||
void EnableTree(const ui::AXTreeID& tree_id);
|
||||
void PerformAction(const ui::AXActionData& action_data);
|
||||
|
||||
private:
|
||||
friend class AutomationInternalBindingsTest;
|
||||
|
||||
@ -112,10 +95,6 @@ class AutomationInternalBindings : public ui::AutomationTreeManagerOwner,
|
||||
|
||||
std::unique_ptr<ui::AutomationV8Bindings> automation_v8_bindings_;
|
||||
|
||||
// We can send automation info back to the main Service thread with the
|
||||
// automation client interface.
|
||||
mojo::Remote<mojom::AutomationClient> automation_client_remote_;
|
||||
|
||||
SEQUENCE_CHECKER(sequence_checker_);
|
||||
|
||||
base::WeakPtrFactory<AutomationInternalBindings> weak_ptr_factory_{this};
|
||||
|
@ -61,16 +61,9 @@ class TestIsolateHolder : public BindingsIsolateHolder {
|
||||
v8::Local<v8::ObjectTemplate> automation_template =
|
||||
v8::ObjectTemplate::New(isolate_holder_->isolate());
|
||||
automation_bindings->AddAutomationRoutesToTemplate(&automation_template);
|
||||
global_template->Set(isolate_holder_->isolate(), "automation",
|
||||
global_template->Set(isolate_holder_->isolate(), "nativeAutomationInternal",
|
||||
automation_template);
|
||||
|
||||
v8::Local<v8::ObjectTemplate> automation_internal_template =
|
||||
v8::ObjectTemplate::New(isolate_holder_->isolate());
|
||||
automation_bindings->AddAutomationInternalRoutesToTemplate(
|
||||
&automation_internal_template);
|
||||
global_template->Set(isolate_holder_->isolate(), "automationInternal",
|
||||
automation_internal_template);
|
||||
|
||||
v8::Local<v8::Context> context =
|
||||
v8::Context::New(isolate_holder_->isolate(),
|
||||
/*extensions=*/nullptr, global_template);
|
||||
@ -108,13 +101,10 @@ class AutomationInternalBindingsTest : public testing::Test {
|
||||
service_client_ = std::make_unique<FakeServiceClient>(nullptr);
|
||||
test_isolate_holder_ = std::make_unique<TestIsolateHolder>();
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation;
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client;
|
||||
service_client_->BindAutomation(
|
||||
automation.InitWithNewEndpointAndPassRemote(),
|
||||
automation_client.InitWithNewPipeAndPassReceiver());
|
||||
automation.InitWithNewEndpointAndPassRemote());
|
||||
automation_bindings_ = std::make_unique<AutomationInternalBindings>(
|
||||
test_isolate_holder_.get(), std::move(automation),
|
||||
std::move(automation_client));
|
||||
test_isolate_holder_.get(), std::move(automation));
|
||||
test_isolate_holder_->StartTestV8AndBindAutomation(
|
||||
automation_bindings_.get());
|
||||
}
|
||||
@ -154,22 +144,21 @@ TEST_F(AutomationInternalBindingsTest, CatchesExceptions) {
|
||||
|
||||
// Spot checks that bindings were added to the correct namespace.
|
||||
TEST_F(AutomationInternalBindingsTest, SpotCheckBindingsAdded) {
|
||||
// This should succeed because automation and automationInternal
|
||||
// This should succeed because automation and nativeAutomationInternal
|
||||
// bindings were added.
|
||||
std::string script =
|
||||
base::StringPrintf(R"JS(
|
||||
automation.GetFocus();
|
||||
automation.SetDesktopID('%s');
|
||||
automation.StartCachingAccessibilityTrees();
|
||||
automationInternal.enableDesktop();
|
||||
nativeAutomationInternal.GetFocus();
|
||||
nativeAutomationInternal.SetDesktopID('%s');
|
||||
nativeAutomationInternal.StartCachingAccessibilityTrees();
|
||||
)JS",
|
||||
ui::AXTreeID::CreateNewAXTreeID().ToString().c_str());
|
||||
EXPECT_TRUE(test_isolate_holder_->ExecuteScriptInContext(script));
|
||||
EXPECT_EQ(0, test_isolate_holder_->ErrorCount());
|
||||
|
||||
// Lowercase getFocus does not exist.
|
||||
// Lowercase getFocus does not exist on nativeAutomationInternal.
|
||||
script = R"JS(
|
||||
automation.getFocus();
|
||||
nativeAutomationInternal.getFocus();
|
||||
)JS";
|
||||
EXPECT_FALSE(test_isolate_holder_->ExecuteScriptInContext(script));
|
||||
EXPECT_EQ(1, test_isolate_holder_->ErrorCount());
|
||||
@ -201,15 +190,16 @@ TEST_F(AutomationInternalBindingsTest, GetsFocusAndNodeData) {
|
||||
// throwing exceptions in these tests.
|
||||
std::string script = base::StringPrintf(R"JS(
|
||||
const treeId = '%s';
|
||||
automation.SetDesktopID(treeId);
|
||||
const focusedNodeInfo = automation.GetFocus();
|
||||
nativeAutomationInternal.SetDesktopID(treeId);
|
||||
const focusedNodeInfo = nativeAutomationInternal.GetFocus();
|
||||
if (!focusedNodeInfo) {
|
||||
throw 'Expected to find a focused object';
|
||||
}
|
||||
if (focusedNodeInfo.nodeId !== 2) {
|
||||
throw 'Expected focused node ID 2, got ' + focusedNodeInfo.nodeId;
|
||||
}
|
||||
const role = automation.GetRole(treeId, focusedNodeInfo.nodeId);
|
||||
const role = nativeAutomationInternal.GetRole(treeId,
|
||||
focusedNodeInfo.nodeId);
|
||||
if (role !== 'button') {
|
||||
throw 'Expected role button, got ' + role;
|
||||
}
|
||||
@ -246,8 +236,8 @@ TEST_F(AutomationInternalBindingsTest, GetsLocation) {
|
||||
// throwing exceptions in these tests.
|
||||
std::string script = base::StringPrintf(R"JS(
|
||||
const treeId = '%s';
|
||||
automation.SetDesktopID(treeId);
|
||||
let bounds = automation.GetLocation(treeId, 2);
|
||||
nativeAutomationInternal.SetDesktopID(treeId);
|
||||
let bounds = nativeAutomationInternal.GetLocation(treeId, 2);
|
||||
if (!bounds) {
|
||||
throw 'Could not get bounds for node';
|
||||
}
|
||||
@ -267,7 +257,7 @@ TEST_F(AutomationInternalBindingsTest, GetsLocation) {
|
||||
DispatchLocationChange(tree_data.tree_id, 1, new_bounds);
|
||||
|
||||
script = R"JS(
|
||||
bounds = automation.GetLocation(treeId, 2);
|
||||
bounds = nativeAutomationInternal.GetLocation(treeId, 2);
|
||||
if (!bounds) {
|
||||
throw 'Could not get bounds for node';
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
copy("js_api_copied_files") {
|
||||
sources = [
|
||||
"accessibility_private.js",
|
||||
"automation.js",
|
||||
"chrome_event.js",
|
||||
"speech_recognition.js",
|
||||
"tts.js",
|
||||
@ -15,6 +16,7 @@ copy("js_api_copied_files") {
|
||||
group("js_api_files") {
|
||||
data = [
|
||||
"$target_gen_dir/accessibility_private.js",
|
||||
"$target_gen_dir/automation.js",
|
||||
"$target_gen_dir/chrome_event.js",
|
||||
"$target_gen_dir/speech_recognition.js",
|
||||
"$target_gen_dir/tts.js",
|
||||
|
2217
services/accessibility/features/javascript/automation.js
Normal file
2217
services/accessibility/features/javascript/automation.js
Normal file
File diff suppressed because it is too large
Load Diff
@ -27,6 +27,7 @@
|
||||
#include "mojo/public/cpp/bindings/generic_pending_receiver.h"
|
||||
#include "services/accessibility/assistive_technology_controller_impl.h"
|
||||
#include "services/accessibility/features/autoclick_client_interface_binder.h"
|
||||
#include "services/accessibility/features/automation_client_interface_binder.h"
|
||||
#include "services/accessibility/features/automation_internal_bindings.h"
|
||||
#include "services/accessibility/features/devtools/os_devtools_agent.h"
|
||||
#include "services/accessibility/features/interface_binder.h"
|
||||
@ -152,10 +153,9 @@ V8Environment::~V8Environment() {
|
||||
}
|
||||
|
||||
void V8Environment::InstallAutomation(
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation,
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client) {
|
||||
automation_bindings_ = std::make_unique<AutomationInternalBindings>(
|
||||
this, std::move(automation), std::move(automation_client));
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation) {
|
||||
automation_bindings_ =
|
||||
std::make_unique<AutomationInternalBindings>(this, std::move(automation));
|
||||
}
|
||||
|
||||
void V8Environment::InstallOSState() {
|
||||
@ -400,13 +400,8 @@ void V8Environment::AddV8Bindings() {
|
||||
Local<v8::ObjectTemplate> automation_template =
|
||||
v8::ObjectTemplate::New(isolate);
|
||||
automation_bindings_->AddAutomationRoutesToTemplate(&automation_template);
|
||||
chrome_template->Set(isolate, "automation", automation_template);
|
||||
Local<v8::ObjectTemplate> automation_internal_template =
|
||||
v8::ObjectTemplate::New(isolate);
|
||||
automation_bindings_->AddAutomationInternalRoutesToTemplate(
|
||||
&automation_internal_template);
|
||||
chrome_template->Set(isolate, "automationInternal",
|
||||
automation_internal_template);
|
||||
global_template->Set(isolate, "nativeAutomationInternal",
|
||||
automation_template);
|
||||
}
|
||||
|
||||
// Add chrome.runtime.
|
||||
@ -478,11 +473,13 @@ V8Manager::~V8Manager() {
|
||||
}
|
||||
|
||||
void V8Manager::ConfigureAutomation(
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation,
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client) {
|
||||
mojom::AccessibilityServiceClient* ax_service_client,
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation) {
|
||||
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
|
||||
v8_env_.AsyncCall(&V8Environment::InstallAutomation)
|
||||
.WithArgs(std::move(automation), std::move(automation_client));
|
||||
.WithArgs(std::move(automation));
|
||||
interface_binders_.push_back(
|
||||
std::make_unique<AutomationClientInterfaceBinder>(ax_service_client));
|
||||
}
|
||||
|
||||
void V8Manager::ConfigureAutoclick(
|
||||
|
@ -83,8 +83,7 @@ class V8Environment : public BindingsIsolateHolder {
|
||||
// All of the APIs needed for this V8Manager (based on the AT type) should be
|
||||
// installed before adding V8 bindings.
|
||||
void InstallAutomation(
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation,
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client);
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation);
|
||||
void InstallOSState();
|
||||
void AddV8Bindings();
|
||||
|
||||
@ -191,8 +190,8 @@ class V8Manager {
|
||||
// done before calling `FinishContextSetUp()`.
|
||||
void ConfigureAutoclick(mojom::AccessibilityServiceClient* ax_service_client);
|
||||
void ConfigureAutomation(
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation,
|
||||
mojo::PendingRemote<mojom::AutomationClient> automation_client);
|
||||
mojom::AccessibilityServiceClient* ax_service_client,
|
||||
mojo::PendingAssociatedReceiver<mojom::Automation> automation);
|
||||
void ConfigureOSState();
|
||||
void ConfigureSpeechRecognition(
|
||||
mojom::AccessibilityServiceClient* ax_service_client);
|
||||
|
@ -13,6 +13,14 @@ mojom("automation") {
|
||||
]
|
||||
}
|
||||
|
||||
mojom("automation_client") {
|
||||
sources = [ "automation_client.mojom" ]
|
||||
public_deps = [
|
||||
"//mojo/public/mojom/base",
|
||||
"//ui/accessibility/mojom",
|
||||
]
|
||||
}
|
||||
|
||||
mojom("assistive_technology_type") {
|
||||
sources = [ "assistive_technology_type.mojom" ]
|
||||
public_deps = [ "//mojo/public/mojom/base" ]
|
||||
@ -25,6 +33,7 @@ mojom("mojom") {
|
||||
sources = [ "accessibility_service.mojom" ]
|
||||
public_deps = [
|
||||
":automation",
|
||||
":automation_client",
|
||||
"//mojo/public/mojom/base",
|
||||
"//sandbox/policy/mojom",
|
||||
]
|
||||
@ -65,6 +74,11 @@ if (supports_os_accessibility_service) {
|
||||
"$root_gen_dir/skia/public/mojom/skcolor.mojom-lite.js",
|
||||
"$root_gen_dir/ui/events/mojom/event_constants.mojom-lite.js",
|
||||
"$root_gen_dir/ui/events/mojom/event.mojom-lite.js",
|
||||
"$root_gen_dir/mojo/public/mojom/base/unguessable_token.mojom-lite.js",
|
||||
"$root_gen_dir/ui/accessibility/ax_enums.mojom-lite.js",
|
||||
"$root_gen_dir/ui/accessibility/mojom/ax_tree_id.mojom-lite.js",
|
||||
"$root_gen_dir/ui/accessibility/mojom/ax_action_data.mojom-lite.js",
|
||||
"$target_gen_dir/automation_client.mojom-lite.js",
|
||||
"$root_gen_dir/ui/gfx/geometry/mojom/geometry.mojom-lite.js",
|
||||
"$root_gen_dir/ui/latency/mojom/latency_info.mojom-lite.js",
|
||||
"$target_gen_dir/assistive_technology_type.mojom-lite.js",
|
||||
@ -76,6 +90,7 @@ if (supports_os_accessibility_service) {
|
||||
]
|
||||
deps = [
|
||||
":assistive_technology_type_js__generator",
|
||||
":automation_client_js__generator",
|
||||
":os_service_mojom_js__generator",
|
||||
]
|
||||
}
|
||||
|
@ -6,6 +6,7 @@ module ax.mojom;
|
||||
|
||||
import "sandbox/policy/mojom/sandbox.mojom";
|
||||
import "services/accessibility/public/mojom/automation.mojom";
|
||||
import "services/accessibility/public/mojom/automation_client.mojom";
|
||||
|
||||
[EnableIf=supports_os_accessibility_service]
|
||||
import "services/accessibility/public/mojom/speech_recognition.mojom";
|
||||
@ -73,13 +74,16 @@ interface AccessibilityService {
|
||||
// TODO(crbug.com/1355633): Other APIs needed in Chrome OS and Fuchsia should be
|
||||
// bound here.
|
||||
interface AccessibilityServiceClient {
|
||||
// Binds an Automation implemented in the service process to a client
|
||||
// implemented in the main OS process.
|
||||
// Binds an Automation implemented in the service process, called by
|
||||
// the main OS browser process.
|
||||
// In Chrome OS this may be called once per feature to provide
|
||||
// automation connections to each V8 isolate.
|
||||
BindAutomation(
|
||||
pending_associated_remote<Automation> automation,
|
||||
pending_receiver<AutomationClient> automation_client);
|
||||
pending_associated_remote<Automation> automation);
|
||||
|
||||
// Binds to an AutomatoinClient implemented in the main OS process,
|
||||
// called by the service.
|
||||
BindAutomationClient(pending_receiver<AutomationClient> automation_client);
|
||||
|
||||
// Binds to a Autoclick client implemented in the main OS process, and
|
||||
// an Autoclick implemented in the service process, called by the service.
|
||||
|
@ -49,34 +49,3 @@ interface Automation {
|
||||
ax.mojom.AXActionData data,
|
||||
gfx.mojom.Rect? rect);
|
||||
};
|
||||
|
||||
// Implemented outside of the service, e.g. Chrome OS Ash, Chrome browser
|
||||
// process, renderers, or other tree sources.
|
||||
// Called by the accessibility service and automation extension API to enable
|
||||
// accessibility and perform actions. For example the accessibility service
|
||||
// might want to do a 'click' because a screen reader requested the default
|
||||
// action. Then the accessibility service would use
|
||||
// AutomationClient::PerformAction to pass that back to the client.
|
||||
interface AutomationClient {
|
||||
// // Enables automation for the client. This will result in the client
|
||||
// // repeatedly calling DispatchAccessibilityEvents() on the Automation
|
||||
// // interface.
|
||||
// // Runs the callback with tree ID for the desktop tree.
|
||||
// Enable() => (ax.mojom.AXTreeID desktop_id);
|
||||
|
||||
// // Disables automation in the client. This has the effect of turning off
|
||||
// // accessibility tree creation within the client. Calling this method
|
||||
// // without calling Enable or calling it multiple times has no adverse
|
||||
// // effects.
|
||||
// Disable();
|
||||
|
||||
// // Enables accessibility for a particular subtree of the client. This will
|
||||
// // result in the client repeatedly calling DispatchAccessibilityEvents()
|
||||
// // on the Automation interface.
|
||||
// EnableTree(ax.mojom.AXTreeID tree_id);
|
||||
|
||||
// // Forwards the action described by AXActionData to all clients. Actions
|
||||
// // are resolved by each client based on tree id, action type and other
|
||||
// // action data fields.
|
||||
// PerformAction(ax.mojom.AXActionData action_data);
|
||||
};
|
||||
|
42
services/accessibility/public/mojom/automation_client.mojom
Normal file
42
services/accessibility/public/mojom/automation_client.mojom
Normal file
@ -0,0 +1,42 @@
|
||||
// Copyright 2023 The Chromium Authors
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
module ax.mojom;
|
||||
|
||||
import "ui/accessibility/mojom/ax_action_data.mojom";
|
||||
import "ui/accessibility/mojom/ax_tree_id.mojom";
|
||||
|
||||
// Implemented outside of the service, e.g. Chrome OS Ash, Chrome browser
|
||||
// process, renderers, or other tree sources.
|
||||
// Called by the accessibility service and automation extension API to enable
|
||||
// accessibility and perform actions. For example the accessibility service
|
||||
// might want to do a 'click' because a screen reader requested the default
|
||||
// action. Then the accessibility service would use
|
||||
// AutomationClient::PerformAction to pass that back to the client.
|
||||
interface AutomationClient {
|
||||
// Enables automation for the client. This will result in the client
|
||||
// repeatedly calling DispatchAccessibilityEvents() on the Automation
|
||||
// interface.
|
||||
// Runs the callback with tree ID for the desktop tree.
|
||||
Enable() => (ax.mojom.AXTreeID desktop_id);
|
||||
|
||||
// Disables automation in the client. This has the effect of turning off
|
||||
// accessibility tree creation within the client. Calling this method
|
||||
// without calling Enable or calling it multiple times has no adverse
|
||||
// effects.
|
||||
// TODO(b/262638176): Implement AutomationClient::Disable.
|
||||
// Disable();
|
||||
|
||||
// Enables accessibility for a particular subtree of the client. This will
|
||||
// result in the client repeatedly calling DispatchAccessibilityEvents()
|
||||
// on the Automation interface.
|
||||
// TODO(b/262638176): Implement AutomationClient::EnableTree.
|
||||
// EnableTree(ax.mojom.AXTreeID tree_id);
|
||||
|
||||
// Forwards the action described by AXActionData to all clients. Actions
|
||||
// are resolved by each client based on tree id, action type and other
|
||||
// action data fields.
|
||||
// TODO(b/262638176): Implement AutomationClient::PerformAction.
|
||||
// PerformAction(ax.mojom.AXActionData action_data);
|
||||
};
|
Reference in New Issue
Block a user