[Switch Access] Add support for nested buttons

This solves an issue where a button element that contains other controls
cannot be accessed using Switch Access, as the previous paradigm forced
each node to be either a group or actionable.

The new icons can be seen here:
https://screenshot.googleplex.com/7ikBinJt8gGpDaD
https://screenshot.googleplex.com/3zxgjHyUPe6vQku

AX-Relnotes: Supports nested buttons with Switch Access.
Fixed: b/345447869
Test: SwitchAccessBasicNodeTest.Actions
Change-Id: Ica19f8d77b50f3e0a1125cf1c06291fc7bea6014
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5698349
Reviewed-by: Kyle Horimoto <khorimoto@chromium.org>
Commit-Queue: Anastasia Helfinstein <anastasi@google.com>
Reviewed-by: Akihiro Ota <akihiroota@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Auto-Submit: Anastasia Helfinstein <anastasi@google.com>
Cr-Commit-Position: refs/heads/main@{#1328464}

ash/ash_strings.grd

@@ -1319,6 +1319,9 @@ Style notes:
       <message name="IDS_ASH_SWITCH_ACCESS_DICTATION" desc="The label for the Switch Access menu option to dictate into the focused input.">
         Dictation
       </message>
+      <message name="IDS_ASH_SWITCH_ACCESS_DRILL_DOWN" desc="The label for the Switch Access menu option to navigate within the current selection.">
+        Drill down
+      </message>
       <message name="IDS_ASH_SWITCH_ACCESS_END_TEXT_SELECTION" desc="The label for the Switch Access menu option to stop changing the text selection.">
         Stop selecting
       </message>

ash/ash_strings_grd/IDS_ASH_SWITCH_ACCESS_DRILL_DOWN.png.sha1

@@ -0,0 +1 @@
+ace42959e231e1f1e3ab3667ae540337b046bcee

ash/ash_strings_grd/IDS_ASH_SWITCH_ACCESS_SELECT.png.sha1

@@ -1 +1 @@
-2b47402c230e3dfdd70ffdb6545e212aa3421f9d
+decceac8ec960cf3fb841149f68e5bf9d42c6962

ash/resources/vector_icons/BUILD.gn

@@ -441,6 +441,7 @@ aggregate_vector_icons("ash_vector_icons") {
     "switch_access_copy.icon",
     "switch_access_cut.icon",
     "switch_access_decrement.icon",
+    "switch_access_drill_down.icon",
     "switch_access_end_text_selection.icon",
     "switch_access_increment.icon",
     "switch_access_item_scan.icon",

ash/resources/vector_icons/switch_access_drill_down.icon

@@ -0,0 +1,46 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+CANVAS_DIMENSIONS, 20,
+FILL_RULE_NONZERO,
+MOVE_TO, 6.63f, 16,
+LINE_TO, 5.56f, 14.94f,
+LINE_TO, 6.63f, 13.88f,
+H_LINE_TO, 6.25f,
+CUBIC_TO, 4.79f, 13.88f, 3.55f, 13.37f, 2.52f, 12.35f,
+CUBIC_TO, 1.51f, 11.33f, 1, 10.08f, 1, 8.63f,
+CUBIC_TO, 1, 7.17f, 1.51f, 5.93f, 2.52f, 4.92f,
+CUBIC_TO, 3.55f, 3.89f, 4.79f, 3.38f, 6.25f, 3.38f,
+H_LINE_TO, 9.5f,
+V_LINE_TO, 4.88f,
+H_LINE_TO, 6.25f,
+CUBIC_TO, 5.21f, 4.88f, 4.32f, 5.24f, 3.58f, 5.98f,
+CUBIC_TO, 2.86f, 6.7f, 2.5f, 7.58f, 2.5f, 8.63f,
+CUBIC_TO, 2.5f, 9.67f, 2.86f, 10.56f, 3.58f, 11.29f,
+CUBIC_TO, 4.32f, 12.01f, 5.21f, 12.38f, 6.25f, 12.38f,
+H_LINE_TO, 6.65f,
+LINE_TO, 5.56f, 11.31f,
+LINE_TO, 6.63f, 10.25f,
+LINE_TO, 9.5f, 13.13f,
+LINE_TO, 6.63f, 16,
+CLOSE,
+MOVE_TO, 11, 15.38f,
+V_LINE_TO, 10.38f,
+H_LINE_TO, 18,
+V_LINE_TO, 15.38f,
+H_LINE_TO, 11,
+CLOSE,
+MOVE_TO, 11, 8.38f,
+V_LINE_TO, 3.38f,
+H_LINE_TO, 18,
+V_LINE_TO, 8.38f,
+H_LINE_TO, 11,
+CLOSE,
+MOVE_TO, 12.5f, 6.88f,
+H_LINE_TO, 16.5f,
+V_LINE_TO, 4.88f,
+H_LINE_TO, 12.5f,
+V_LINE_TO, 6.88f,
+CLOSE,
+NEW_PATH

ash/resources/vector_icons/switch_access_select.icon

@@ -1,65 +1,24 @@
-// Copyright 2020 The Chromium Authors
+// Copyright 2024 The Chromium Authors
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 CANVAS_DIMENSIONS, 20,
-MOVE_TO, 3, 3,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-H_LINE_TO, 3,
+R_MOVE_TO, 8, 16.71f,
+R_CUBIC_TO, -2.89f, -0.86f, -5, -3.54f, -5, -6.71f,
+R_CUBIC_TO, 0, -3.87f, 3.13f, -7, 7, -7,
+R_CUBIC_TO, 3.17f, 0, 5.85f, 2.11f, 6.71f, 5,
+R_H_LINE_TO, -2.13f,
+R_CUBIC_TO, -0.77f, -1.77f, -2.53f, -3, -4.58f, -3,
+R_CUBIC_TO, -2.76f, 0, -5, 2.24f, -5, 5,
+R_CUBIC_TO, 0, 2.05f, 1.23f, 3.81f, 3, 4.58f,
 CLOSE,
-R_MOVE_TO, 0, 4,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-H_LINE_TO, 3,
-CLOSE,
-R_MOVE_TO, 0, 4,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-H_LINE_TO, 3,
-CLOSE,
-R_MOVE_TO, 0, 4,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-H_LINE_TO, 3,
-CLOSE,
-R_MOVE_TO, 4, 0,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-H_LINE_TO, 7,
-CLOSE,
-R_MOVE_TO, 4, 0,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
+R_MOVE_TO, 7, -5.71f,
+R_H_LINE_TO, -2.59f,
+R_LINE_TO, 5.07f, 5.07f,
+R_LINE_TO, -1.41f, 1.41f,
+R_LINE_TO, -5.07f, -5.07f,
+R_V_LINE_TO, 2.59f,
 R_H_LINE_TO, -2,
-CLOSE,
-R_MOVE_TO, 4, 0,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-R_H_LINE_TO, -2,
-CLOSE,
-R_MOVE_TO, 0, -4,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-R_H_LINE_TO, -2,
-CLOSE,
-R_MOVE_TO, 0, -4,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-R_H_LINE_TO, -2,
-CLOSE,
-R_MOVE_TO, 0, -4,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-R_H_LINE_TO, -2,
-CLOSE,
-R_MOVE_TO, -4, 0,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-R_H_LINE_TO, -2,
-CLOSE,
-MOVE_TO, 7, 3,
-R_H_LINE_TO, 2,
-R_V_LINE_TO, 2,
-H_LINE_TO, 7,
-CLOSE
+R_V_LINE_TO, -6,
+R_H_LINE_TO, 6,
+CLOSE

ash/system/accessibility/switch_access/switch_access_menu_view.cc

@@ -45,6 +45,8 @@ const base::flat_map<std::string, ButtonInfo>& GetMenuButtonDetails() {
           {&kSwitchAccessDecrementIcon, IDS_ASH_SWITCH_ACCESS_DECREMENT}},
          {"dictation",
           {&kDictationOnNewuiIcon, IDS_ASH_SWITCH_ACCESS_DICTATION}},
+         {"drillDown",
+          {&kSwitchAccessDrillDownIcon, IDS_ASH_SWITCH_ACCESS_DRILL_DOWN}},
          {"endTextSelection",
           {&kSwitchAccessEndTextSelectionIcon,
            IDS_ASH_SWITCH_ACCESS_END_TEXT_SELECTION}},

chrome/browser/ash/accessibility/switch_access_browsertest.cc

@@ -188,6 +188,12 @@ IN_PROC_BROWSER_TEST_F(SwitchAccessTest, NavigateButtonsInTextFieldMenu) {
   // Send "next".
   SendVirtualKeyPress(ui::KeyboardCode::VKEY_2);
 
+  // The next menu item is the "enter" button.
+  utils()->WaitForFocusRing("primary", "button", "Drill down");
+
+  // Send "next".
+  SendVirtualKeyPress(ui::KeyboardCode::VKEY_2);
+
   // The next menu item is the "point scanning" button.
   utils()->WaitForFocusRing("primary", "button", "Point scanning");
 

chrome/browser/resources/chromeos/accessibility/definitions/accessibility_private_mv2.d.ts

@@ -96,6 +96,7 @@ declare global {
         CUT = 'cut',
         DECREMENT = 'decrement',
         DICTATION = 'dictation',
+        DRILL_DOWN = 'drillDown',
         END_TEXT_SELECTION = 'endTextSelection',
         INCREMENT = 'increment',
         ITEM_SCAN = 'itemScan',

chrome/browser/resources/chromeos/accessibility/switch_access/action_manager.ts

@@ -168,6 +168,7 @@ export class ActionManager {
           MenuAction.CUT,
           MenuAction.DECREMENT,
           MenuAction.DICTATION,
+          MenuAction.DRILL_DOWN,
           MenuAction.INCREMENT,
           MenuAction.KEYBOARD,
           MenuAction.MOVE_CURSOR,

chrome/browser/resources/chromeos/accessibility/switch_access/nodes/basic_node.ts

@@ -43,19 +43,27 @@ export class BasicNode extends SAChildNode {
   private baseNode_: AutomationNode;
   private parent_: SARootNode | null;
   private locationChangedHandler_?: RepeatedEventHandler;
+  private isActionable_: boolean;
   private static creators_: Creator[] = [];
 
   protected constructor(baseNode: AutomationNode, parent: SARootNode | null) {
     super();
     this.baseNode_ = baseNode;
     this.parent_ = parent;
+    this.isActionable_ = !this.isGroup() ||
+        SwitchAccessPredicate.isActionable(baseNode, new SACache());
   }
 
   // ================= Getters and setters =================
 
   override get actions(): MenuAction[] {
     const actions: MenuAction[] = [];
-    actions.push(MenuAction.SELECT);
+    if (this.isActionable_) {
+      actions.push(MenuAction.SELECT);
+    }
+    if (this.isGroup()) {
+      actions.push(MenuAction.DRILL_DOWN);
+    }
 
     const ancestor = this.getScrollableAncestor_();
     // TODO(b/314203187): Not null asserted, check that this is correct.
@@ -162,12 +170,16 @@ export class BasicNode extends SAChildNode {
   override performAction(action: MenuAction): ActionResponse {
     let ancestor;
     switch (action) {
-      case MenuAction.SELECT:
+      case MenuAction.DRILL_DOWN:
         if (this.isGroup()) {
           Navigator.byItem.enterGroup();
-        } else {
-          this.baseNode_.doDefault();
+          return ActionResponse.CLOSE_MENU;
         }
+        // Should not happen.
+        console.error('Action DRILL_DOWN received on non-group node.');
+        return ActionResponse.NO_ACTION_TAKEN;
+      case MenuAction.SELECT:
+        this.baseNode_.doDefault();
         return ActionResponse.CLOSE_MENU;
       case MenuAction.SCROLL_DOWN:
         ancestor = this.getScrollableAncestor_();

chrome/browser/resources/chromeos/accessibility/switch_access/nodes/basic_node_test.js

@@ -117,7 +117,10 @@ TEST_F('SwitchAccessBasicNodeTest', 'Equals', function() {
 
 AX_TEST_F('SwitchAccessBasicNodeTest', 'Actions', async function() {
   const website = `<input type="text">
-                   <button></button>
+                   <div role="button" aria-label="group">
+                     <button>A</button>
+                     <button>B</button>
+                   </div>
                    <input type="range" min=1 max=5 value=3>`;
   const rootWebArea = await this.runWithLoadedTree(website);
   const textField = BasicNode.create(
@@ -136,20 +139,44 @@ AX_TEST_F('SwitchAccessBasicNodeTest', 'Actions', async function() {
   assertFalse(
       textField.hasAction(MenuAction.SELECT), 'Text field has action SELECT');
 
-  const button = BasicNode.create(
-      rootWebArea.find({role: chrome.automation.RoleType.BUTTON}),
+  const buttonGroup = BasicNode.create(
+      rootWebArea.find({
+        role: chrome.automation.RoleType.BUTTON,
+        attributes: {name: 'group'},
+      }),
       new SARootNode());
+  assertNotNullNorUndefined(buttonGroup);
 
   assertEquals(
-      chrome.automation.RoleType.BUTTON, button.role,
-      'Button node is not a button');
+      chrome.automation.RoleType.BUTTON, buttonGroup.role,
+      'Button group node is not a button');
   assertTrue(
-      button.hasAction(MenuAction.SELECT),
-      'Button does not have action SELECT');
+      buttonGroup.hasAction(MenuAction.SELECT),
+      'Button group does not have action SELECT');
   assertFalse(
-      button.hasAction(MenuAction.KEYBOARD), 'Button has action KEYBOARD');
+      buttonGroup.hasAction(MenuAction.KEYBOARD), 'Button has action KEYBOARD');
   assertFalse(
-      button.hasAction(MenuAction.DICTATION), 'Button has action DICTATION');
+      buttonGroup.hasAction(MenuAction.DICTATION),
+      'Button has action DICTATION');
+  assertTrue(buttonGroup.isGroup(), 'Button group is not a group');
+  assertTrue(
+      buttonGroup.hasAction(MenuAction.DRILL_DOWN),
+      'Button group does not have action DRILL_DOWN');
+  assertTrue(
+      buttonGroup.asRootNode().children.length === 3,
+      'Button group does not have three children (A, B, and the back button)');
+
+  const buttonA = buttonGroup.asRootNode().firstChild;
+  assertEquals(
+      chrome.automation.RoleType.BUTTON, buttonA.role,
+      'Button node A is not a button');
+  assertTrue(
+      buttonA.hasAction(MenuAction.SELECT),
+      'Button A does not have action SELECT');
+  assertFalse(
+      buttonA.hasAction(MenuAction.DRILL_DOWN),
+      'Button A should not have action DRILL_DOWN');
+  assertFalse(buttonA.isGroup(), 'Button A should not be a group');
 
   const slider = BasicNode.create(
       rootWebArea.find({role: chrome.automation.RoleType.SLIDER}),

chrome/browser/resources/chromeos/accessibility/switch_access/nodes/group_node.js

@@ -42,7 +42,7 @@ export class GroupNode extends SAChildNode {
 
   /** @override */
   get actions() {
-    return [MenuAction.SELECT];
+    return [MenuAction.DRILL_DOWN];
   }
 
   /** @override */
@@ -127,7 +127,7 @@ export class GroupNode extends SAChildNode {
 
   /** @override */
   performAction(action) {
-    if (action === MenuAction.SELECT) {
+    if (action === MenuAction.DRILL_DOWN) {
       Navigator.byItem.enterGroup();
       return ActionResponse.CLOSE_MENU;
     }

chrome/browser/resources/chromeos/accessibility/switch_access/nodes/switch_access_node.ts

@@ -110,7 +110,11 @@ export abstract class SAChildNode {
     if (!this.isFocused_) {
       return;
     }
-    this.performAction(MenuAction.SELECT);
+    if (this.isGroup()) {
+      this.performAction(MenuAction.DRILL_DOWN);
+    } else {
+      this.performAction(MenuAction.SELECT);
+    }
   }
 
   /** Given a menu action, returns whether it can be performed on this node. */

chrome/browser/resources/chromeos/accessibility/switch_access/nodes/tab_node.js

@@ -36,7 +36,7 @@ export class TabNode extends BasicNode {
 
   /** @override */
   get actions() {
-    return [MenuAction.SELECT];
+    return [MenuAction.DRILL_DOWN];
   }
 
   // ================= General methods =================
@@ -53,7 +53,7 @@ export class TabNode extends BasicNode {
 
   /** @override */
   performAction(action) {
-    if (action !== MenuAction.SELECT) {
+    if (action !== MenuAction.DRILL_DOWN) {
       return ActionResponse.NO_ACTION_TAKEN;
     }
     Navigator.byItem.enterGroup();

chrome/browser/resources/chromeos/accessibility/switch_access/nodes/tab_node_test.js

@@ -39,10 +39,10 @@ AX_TEST_F('SwitchAccessTabNodeTest', 'Construction', async function() {
       chrome.automation.RoleType.TAB, tab.role, 'Tab node is not a tab');
   assertTrue(tab.isGroup(), 'Tab node should be a group');
   assertEquals(
-      1, tab.actions.length, 'Tab as a group should have 1 action (select)');
+      1, tab.actions.length, 'Tab as a group should have 1 action (drill down)');
   assertEquals(
-      MenuAction.SELECT, tab.actions[0],
-      'Tab as a group should have the action SELECT');
+      MenuAction.DRILL_DOWN, tab.actions[0],
+      'Tab as a group should have the action DRILL_DOWN');
 
   Navigator.byItem.node_.doDefaultAction();
 

chrome/common/extensions/api/accessibility_private.json

@@ -89,7 +89,7 @@
       {
         "id": "SwitchAccessMenuAction",
         "type": "string",
-        "enum": [ "copy", "cut", "decrement", "dictation", "endTextSelection", "increment", "itemScan", "jumpToBeginningOfText", "jumpToEndOfText", "keyboard", "leftClick", "moveBackwardOneCharOfText", "moveBackwardOneWordOfText", "moveCursor", "moveDownOneLineOfText", "moveForwardOneCharOfText", "moveForwardOneWordOfText", "moveUpOneLineOfText", "paste", "pointScan", "rightClick", "scrollDown", "scrollLeft", "scrollRight", "scrollUp", "select", "settings", "startTextSelection" ],
+        "enum": [ "copy", "cut", "decrement", "dictation", "drillDown", "endTextSelection", "increment", "itemScan", "jumpToBeginningOfText", "jumpToEndOfText", "keyboard", "leftClick", "moveBackwardOneCharOfText", "moveBackwardOneWordOfText", "moveCursor", "moveDownOneLineOfText", "moveForwardOneCharOfText", "moveForwardOneWordOfText", "moveUpOneLineOfText", "paste", "pointScan", "rightClick", "scrollDown", "scrollLeft", "scrollRight", "scrollUp", "select", "settings", "startTextSelection" ],
         "description": "Available actions to be shown in the Switch Access menu. Must be kept in sync with the strings in ash/system/accessibility/switch_access/switch_access_menu_view.cc"
       },
       {

third_party/closure_compiler/externs/accessibility_private.js

@@ -124,6 +124,7 @@ chrome.accessibilityPrivate.SwitchAccessMenuAction = {
   CUT: 'cut',
   DECREMENT: 'decrement',
   DICTATION: 'dictation',
+  DRILL_DOWN: 'drillDown',
   END_TEXT_SELECTION: 'endTextSelection',
   INCREMENT: 'increment',
   ITEM_SCAN: 'itemScan',