chromium/src

Fork 0

Files

android_webview

apps

ash

base

build

build_overrides

buildtools

chrome

chromecast

chromeos

clank

codelabs

components

content

crypto

dbus

device

docs

accessibility

autofill

chromeos

design

enterprise

experiments

fuchsia

gpu

graphics

images

infra

intl

ios

linux

mac

media

memory

memory-infra

patterns

privacy

privacy_budget

process

security

speed

speed_metrics

standards

telemetry_extension

testing

chromeos_integration

images

DIR_METADATA

android_gtests.md

android_instrumentation_tests.md

android_robolectric_tests.md

android_test_instructions.md

batching_instrumentation_tests.md

chromeos_debugging_tips.md

code_coverage.md

code_coverage_in_gerrit.md

expectation_files.md

gtest_flake_tips.md

how_to_repro_bot_failures.md

identifying_tests_that_depend_on_order.md

ipc_fuzzer.md

json_test_results_format.md

life_of_increasing_code_coverage.md

linux_running_asan_tests.md

navbar.md

on_disabling_tests.md

resultdb.md

run_web_platform_tests.md

run_web_platform_tests_on_android.md

test_browser_dialog.md

test_descriptions.md

test_executable_api.md

test_wrapper_api.md

testing_in_chromium.md

using_crashpad_with_content_shell.md

web_platform_tests.md

web_platform_tests_addressing_flake.md

web_test_baseline_fallback.md

web_test_expectations.md

web_tests.md

web_tests_addressing_flake.md

web_tests_in_content_shell.md

web_tests_linux.md

web_tests_tips.md

web_tests_with_manual_fallback.md

writing_web_tests.md

transcripts

updater

webapps

website

webui

workflow

DIR_METADATA

OWNERS

README.md

accessibility.md

ad_tagging.md

adding_to_third_party.md

android_accessing_cpp_enums_in_java.md

android_accessing_cpp_features_in_java.md

android_accessing_cpp_switches_in_java.md

android_build_instructions.md

android_cast_build_instructions.md

android_debugging_instructions.md

android_dynamic_feature_modules.md

android_emulator.md

android_isolated_splits.md

android_jni_ownership_best_practices.md

android_logging.md

android_native_libraries.md

android_studio.md

angle_in_chromium.md

api_keys.md

asan.md

atom.md

benchmark_performance_regressions.md

bfcache.md

bitmap_pipeline.md

branch_gardener.md

building_old_revisions.md

callback.md

ccache_mac.md

chrome_browser_design_principles.md

chrome_os_logging.md

chrome_settings.md

chromedriver_status.md

chromeos_build_instructions.md

chromeos_glossary.md

chromium_browser_vs_google_chrome.md

cipd_and_3pp.md

cl_respect.md

cl_tips.md

clang.md

clang_code_coverage_wrapper.md

clang_format.md

clang_gardening.md

clang_sheriffing.md

clang_static_analyzer.md

clang_tidy.md

clang_tool_refactoring.md

clangd.md

clion.md

closure_compilation.md

cocoa_tips_and_tricks.md

code_review_owners.md

code_reviews.md

commit_checklist.md

component_build.md

configuration.md

contributing.md

cq_fault_attribution.md

cr_respect.md

cr_user_manual.md

cross_platform_ui.md

cygwin_dll_remapping_failure.md

dangling_ptr.md

dangling_ptr_guide.md

dbus_mojo_connection_service.md

debugging_with_crash_keys.md

dependencies.md

deterministic_builds.md

disassemble_code.md

documentation_best_practices.md

documentation_guidelines.md

early-hints.md

eclipse.md

emacs.md

erc_irc.md

flag_expiry.md

flag_guarding_guidelines.md

flag_ownership.md

frame_trees.md

gardener.md

gcs_dependencies.md

gdbinit.md

get_the_code.md

git_cookbook.md

git_submodules.md

git_tips.md

google_chrome_branded_builds.md

google_play_services.md

graphical_debugging_aid_chromium_views.md

gwp_asan.md

history_manipulation_intervention.md

how_cc_works.md

how_to_add_your_feature_flag.md

how_to_extend_web_test_framework.md

idn.md

initialize_blink_features.md

inlined_stack_traces.md

installation_at_vmware.md

ios_build_instructions.md

ios_infra.md

ios_voiceover.md

kiosk_mode.md

life_of_a_frame.md

lldbinit.md

mac_arm64.md

mac_build_instructions.md

mac_lld.md

modifying_session_history_serialization.md

modules.md

mojo_and_services.md

mojo_ipc_conversion.md

mojo_testing.md

native_relocations.md

navbar.md

navigation-request-navigation-state.gv

navigation-request-navigation-state.png

navigation.md

navigation_concepts.md

network_traffic_annotations.md

no_sources_assignment_filter.md

orderfile.md

origin_trials_integration.md

ozone_overview.md

parsing_test_results.md

pgo.md

piranha_plant.md

process_model_and_site_isolation.md

profiling.md

profiling_content_shell_on_android.md

proxy_auto_config.md

qtcreator.md

release_branch_guidance.md

render-frame-host-lifecycle-state.gv

render-frame-host-lifecycle-state.png

render_document.md

rust-unsafe.md

rust.md

seccomp_sandbox_crash_dumping.md

servicification.md

session_history.md

sheriff.md

shutdown.md

special_case_urls.md

static_initializers.md

sublime_ide.md

system_hardening_features.md

tab_helpers.md

threading_and_tasks.md

threading_and_tasks_faq.md

threading_and_tasks_testing.md

toolchain_support.md

tour_of_luci_ui.md

tpm_quick_ref.md

translation_screenshots.md

unretained_dangling_ptr_guide.md

unsafe_buffers.md

updating_clang.md

updating_clang_format_binaries.md

use_counter_wiki.md

useful_urls.md

user_data_dir.md

user_data_storage.md

user_handle_mapping.md

vanilla_msysgit_workflow.md

vscode.md

vscode_python.md

webview_policies.md

win_cross.md

win_order_files.md

windows_build_instructions.md

windows_native_window_occlusion_tracking.md

windows_pwa_integration.md

windows_shortcut_and_taskbar_handling.md

windows_split_dll.md

windows_virtual_desktop_handling.md

wmax_tokens.md

working_remotely_with_android.md

writing_clang_plugins.md

extensions

fuchsia_web

gin

google_apis

gpu

headless

infra

internal

ios

ios_internal

ipc

media

mojo

native_client

native_client_sdk

net

pdf

ppapi

printing

remoting

rlz

sandbox

services

signing_keys

skia

sql

storage

styleguide

testing

third_party

tools

url

webkit

.clang-format

.clang-tidy

.clangd

.git-blame-ignore-revs

.gitallowed

.gitattributes

.gitignore

.gitmodules

.gn

.mailmap

.rustfmt.toml

.vpython3

.yapfignore

ATL_OWNERS

AUTHORS

BUILD.gn

CODE_OF_CONDUCT.md

CPPLINT.cfg

CRYPTO_OWNERS

DEPS

DIR_METADATA

LICENSE

LICENSE.chromium_os

OWNERS

PRESUBMIT.py

PRESUBMIT_test.py

PRESUBMIT_test_mocks.py

README.md

WATCHLISTS

codereview.settings

Mustaq Ahmed 3a40f9f2a4 [CodeHealth] Fix stale link to HTML User Activation spec.

Change-Id: I86c0e79a6b22dbb2d98d5a5d8dd9cacf8baca79a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3935170
Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Nicolás Peña <npm@chromium.org>
Commit-Queue: Mustaq Ahmed <mustaq@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1057589}

2022-10-11 19:16:17 +00:00

5.2 KiB

Raw Blame History

Web Tests with Manual Fallback

Some Blink features cannot be automatically tested using the Web Platform. Prime examples are the APIs that require user activation (also known as a user gesture), such as Fullscreen. Automated tests for these Blink features must rely on special APIs, which are only exposed in testing environments, and are therefore not available in a normal browser session.

A popular pattern used in these tests is to rely on the user to perform some manual steps in order to run the test case in a normal browser session. These tests are effectively manual tests, with additional JavaScript code that automatically performs the desired manual steps, when loaded in an environment that exposes the needed testing APIs.

Motivation

Web tests that degrade to manual tests in the absence of testing APIs have the following benefits.

The manual test component can be debugged in a normal browser session, using the rich developer tools. Tests without a manual fallback can only be debugged in the test runner.
The manual tests can run in other browsers, making it easy to check whether our behavior matches other browsers.
The web tests can form the basis for manual tests that are contributed to web-platform-tests.

Therefore, the desirability of adding a manual fallback to a test heavily depends on whether the feature under test is a Web Platform feature or a Blink-only feature, and on the developer's working style. The benefits above should be weighed against the added design effort needed to build a manual test, and the size and complexity introduced by the manual fallback.

Development Tips

A natural workflow for writing a web test that gracefully degrades to a manual test is to first develop the manual test in a browser, and then add code that feature-checks for testing APIs, and uses them to automate the test's manual steps.

Manual tests should minimize the chance of user error. This implies keeping the manual steps to a minimum, and having simple and clear instructions that describe all the configuration changes and user gestures that match the effect of the Blink-specific APIs used by the test.

Example

Below is an example of a fairly minimal test that uses a Blink-Specific API (window.eventSender), and gracefully degrades to a manual test.

<!doctype html>
<meta charset="utf-8">
<title>DOM: Event.isTrusted for UI events</title>
<link rel="help" href="https://dom.spec.whatwg.org/#dom-event-istrusted">
<link rel="help" href="https://dom.spec.whatwg.org/#constructing-events">
<meta name="assert"
    content="Event.isTrusted is true for events generated by user interaction">
<script src="../../resources/testharness.js"></script>
<script src="../../resources/testharnessreport.js"></script>

<p>Please click on the button below.</p>
<button>Click Me!</button>

<script>
'use strict';

setup({ explicit_timeout: true });

promise_test(() => {
  const button = document.querySelector('button');
  return new Promise((resolve, reject) => {
    const button = document.querySelector('button');
    button.addEventListener('click', (event) => {
      resolve(event);
    });

    if (window.eventSender) {
      eventSender.mouseMoveTo(button.offsetLeft, button.offsetTop);
      eventSender.mouseDown();
      eventSender.mouseUp();
    }
  }).then((clickEvent) => {
    assert_true(clickEvent.isTrusted);
  });

}, 'Click generated by user interaction');

</script>

The test exhibits the following desirable features:

It has a second specification URL (<link rel="help">), because the paragraph that documents the tested feature (referenced by the primary URL) is not very informative on its own.
It links to the WHATWG Living Standard, rather than to a frozen version of the specification.
It contains clear instructions for manually triggering the test conditions. The test starts with a paragraph (<p>) that tells the tester exactly what to do, and the <button> that needs to be clicked is clearly labeled.
It disables the timeout mechanism built into testharness.js by calling setup({ explicit_timeout: true });
It checks for the presence of the Blink-specific testing APIs (window.eventSender) before invoking them. The test does not automatically fail when the APIs are not present.
It uses Promises to separate the test setup from the assertions. This is particularly helpful for manual tests that depend on a sequence of events to occur, as Promises offer a composable way to express waiting for asynchronous events that avoids callback hell.

Notice that the test is pretty heavy compared to a minimal JavaScript test that does not rely on testing APIs. Only use testing APIs when the desired testing conditions cannot be set up using Web Platform APIs.

5.2 KiB Raw Blame History

Web Tests with Manual Fallback

Motivation

Development Tips

Example

5.2 KiB

Raw Blame History