0

Revert "[blinkpy] Add run_{web,wpt}_tests.py --exclude-test-list"

This reverts commit 7f02008ce2.

Reason for revert: <I think we should use a negative test filter instead, otherwise it will not work well with smoke test list.

We can not use two `--test-list` for high_dpi_blink_wpt_tests. Due to test filter has a concept of glob, we can not use the file as both test list and test filter either.

Introduce --inverted-test-launcher-filter-file solved this problem.>

Original change's description:
> [blinkpy] Add `run_{web,wpt}_tests.py --exclude-test-list`
>
> During the migration to wptrunner, this will let Chromium CI use the
> same file to skip tests in one suite, and run them in another via the
> symmetric `--test-list`.
>
> `--exclude-test-list` is similar to `--isolated-script-test-filter-file`
> with every line negated, but the former must contain valid test
> (directories), not just prefixes, and cannot contain negative terms.
>
> Bug: 328079854
> Change-Id: I090a59810fc3fcccd6a2f365bc24c646c85df118
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5760676
> Reviewed-by: Weizhong Xia <weizhong@google.com>
> Commit-Queue: Jonathan Lee <jonathanjlee@google.com>
> Cr-Commit-Position: refs/heads/main@{#1337398}

Bug: 328079854
Change-Id: I75129ee31cc792321657a55ab1310b70ecfa608c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5799817
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Commit-Queue: Weizhong Xia <weizhong@google.com>
Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
Reviewed-by: Jonathan Lee <jonathanjlee@google.com>
Cr-Commit-Position: refs/heads/main@{#1345308}
This commit is contained in:
Weizhong Xia
2024-08-22 09:13:06 +00:00
committed by Chromium LUCI CQ
parent 2543704e29
commit 6994f47722
30 changed files with 138 additions and 193 deletions
BUILD.gn
infra/config
generated
builders
ci
Linux Builder (dbg)
Linux Tests (dbg)(1)
linux-blink-web-tests-force-accessibility-rel
try
linux-blink-web-tests-force-accessibility-rel
linux_chromium_compile_dbg_ng
linux_chromium_dbg_ng
testing
targets
testing/buildbot
third_party/blink

@ -1513,7 +1513,7 @@ if (use_blink && !is_cronet_build) {
# Use --wpt-only to skip tests under web_tests/virtual/
args += [
"--wpt-only",
"--exclude-test-list=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--inverted-test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter",
]
data_deps = [

@ -1295,7 +1295,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -1290,7 +1290,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -121,7 +121,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--flag-specific=force-renderer-accessibility"
],
"merge": {

@ -121,7 +121,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--flag-specific=force-renderer-accessibility"
],
"merge": {

@ -1295,7 +1295,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -1295,7 +1295,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -1644,7 +1644,7 @@
'has_native_resultdb_integration',
],
'args': [
'--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter',
'--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter',
],
'swarming': {
'shards': 1,
@ -1889,7 +1889,7 @@
'has_native_resultdb_integration',
],
'args': [
'--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter',
'--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter',
],
'swarming': {
'shards': 1,

@ -529,7 +529,7 @@ targets.tests.isolated_script_test(
"has_native_resultdb_integration",
],
args = [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter",
],
)

@ -1393,7 +1393,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {

@ -1379,7 +1379,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -3227,7 +3227,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -5148,7 +5148,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {

@ -329,7 +329,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -832,7 +832,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1274,7 +1274,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -20607,7 +20607,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {

@ -38997,7 +38997,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -39083,7 +39083,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -42501,7 +42501,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -46949,7 +46949,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {

@ -1533,7 +1533,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -2157,7 +2157,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -1395,7 +1395,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -3060,7 +3060,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -4725,7 +4725,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -22279,7 +22279,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -23866,7 +23866,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"ci_only": true,
"merge": {
@ -25463,7 +25463,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"ci_only": true,
"merge": {
@ -27126,7 +27126,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"ci_only": true,
"merge": {
@ -28747,7 +28747,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -30313,7 +30313,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -10708,7 +10708,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -10804,7 +10804,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -10903,7 +10903,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter",
"--additional-expectations",
"../../third_party/blink/web_tests/MSANExpectations"
],

@ -59,7 +59,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1917,7 +1917,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -3694,7 +3694,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -5568,7 +5568,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"isolate_profile_data": true,
"merge": {
@ -7471,7 +7471,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -9263,7 +9263,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -11106,7 +11106,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -1908,7 +1908,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -2010,7 +2010,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -2114,7 +2114,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -2219,7 +2219,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -2306,7 +2306,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -1644,7 +1644,7 @@
'has_native_resultdb_integration',
],
'args': [
'--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter',
'--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter',
],
'swarming': {
'shards': 1,
@ -1889,7 +1889,7 @@
'has_native_resultdb_integration',
],
'args': [
'--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter',
'--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter',
],
'swarming': {
'shards': 1,

@ -61,7 +61,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -315,7 +315,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -403,7 +403,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -491,7 +491,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -579,7 +579,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -665,7 +665,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -751,7 +751,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -837,7 +837,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -923,7 +923,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1009,7 +1009,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1095,7 +1095,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1181,7 +1181,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1269,7 +1269,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [
@ -1356,7 +1356,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -64,7 +64,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -59,7 +59,7 @@
},
{
"args": [
"--test-list=../../third_party/blink/web_tests/TestLists/chrome.filter"
"--test-launcher-filter-file=../../third_party/blink/web_tests/TestLists/chrome.filter"
],
"merge": {
"args": [

@ -324,8 +324,9 @@ class Manager(object):
return self._finder.find_tests(
args,
test_lists=self._options.test_list,
exclude_test_lists=self._options.exclude_test_lists,
filter_files=self._options.isolated_script_test_filter_file,
inverted_filter_files=self._options.
inverted_test_launcher_filter_file,
fastest_percentile=self._options.fastest,
filters=self._options.isolated_script_test_filter)

@ -53,56 +53,48 @@ class WebTestFinder(object):
self,
args,
test_lists=None,
exclude_test_lists=None,
filter_files=None,
inverted_filter_files=None,
fastest_percentile=None,
filters=None,
):
# The second entry of each element is true if the path is to be
# included, and false if excluded.
paths = [(path, True) for path in self._strip_test_dir_prefixes(args)]
filters = filters or []
paths = self._strip_test_dir_prefixes(args)
if test_lists:
new_paths = self._read_test_list_files(
test_lists, self._port.TEST_PATH_SEPARATOR)
paths.extend((path, True) for path in new_paths)
new_paths = [
self._strip_test_dir_prefix(new_path) for new_path in new_paths
]
paths += new_paths
all_tests = []
if not paths or fastest_percentile:
all_tests = self._port.tests(None)
test_files = set()
path_tests = []
if paths:
path_tests = self._port.tests(paths)
test_files = None
running_all_tests = False
if fastest_percentile:
times_trie = self._times_trie()
if times_trie:
fastest_tests = self._fastest_tests(times_trie, all_tests,
fastest_percentile)
test_files.update(fastest_tests)
test_files = list(set(fastest_tests).union(path_tests))
else:
_log.warning(
'Running all the tests the first time to generate timing data.'
)
test_files = all_tests
running_all_tests = True
elif paths:
test_files = path_tests
else:
running_all_tests = not paths and not test_lists
if running_all_tests:
test_files.update(all_tests)
if exclude_test_lists:
new_paths = self._read_test_list_files(
exclude_test_lists, self._port.TEST_PATH_SEPARATOR)
paths.extend((path, False) for path in new_paths)
if new_paths:
running_all_tests = False
for path, include in sorted(paths):
path_tests = self._port.tests([path])
if include:
test_files.update(path_tests)
else:
test_files -= set(path_tests)
test_files = sorted(test_files)
test_files = all_tests
running_all_tests = True
all_filters = []
if filters:
@ -111,13 +103,16 @@ class WebTestFinder(object):
file_filters = self._read_filter_files(
filter_files, self._port.TEST_PATH_SEPARATOR)
all_filters = all_filters + file_filters
if inverted_filter_files:
file_filters = self._read_filter_files(
inverted_filter_files, self._port.TEST_PATH_SEPARATOR)
all_filters = all_filters + self._invert_filters(file_filters)
test_files = filter_tests(test_files, all_filters)
# de-dupe the test list and paths here before running them.
test_files = list(OrderedDict.fromkeys(test_files))
paths = list(
OrderedDict.fromkeys(path for path, include in paths if include))
paths = list(OrderedDict.fromkeys(paths))
return (paths, test_files, running_all_tests)
def _times_trie(self):
@ -194,6 +189,20 @@ class WebTestFinder(object):
raise
return filters
def _invert_filters(self, filters):
inverted_filters = []
for terms in filters:
inverted_filter = []
for term in terms:
if term.startswith('-'):
inverted_filter.append(term[1:])
elif term.startswith('+'):
inverted_filter.append('-' + term[1:])
else:
inverted_filter.append('-' + term)
inverted_filters.append(inverted_filter)
return inverted_filters
def _read_test_list_files(self, filenames, test_path_separator):
fs = self._filesystem
positive_matches = []
@ -210,13 +219,11 @@ class WebTestFinder(object):
_log.debug(
'test-list %s contains a negative filter %s' %
(filename, line))
continue
positive_matches.append(self._strip_test_dir_prefix(line))
positive_matches.append(line)
except IOError as error:
if error.errno == errno.ENOENT:
_log.critical('')
_log.critical('--(exclude-)test-list file "%s" not found',
filename)
_log.critical('--test-list file "%s" not found', filename)
raise
return positive_matches
@ -371,9 +378,9 @@ def filter_tests(tests, filters):
else:
return (len(k), k)
for filter in filters:
for terms in filters:
# Validate the filter
for term in filter:
for term in terms:
if (term.startswith('-') and not term[1:]) or not term:
raise ValueError('Empty filter entry "%s"' % (term, ))
for i, c in enumerate(term):
@ -383,13 +390,13 @@ def filter_tests(tests, filters):
raise ValueError('Bad test filter "%s" specified; '
'unescaped wildcards are only allowed at '
'the end' % (term, ))
if term.startswith('-') and term[1:] in filter:
if term.startswith('-') and term[1:] in terms:
raise ValueError('Both "%s" and "%s" specified in test '
'filter' % (term, term[1:]))
# Separate the negative/positive globless terms and glob terms
include_by_default = all(term.startswith('-') for term in filter)
exact_neg_terms, exact_pos_terms, glob_terms = _extract_terms(filter)
include_by_default = all(term.startswith('-') for term in terms)
exact_neg_terms, exact_pos_terms, glob_terms = _extract_terms(terms)
filtered_tests = []
for test in tests:

@ -3,9 +3,7 @@
# found in the LICENSE file.
import optparse
import textwrap
import unittest
from unittest import mock
from blinkpy.common.host_mock import MockHost
from blinkpy.common.system.filesystem_mock import MockFileSystem
@ -155,6 +153,8 @@ class WebTestFinderTests(unittest.TestCase):
port = host.port_factory.get('test-win-win7', None)
all_tests = [
'path/test.html',
'new/test.html',
'fast/css/1.html',
'fast/css/2.html',
'fast/css/3.html',
@ -163,8 +163,6 @@ class WebTestFinderTests(unittest.TestCase):
'fast/css/skip3.html',
'fast/css/skip4.html',
'fast/css/skip5.html',
'new/test.html',
'path/test.html',
]
port.tests = lambda paths: paths or all_tests
@ -289,85 +287,31 @@ class WebTestFinderTests(unittest.TestCase):
set(tests[1]),
set(['path/test.html','virtual/path/test.html',]))
def test_empty_test_list_find_tests(self):
host = MockHost()
port = host.port_factory.get('test-win-win7')
host.filesystem.write_text_file(
'test-list.txt',
textwrap.dedent("""\
# this file is deliberately empty to signal no tests to run
"""))
all_tests = [
'path/test.html',
]
finder = web_test_finder.WebTestFinder(port, {})
with mock.patch.object(port, 'tests',
lambda paths: paths or all_tests):
paths, all_test_names, running_all_tests = finder.find_tests(
args=[], test_lists=['test-list.txt'])
self.assertEqual(paths, [])
self.assertEqual(all_test_names, [])
self.assertFalse(running_all_tests)
def test_exclude_test_list_find_tests(self):
def test_inverted_test_filter_find_tests(self):
host = MockHost()
port = host.port_factory.get('test-win-win7', None)
host.filesystem.write_text_file(
'test-list.txt',
textwrap.dedent("""\
path/test.html
"""))
all_tests = [
mock_files = {
'test-list.txt': 'path/test.html\nvirtual/path/test.html',
'inverted-filter.txt': 'path/test.html'
}
host.filesystem = MockFileSystem(files=mock_files)
port_tests = [
'path/test.html',
'virtual/fake-vts/path/test.html',
'not/in/test/list.html',
]
finder = web_test_finder.WebTestFinder(port, {})
with mock.patch.object(port, 'tests',
lambda paths: paths or all_tests):
paths, all_test_names, running_all_tests = finder.find_tests(
args=[], exclude_test_lists=['test-list.txt'])
self.assertEqual(paths, [])
self.assertEqual(set(all_test_names),
{'virtual/fake-vts/path/test.html'})
self.assertFalse(running_all_tests)
def test_include_and_exclude_test_list_find_tests(self):
host = MockHost()
port = host.port_factory.get('test-win-win7', None)
fs = host.filesystem
fs.write_text_file(
'include.txt',
textwrap.dedent("""\
http/tests
failures/flaky
"""))
fs.write_text_file(
'exclude.txt',
textwrap.dedent("""\
http/tests/passes # excludes two tests
# these lines should have no effect
passes/skipped
failures
"""))
port.tests = lambda paths: paths or port_tests
finder = web_test_finder.WebTestFinder(port, {})
paths, all_test_names, running_all_tests = finder.find_tests(
tests = finder.find_tests(
args=[],
test_lists=['include.txt'],
exclude_test_lists=['exclude.txt'])
self.assertEqual(set(paths), {'http/tests', 'failures/flaky'})
self.assertEqual(set(all_test_names), {
'http/tests/ssl/text.html',
'failures/flaky/text.html',
})
self.assertFalse(running_all_tests)
test_lists=['test-list.txt'],
inverted_filter_files=['inverted-filter.txt'])
self.assertEqual(set(tests[1]), set([
'virtual/path/test.html',
]))
class FilterTestsTests(unittest.TestCase):
simple_test_filter = ['a/a1.html', 'a/a2.html', 'b/b1.html']

@ -452,13 +452,10 @@ def add_testing_options_group(parser: argparse.ArgumentParser,
metavar='FILE',
help='read filters for tests to run')
testing_group.add_argument(
'--exclude-test-list',
dest='exclude_test_lists',
'--inverted-test-launcher-filter-file',
action='append',
metavar='FILE',
help=('read test (directories) that should not run '
'(the most specific prefix among `--(exclude-)test-list` and '
'the positional arguments wins)'))
help=('Filters in the file will be inverted before applied.'))
testing_group.add_argument(
'--isolated-script-test-filter-file',
'--test-launcher-filter-file',

@ -183,15 +183,10 @@ def _set_up_derived_options(port, options, args):
port.host.filesystem.abspath(path))
options.additional_platform_directory = additional_platform_directories
# Do not set has_explicit_tests to true when exclude_test_lists is provided.
# For builders with an implicit smoke test list, e.g. highdpi flag specific
# builder, the smoke test list should always be honored, no matter
# exclude_test_lists exists or not.
has_explicit_tests = args or options.test_list
if not has_explicit_tests and options.smoke is None:
if not args and not options.test_list and options.smoke is None:
options.smoke = port.default_smoke_test_only()
if options.smoke:
if not has_explicit_tests and options.num_retries is None:
if not args and not options.test_list and options.num_retries is None:
# Retry failures 3 times if we're running a smoke test without
# additional tests. SmokeTests is an explicit list of tests, so we
# wouldn't retry by default without this special case.

@ -398,8 +398,9 @@ class WPTAdapter:
self.paths, all_test_names, _ = finder.find_tests(
self.paths,
test_lists=self.options.test_list,
exclude_test_lists=self.options.exclude_test_lists,
filter_files=self.options.isolated_script_test_filter_file,
inverted_filter_files=self.options.
inverted_test_launcher_filter_file,
fastest_percentile=None,
filters=self.options.isolated_script_test_filter)
except IOError:

@ -8,4 +8,4 @@
# https://chromium.googlesource.com/chromium/src/+/HEAD/docs/testing/run_web_platform_tests.md
# crbug.com/336866597 set_spc_transaction_mode not implemented for content shell
external/wpt/secure-payment-confirmation/
external/wpt/secure-payment-confirmation/*