pixel_tests: refactor check_pixel_test_flakiness.py
Use pathlib.Path everywhere possible. Allow passing in an absolute root_dir such as /tmp/skia_gold. Change some return values to exception based control flow handling - this causes the script to exit with non-zero return code when failure is found. Also run `git cl format --python --full` on the file. Bug: None Test: check_pixel_test_flakiness.py still runs examples Change-Id: I455d66ba153d00a9b42b7db1b25e1979cb3498b4 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4477811 Reviewed-by: Andrew Xu <andrewxu@chromium.org> Commit-Queue: Jeffrey Young <cowmoo@google.com> Cr-Commit-Position: refs/heads/main@{#1136828}
This commit is contained in:
Jeffrey Young
committed by
Chromium LUCI CQ
Chromium LUCI CQ
parent
ca773d7fbf
commit
252acffea0
@ -47,6 +47,14 @@ command uses the option --root_dir to designate the root path for outputs
|
||||
detected). In this example, the absolute path of the output directory is
|
||||
.../chromium/../var rather than .../chromium/var.
|
||||
|
||||
./tools/pixel_test/check_pixel_test_flakiness.py --gtest_filter=\
|
||||
*PersonalizationAppIntegrationPixel* --test_target=out/debug/browser_tests
|
||||
--root_dir=/tmp/skia_gold --output_dir=var --browser-ui-tests-verify-pixels
|
||||
--enable-pixel-output-in-tests
|
||||
|
||||
Finally, the above command runs the browser_tests target and adds extra
|
||||
arguments necessary for experimental browser pixel tests to run properly.
|
||||
|
||||
* options:
|
||||
|
||||
--test_target: it specifies the path to the executable file of pixel tests. It
|
||||
@ -63,12 +71,14 @@ only error messages from gTest runs are printed; 'all' shows all logs.
|
||||
'none' is used by default.
|
||||
|
||||
--gtest_repeat: it specifies the count of repeated runs. Use ten by default.
|
||||
"""
|
||||
|
||||
Any additional unknown args, such as --browser-ui-test-verify-pixels, are passed
|
||||
to the gtest runner.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
@ -82,258 +92,254 @@ _ENDC = '\033[0m'
|
||||
_TEMP_DIRECTORY_NAME_BASE = '@@check_pixel_test_flakiness!#'
|
||||
|
||||
|
||||
def _get_md5(absolute_file_path):
|
||||
"""Returns the Md5 digest of the specified file."""
|
||||
with open(absolute_file_path, 'rb') as target_file:
|
||||
return hashlib.md5(target_file.read()).hexdigest()
|
||||
class FlakyScreenshotError(Exception):
|
||||
"""One of the screenshots has been detected to be flaky."""
|
||||
|
||||
|
||||
class MissingScreenshotsError(Exception):
|
||||
"""There were no screenshots found."""
|
||||
|
||||
|
||||
def _get_md5(path):
|
||||
"""Returns the Md5 digest of the specified file."""
|
||||
if not path.is_absolute():
|
||||
raise ValueError(f'{path} must be absolute')
|
||||
|
||||
with path.open(mode='rb') as target_file:
|
||||
return hashlib.md5(target_file.read()).hexdigest()
|
||||
|
||||
|
||||
def _compare_with_last_iteration(screenshots, prev_temp_dir, temp_dir,
|
||||
names_hash_mappings, flaky_screenshot_dir):
|
||||
"""Compares the screenshots generated in the current iteration with those
|
||||
from the previous iteration. If flakiness is detected, returns a flaky
|
||||
screenshot's name. Otherwise, returns an empty string.
|
||||
"""Compares the screenshots generated in the current iteration with those
|
||||
from the previous iteration. If flakiness is detected, returns a flaky
|
||||
screenshot's name. Otherwise, returns an empty string.
|
||||
|
||||
Args:
|
||||
screenshots: A list of screenshot names.
|
||||
prev_temp_dir: The absolute file path to the directory that hosts the
|
||||
screenshots generated by the previous iteration.
|
||||
temp_dir: The absolute file path to the directory that hosts the
|
||||
screenshots generated by the current iteration.
|
||||
names_hash_mappings: The mappings from screenshot names to hash code.
|
||||
flaky_screenshot_dir: The absolute file path to the directory used to host
|
||||
flaky screenshots. If it is null, flaky screenshots are not written into
|
||||
files.
|
||||
Args:
|
||||
screenshots: A list of screenshot Paths.
|
||||
prev_temp_dir: The absolute file path to the directory that hosts the
|
||||
screenshots generated by the previous iteration.
|
||||
temp_dir: The absolute file path to the directory that hosts the screenshots
|
||||
generated by the current iteration.
|
||||
names_hash_mappings: The mappings from screenshot names to hash code.
|
||||
flaky_screenshot_dir: The absolute file path to the directory used to host
|
||||
flaky screenshots. If it is None, flaky screenshots are not written into
|
||||
files.
|
||||
|
||||
Returns:
|
||||
A string that indicates the name of the flaky screenshot. If no flakiness
|
||||
is detected, the return string is empty.
|
||||
Returns: None
|
||||
|
||||
Raises:
|
||||
FlakyScreenshotError: if screenshots from prev_temp_dir do not match
|
||||
temp_dir
|
||||
"""
|
||||
|
||||
if prev_temp_dir is None:
|
||||
raise TypeError('prev_temp_dir is required to be a valid Path')
|
||||
|
||||
if not screenshots:
|
||||
raise ValueError('screenshots must be non-empty')
|
||||
|
||||
for screenshot in screenshots:
|
||||
# The screenshot hash code does not change so no flakiness is detected
|
||||
# on `screenshot`.
|
||||
if names_hash_mappings[screenshot.name] == _get_md5(screenshot):
|
||||
continue
|
||||
|
||||
if flaky_screenshot_dir is not None:
|
||||
# Delete the output directory if it already exists.
|
||||
if flaky_screenshot_dir.exists():
|
||||
shutil.rmtree(flaky_screenshot_dir)
|
||||
|
||||
flaky_screenshot_dir.mkdir(parents=True)
|
||||
|
||||
# Move the screenshot generated by the last iteration to the dest
|
||||
# directory.
|
||||
shutil.move(
|
||||
prev_temp_dir / screenshot.name, flaky_screenshot_dir /
|
||||
f'{screenshot.stem}_Version_1{screenshot.suffix}')
|
||||
|
||||
# Move the screenshot generated by the current iteration to the dest
|
||||
# directory.
|
||||
shutil.move(
|
||||
screenshot, flaky_screenshot_dir /
|
||||
f'{screenshot.stem}_Version_2{screenshot.suffix}')
|
||||
|
||||
raise FlakyScreenshotError(
|
||||
f'{_FAIL_RED}[Failure]{_ENDC} Detect flakiness in: {screenshot.name}')
|
||||
|
||||
# No flakiness detected.
|
||||
return None
|
||||
|
||||
|
||||
def _analyze_screenshots(prev_temp_dir, temp_dir, names_hash_mappings,
|
||||
flaky_screenshot_dir):
|
||||
"""Analyzes the screenshots generated by one iteration.
|
||||
|
||||
Args:
|
||||
prev_temp_dir: The absolute file path to the directory that hosts the
|
||||
screenshots generated by the previous iteration, or None if this is the
|
||||
first iteration.
|
||||
temp_dir: The absolute file path to the directory that hosts the screenshots
|
||||
generated by the current iteration.
|
||||
names_hash_mappings: The mappings from screenshot names to hash code.
|
||||
flaky_screenshot_dir: The absolute file path to the directory used to host
|
||||
flaky screenshots. If it is None, flaky screenshots are not written into
|
||||
files.
|
||||
|
||||
Returns: None
|
||||
|
||||
Raises:
|
||||
FlakyScreenshotError
|
||||
MissingScreenshotsError
|
||||
"""
|
||||
screenshots = list(temp_dir.iterdir())
|
||||
if not screenshots:
|
||||
raise MissingScreenshotsError(
|
||||
f'{_FAIL_RED}[Failure]{_ENDC} no screenshots are generated in the '
|
||||
'specified tests: are you using the correct test filter?')
|
||||
|
||||
# For the first iteration, nothing to compare with. Therefore, fill
|
||||
# `names_hash_mappings` and return.
|
||||
if prev_temp_dir is None:
|
||||
for screenshot in screenshots:
|
||||
screenshot_path = os.path.join(temp_dir, screenshot)
|
||||
names_hash_mappings[screenshot.name] = _get_md5(screenshot)
|
||||
return
|
||||
|
||||
# The screenshot hash code does not change so no flakiness is detected
|
||||
# on `screenshot`.
|
||||
if names_hash_mappings[screenshot] == _get_md5(screenshot_path):
|
||||
continue
|
||||
|
||||
# Return if writing flaky screenshots to files is not required.
|
||||
if flaky_screenshot_dir is None:
|
||||
return screenshot
|
||||
|
||||
# Delete the output directory if it already exists.
|
||||
if os.path.exists(flaky_screenshot_dir):
|
||||
shutil.rmtree(flaky_screenshot_dir)
|
||||
|
||||
os.mkdir(flaky_screenshot_dir)
|
||||
split_image_name = os.path.splitext(screenshot)
|
||||
|
||||
# Move the screenshot generated by the last iteration to the dest
|
||||
# directory.
|
||||
shutil.move(
|
||||
os.path.join(prev_temp_dir, screenshot),
|
||||
os.path.join(
|
||||
flaky_screenshot_dir,
|
||||
split_image_name[0] + '_Version_1' + split_image_name[1]))
|
||||
|
||||
# Move the screenshot generated by the current iteration to the dest
|
||||
# directory.
|
||||
shutil.move(
|
||||
screenshot_path,
|
||||
os.path.join(
|
||||
flaky_screenshot_dir,
|
||||
split_image_name[0] + '_Version_2' + split_image_name[1]))
|
||||
|
||||
return screenshot
|
||||
|
||||
# No flakiness detected.
|
||||
return ''
|
||||
|
||||
|
||||
def _analyze_screenshots(iteration_index, prev_temp_dir, temp_dir,
|
||||
names_hash_mappings, flaky_screenshot_dir):
|
||||
"""Analyzes the screenshots generated by one iteration.
|
||||
|
||||
Args:
|
||||
iteration_index: An integer that indicates the iteration index.
|
||||
prev_temp_dir: The absolute file path to the directory that hosts the
|
||||
screenshots generated by the previous iteration.
|
||||
temp_dir: The absolute file path to the directory that hosts the
|
||||
screenshots generated by the current iteration.
|
||||
names_hash_mappings: The mappings from screenshot names to hash code.
|
||||
flaky_screenshot_dir: The absolute file path to the directory used to host
|
||||
flaky screenshots. If it is null, flaky screenshots are not written into
|
||||
files.
|
||||
|
||||
Returns: A boolean value that indicates the execution result. True if
|
||||
flakiness is detected or the specified pixel tests do not generate any
|
||||
screenshot.
|
||||
"""
|
||||
screenshots = os.listdir(temp_dir)
|
||||
|
||||
# For the first iteration, nothing to compare with. Therefore, fill
|
||||
# `previous_temp_data_dir` and `names_hash_mappings`.
|
||||
if iteration_index == 0:
|
||||
if not screenshots:
|
||||
print(_FAIL_RED + '[Failure]' + _ENDC + ' no screenshots are '
|
||||
'generated in the specified tests: are you using the '
|
||||
'correct test filter?')
|
||||
return True
|
||||
|
||||
for screenshot in screenshots:
|
||||
screenshot_absolute_path = os.path.join(temp_dir, screenshot)
|
||||
names_hash_mappings[screenshot] = _get_md5(
|
||||
screenshot_absolute_path)
|
||||
print(_OK_GREEN + '[OK]' + _ENDC + ' the iteration ' +
|
||||
str(iteration_index) + ' succeeds')
|
||||
return False
|
||||
|
||||
flaky_image_name = _compare_with_last_iteration(screenshots, prev_temp_dir,
|
||||
temp_dir, names_hash_mappings,
|
||||
flaky_screenshot_dir)
|
||||
|
||||
if len(flaky_image_name) > 0:
|
||||
print(_FAIL_RED + '[Failure]' + _ENDC + ' Detect flakiness in: ' +
|
||||
flaky_image_name)
|
||||
return True
|
||||
|
||||
print(_OK_GREEN + '[OK]' + _ENDC + ' the iteration ' +
|
||||
str(iteration_index) + ' succeeds')
|
||||
|
||||
return False
|
||||
_compare_with_last_iteration(screenshots, prev_temp_dir, temp_dir,
|
||||
names_hash_mappings, flaky_screenshot_dir)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Detect flakiness in the Skia Gold based pixel tests by '
|
||||
'running the specified pixel test executable file multiple iterations '
|
||||
'and comparing screenshots generated by neighboring iterations through '
|
||||
'file hash code. Warning: this script can only be used to detect '
|
||||
'flakiness in the pixel tests that use precise comparison.')
|
||||
parser.add_argument('--test_target', type=str, required=True, help='a '
|
||||
'relative file path from the current working directory '
|
||||
'to the test executable based on Skia Gold, such as '
|
||||
'ash_pixeltests')
|
||||
parser.add_argument('--gtest_repeat', type=int, default=10, help='the count'
|
||||
' of the repeated runs. The default value is ten.')
|
||||
parser.add_argument('--root_dir', type=str, default='', help='a relative '
|
||||
'file path from the current working directory to the '
|
||||
'root directory that hosts output data including the '
|
||||
'screenshots generated in each iteration and the '
|
||||
'detected flaky screenshots')
|
||||
parser.add_argument('--output_dir', type=str, help='a relative path'
|
||||
' starting from the output root path specified by'
|
||||
' --root_dir or the current working directory if'
|
||||
' --root_dir is omitted. It specifies a directory used'
|
||||
' to host the flaky screenshots if any.')
|
||||
parser.add_argument('--log_mode',
|
||||
choices=['none', 'error_only', 'all'],
|
||||
default='none', help='the option to control the log '
|
||||
'output during test runs. `none` means that the log '
|
||||
'generated by test runs does not show; `error_only` '
|
||||
'means that only error logs are printed; `all` shows '
|
||||
'all logs. `none` is used by default.')
|
||||
[known_args, unknown_args] = parser.parse_known_args()
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Detect flakiness in the Skia Gold based pixel tests by '
|
||||
'running the specified pixel test executable file multiple iterations '
|
||||
'and comparing screenshots generated by neighboring iterations through '
|
||||
'file hash code. Warning: this script can only be used to detect '
|
||||
'flakiness in the pixel tests that use precise comparison.')
|
||||
parser.add_argument(
|
||||
'--test_target',
|
||||
type=str,
|
||||
required=True,
|
||||
help='a '
|
||||
'relative file path from the current working directory, or absolute file '
|
||||
'path, to the test executable based on Skia Gold, such as ash_pixeltests')
|
||||
parser.add_argument('--gtest_repeat',
|
||||
type=int,
|
||||
default=10,
|
||||
help='the count of the repeated runs. The default value '
|
||||
'is ten.')
|
||||
parser.add_argument('--root_dir',
|
||||
type=str,
|
||||
default='',
|
||||
help='a relative file path from the current working '
|
||||
'directory, or an absolute path, to the root directory '
|
||||
'that hosts output data including the screenshots '
|
||||
'generated in each iteration and the detected flaky '
|
||||
'screenshots')
|
||||
parser.add_argument('--output_dir',
|
||||
type=str,
|
||||
help='a relative path starting from the output root path '
|
||||
'specified by --root_dir or the current working '
|
||||
'directory if --root_dir is omitted. It specifies a '
|
||||
'directory used to host the flaky screenshots if any.')
|
||||
parser.add_argument('--log_mode',
|
||||
choices=['none', 'error_only', 'all'],
|
||||
default='none',
|
||||
help='the option to control the log output during test '
|
||||
'runs. `none` means that the log generated by test runs '
|
||||
'does not show; `error_only` means that only error logs '
|
||||
'are printed; `all` shows all logs. `none` is used by '
|
||||
'default.')
|
||||
[known_args, unknown_args] = parser.parse_known_args()
|
||||
|
||||
# Calculate the absolute path to the pixel test executable file.
|
||||
cwd = os.getcwd()
|
||||
executable_full_path = os.path.join(cwd, known_args.test_target)
|
||||
# Calculate the absolute path to the pixel test executable file.
|
||||
executable_full_path = pathlib.Path(known_args.test_target).resolve()
|
||||
|
||||
# Calculate the absolute path to the directory that hosts output data.
|
||||
output_root_path = os.path.join(cwd, known_args.root_dir)
|
||||
# Calculate the absolute path to the directory that hosts output data.
|
||||
output_root_path = pathlib.Path(known_args.root_dir).resolve()
|
||||
|
||||
# Skip the Skia Gold functionality. Because this script compares images
|
||||
# through hash code.
|
||||
pixel_test_command_base = [
|
||||
executable_full_path, '--bypass-skia-gold-functionality'
|
||||
]
|
||||
# Skip the Skia Gold functionality. Because this script compares images
|
||||
# through hash code.
|
||||
pixel_test_command_base = [
|
||||
str(executable_full_path), '--bypass-skia-gold-functionality'
|
||||
]
|
||||
|
||||
# Pass unknown args to gtest.
|
||||
if unknown_args:
|
||||
pixel_test_command_base += unknown_args
|
||||
# Pass unknown args to gtest.
|
||||
if unknown_args:
|
||||
pixel_test_command_base += unknown_args
|
||||
|
||||
# Print the command to run pixel tests.
|
||||
print(_OK_GREEN + '[Begin] ' + _ENDC + ' '.join(pixel_test_command_base))
|
||||
# Print the command to run pixel tests.
|
||||
full_command = ' '.join(pixel_test_command_base)
|
||||
print(f'{_OK_GREEN}[Begin]{_ENDC} {full_command}')
|
||||
|
||||
# Configure log output.
|
||||
std_out_mode = subprocess.DEVNULL
|
||||
if known_args.log_mode == 'all':
|
||||
std_out_mode = None
|
||||
std_err_mode = None
|
||||
if known_args.log_mode == 'none':
|
||||
std_err_mode = subprocess.DEVNULL
|
||||
# Configure log output.
|
||||
std_out_mode = subprocess.DEVNULL
|
||||
if known_args.log_mode == 'all':
|
||||
std_out_mode = None
|
||||
std_err_mode = None
|
||||
if known_args.log_mode == 'none':
|
||||
std_err_mode = subprocess.DEVNULL
|
||||
|
||||
# Cache the screenshot host directory used in the last iteration. It updates
|
||||
# at the end of each iteration.
|
||||
prev_temp_dir = ''
|
||||
# Cache the screenshot host directory used in the last iteration. It updates
|
||||
# at the end of each iteration.
|
||||
prev_temp_dir = None
|
||||
|
||||
# Similar to `prev_temp_dir` but it caches data for the active
|
||||
# iteration.
|
||||
temp_dir = ''
|
||||
# Similar to `prev_temp_dir` but it caches data for the active
|
||||
# iteration.
|
||||
temp_dir = None
|
||||
|
||||
# Mappings screenshot names to hash code.
|
||||
names_hash_mappings = {}
|
||||
# Mappings screenshot names to hash code.
|
||||
names_hash_mappings = {}
|
||||
|
||||
# Calculate the directory path for saving flaky screenshots.
|
||||
flaky_screenshot_dir = None
|
||||
if known_args.output_dir is not None:
|
||||
flaky_screenshot_dir = os.path.join(
|
||||
output_root_path, known_args.output_dir)
|
||||
# Calculate the directory path for saving flaky screenshots.
|
||||
flaky_screenshot_dir = None
|
||||
if known_args.output_dir is not None:
|
||||
flaky_screenshot_dir = output_root_path / known_args.output_dir
|
||||
|
||||
try:
|
||||
for iteration_index in range(known_args.gtest_repeat):
|
||||
# Calculate the absolute path to the screenshot host directory used for
|
||||
# this iteration. Recreate the host directory if it already exists.
|
||||
temp_dir = os.path.join(
|
||||
output_root_path, _TEMP_DIRECTORY_NAME_BASE + str(iteration_index))
|
||||
try:
|
||||
|
||||
# Ensure `temp_dir` is an absolute path. Otherwise, screenshots
|
||||
# generated during test runs will fail to be written into files.
|
||||
temp_dir = os.path.abspath(temp_dir)
|
||||
for i in range(known_args.gtest_repeat):
|
||||
# Calculate the absolute path to the screenshot host directory used for
|
||||
# this iteration. Recreate the host directory if it already exists.
|
||||
temp_dir = output_root_path / f'{_TEMP_DIRECTORY_NAME_BASE}{i}'
|
||||
|
||||
if os.path.exists(temp_dir):
|
||||
shutil.rmtree(temp_dir)
|
||||
os.mkdir(temp_dir)
|
||||
if temp_dir.exists():
|
||||
shutil.rmtree(temp_dir)
|
||||
temp_dir.mkdir(parents=True)
|
||||
|
||||
# Append the option so that the screenshots generated in pixel tests are
|
||||
# written into `temp_dir`.
|
||||
pixel_test_command = pixel_test_command_base[:]
|
||||
pixel_test_command.append('--skia-gold-local-png-write-directory=' +
|
||||
temp_dir)
|
||||
# Append the option so that the screenshots generated in pixel tests are
|
||||
# written into `temp_dir`.
|
||||
pixel_test_command = pixel_test_command_base[:]
|
||||
pixel_test_command.append(
|
||||
f'--skia-gold-local-png-write-directory={temp_dir}')
|
||||
|
||||
# Run pixel tests.
|
||||
subprocess.run(pixel_test_command,
|
||||
stdout=std_out_mode,
|
||||
stderr=std_err_mode,
|
||||
check=True)
|
||||
# Run pixel tests.
|
||||
subprocess.run(pixel_test_command,
|
||||
stdout=std_out_mode,
|
||||
stderr=std_err_mode,
|
||||
check=True)
|
||||
|
||||
result = _analyze_screenshots(
|
||||
iteration_index, prev_temp_dir, temp_dir, names_hash_mappings,
|
||||
flaky_screenshot_dir)
|
||||
_analyze_screenshots(prev_temp_dir, temp_dir, names_hash_mappings,
|
||||
flaky_screenshot_dir)
|
||||
|
||||
# Get an error so exit the loop.
|
||||
if result:
|
||||
shutil.rmtree(temp_dir)
|
||||
break
|
||||
print(f'{_OK_GREEN}[OK]{_ENDC} the iteration {i} succeeds')
|
||||
|
||||
# Delete the temporary data directory used by the previous loop
|
||||
# iteration before overwriting it.
|
||||
if iteration_index > 0:
|
||||
shutil.rmtree(prev_temp_dir)
|
||||
# Delete the temporary data directory used by the previous loop iteration
|
||||
# before overwriting it.
|
||||
if prev_temp_dir is not None:
|
||||
shutil.rmtree(prev_temp_dir)
|
||||
|
||||
prev_temp_dir = temp_dir
|
||||
prev_temp_dir = temp_dir
|
||||
else:
|
||||
# The for loop has finished without exceptions.
|
||||
print(f'{_OK_GREEN}[Success]{_ENDC} no flakiness is detected')
|
||||
|
||||
# All iterations end. Print the success message.
|
||||
if iteration_index == known_args.gtest_repeat - 1:
|
||||
print(_OK_GREEN + '[Success]' + _ENDC +
|
||||
' no flakiness is detected')
|
||||
|
||||
finally:
|
||||
# ensure that temp data are removed.
|
||||
if os.path.isdir(prev_temp_dir):
|
||||
shutil.rmtree(prev_temp_dir)
|
||||
finally:
|
||||
# ensure that temp data are removed.
|
||||
for dir_to_rm in (prev_temp_dir, temp_dir):
|
||||
if dir_to_rm is not None and dir_to_rm.exists():
|
||||
shutil.rmtree(dir_to_rm)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user