chromium/src
0
Fork 0
Code Activity

Refactor GPU Skia Gold code

Browse Source 
Refactors the Skia Gold-related code in //content/test/gpu/gpu_tests
to be the same as //build/android/pylib. All the code that interacts
with Gold is encapsulated in its own class, and users simply run one
method and check its output to perform an image comparison.

Also adds a bunch of unittests (also taken from //build/android/pylib)
since the split makes the code unittest-able.

Drive-by fixes a related TODO in //build/android since the same TODO
was fixed in the GPU code in this CL.

Bug: 1093994
Change-Id: Id2fe1506796695f258069334149550cfaea49f71
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2243718
Reviewed-by: Tibor Goldschwendt <tiborg@chromium.org>
Reviewed-by: Zhenyao Mo <zmo@chromium.org>
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#778514}
This commit is contained in:
Brian Sheedy
2020-06-15 21:49:33 +00:00
committed by Commit Bot
parent b8fdd110df
commit 2df4e14530
13 changed files with 1712 additions and 247 deletions

19
build/android/pylib/utils/gold_utils.py

@ -1,7 +1,12 @@
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utilities for interacting with the Skia Gold image diffing service."""
"""Utilities for interacting with the Skia Gold image diffing service.
The files in //content/test/gpu/gpu_tests/skia_gold are heavily based on this.
If you need to make a change to this file, check to see if the same change needs
to be made there.
"""
import json
import logging
@ -260,16 +265,10 @@ class SkiaGoldSession(object):
self._comparison_results[name].triage_link_omission_reason = (
'Comparison succeeded, no triage link')
elif self._gold_properties.IsTryjobRun():
# TODO(skbug.com/9879): Remove the explicit corpus when Gold's UI is
# updated to show results from all corpora for tryjobs.
cl_triage_link = ('https://{instance}-gold.skia.org/search?'
'issue={issue}&'
'new_clstore=true&'
'query=source_type%3D{corpus}')
cl_triage_link = cl_triage_link.format(
instance=self._instance,
issue=self._gold_properties.issue,
corpus=self._corpus)
'issue={issue}')
cl_triage_link = cl_triage_link.format(instance=self._instance,
issue=self._gold_properties.issue)
self._comparison_results[name].triage_link = cl_triage_link
else:
try:

10
content/test/gpu/gpu_tests/maps_integration_test.py

@ -95,17 +95,17 @@ class MapsIntegrationTest(
print 'Maps\' devicePixelRatio is ' + str(dpr)
page = _GetMapsPageForUrl(url)
# The bottom corners of Mac screenshots have black triangles due to the rounded corners of Mac
# windows. So, crop the bottom few rows off now to get rid of those. The triangles appear to be
# 5 pixels wide and tall regardless of DPI, so 10 pixels should be sufficient.
# The bottom corners of Mac screenshots have black triangles due to the
# rounded corners of Mac windows. So, crop the bottom few rows off now to
# get rid of those. The triangles appear to be 5 pixels wide and tall
# regardless of DPI, so 10 pixels should be sufficient.
if self.browser.platform.GetOSName() == 'mac':
img_height, img_width = screenshot.shape[:2]
screenshot = image_util.Crop(screenshot, 0, 0, img_width, img_height - 10)
x1, y1, x2, y2 = _GetCropBoundaries(screenshot)
screenshot = image_util.Crop(screenshot, x1, y1, x2 - x1, y2 - y1)
self._UploadTestResultToSkiaGold(_TEST_NAME, screenshot, page,
self._GetBuildIdArgs())
self._UploadTestResultToSkiaGold(_TEST_NAME, screenshot, page)
@classmethod
def ExpectationsFiles(cls):

8
content/test/gpu/gpu_tests/pixel_integration_test.py

@ -153,17 +153,13 @@ class PixelIntegrationTest(
int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
build_id_args = self._GetBuildIdArgs()
# Compare images against approved images/colors.
if page.expected_colors:
# Use expected colors instead of hash comparison for validation.
self._ValidateScreenshotSamplesWithSkiaGold(tab, page, screenshot, dpr,
build_id_args)
self._ValidateScreenshotSamplesWithSkiaGold(tab, page, screenshot, dpr)
return
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(
image_name, screenshot, page, build_id_args=build_id_args)
self._UploadTestResultToSkiaGold(image_name, screenshot, page)
def _DoPageAction(self, tab, page):
getattr(self, '_' + page.optional_action)(tab, page)

3
content/test/gpu/gpu_tests/skia_gold/__init__.py Normal file

@ -0,0 +1,3 @@
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

150
content/test/gpu/gpu_tests/skia_gold/skia_gold_properties.py Normal file

@ -0,0 +1,150 @@
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Class for storing Skia Gold comparison properties.
Examples:
* git revision being tested
* Whether the test is being run locally or on a bot
* What the continuous integration system is
"""
import logging
import os
import subprocess
import sys
from gpu_tests import path_util
class SkiaGoldProperties(object):
def __init__(self, args):
"""Class to validate and store properties related to Skia Gold.
Args:
args: The parsed arguments from an argparse.ArgumentParser.
"""
self._git_revision = None
self._issue = None
self._patchset = None
self._job_id = None
self._local_pixel_tests = None
self._no_luci_auth = None
self._bypass_skia_gold_functionality = None
# Could in theory be configurable, but hard-coded for now since there's
# no plan to support anything else.
self._code_review_system = 'gerrit'
self._continuous_integration_system = 'buildbucket'
self._InitializeProperties(args)
def IsTryjobRun(self):
return self.issue is not None
@property
def continuous_integration_system(self):
return self._continuous_integration_system
@property
def code_review_system(self):
return self._code_review_system
@property
def git_revision(self):
return self._GetGitRevision()
@property
def issue(self):
return self._issue
@property
def job_id(self):
return self._job_id
@property
def local_pixel_tests(self):
return self._IsLocalRun()
@property
def no_luci_auth(self):
return self._no_luci_auth
@property
def patchset(self):
return self._patchset
@property
def bypass_skia_gold_functionality(self):
return self._bypass_skia_gold_functionality
def _GetGitRevision(self):
if not self._git_revision:
# Automated tests should always pass the revision, so assume we're on
# a workstation and try to get the local origin/master HEAD.
if not self._IsLocalRun():
raise RuntimeError(
'--git-revision was not passed when running on a bot')
revision = _GetGitOriginMasterHeadSha1()
if not revision or len(revision) != 40:
raise RuntimeError(
'--git-revision not passed and unable to determine from git')
self._git_revision = revision
return self._git_revision
def _IsLocalRun(self):
if self._local_pixel_tests is None:
# Look for the presence of the SWARMING_SERVER environment variable as a
# heuristic to determine whether we're running on a workstation or a bot.
# This should always be set on swarming, but would be strange to be set on
# a workstation.
self._local_pixel_tests = 'SWARMING_SERVER' not in os.environ
if self._local_pixel_tests:
logging.warning(
'Automatically determined that test is running on a workstation')
else:
logging.warning(
'Automatically determined that test is running on a bot')
return self._local_pixel_tests
def _InitializeProperties(self, args):
if hasattr(args, 'local_pixel_tests'):
# If not set, will be automatically determined later if needed.
self._local_pixel_tests = args.local_pixel_tests
if hasattr(args, 'no_luci_auth'):
self._no_luci_auth = args.no_luci_auth
if hasattr(args, 'bypass_skia_gold_functionality'):
self._bypass_skia_gold_functionality = args.bypass_skia_gold_functionality
# Will be automatically determined later if needed.
if not hasattr(args, 'git_revision') or not args.git_revision:
return
self._git_revision = args.git_revision
# Only expected on tryjob runs.
if not hasattr(args, 'gerrit_issue') or not args.gerrit_issue:
return
self._issue = args.gerrit_issue
if not hasattr(args, 'gerrit_patchset') or not args.gerrit_patchset:
raise RuntimeError(
'--gerrit-issue passed, but --gerrit-patchset not passed.')
self._patchset = args.gerrit_patchset
if not hasattr(args, 'buildbucket_id') or not args.buildbucket_id:
raise RuntimeError(
'--gerrit-issue passed, but --buildbucket-id not passed.')
self._job_id = args.buildbucket_id
def _IsWin():
return sys.platform == 'win32'
def _GetGitOriginMasterHeadSha1():
try:
return subprocess.check_output(['git', 'rev-parse', 'origin/master'],
shell=_IsWin(),
cwd=path_util.GetChromiumSrcDir()).strip()
except subprocess.CalledProcessError:
return None

172
content/test/gpu/gpu_tests/skia_gold/skia_gold_properties_unittest.py Executable file

@ -0,0 +1,172 @@
#!/usr/bin/env vpython
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#pylint: disable=protected-access
import os
import unittest
import mock
from gpu_tests.skia_gold import skia_gold_properties
from gpu_tests.skia_gold import unittest_utils
createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs
class SkiaGoldPropertiesInitializationTest(unittest.TestCase):
"""Tests that SkiaGoldProperties initializes (or doesn't) when expected."""
def verifySkiaGoldProperties(self, instance, expected):
self.assertEqual(instance._local_pixel_tests,
expected.get('local_pixel_tests'))
self.assertEqual(instance._no_luci_auth, expected.get('no_luci_auth'))
self.assertEqual(instance._git_revision, expected.get('git_revision'))
self.assertEqual(instance._issue, expected.get('gerrit_issue'))
self.assertEqual(instance._patchset, expected.get('gerrit_patchset'))
self.assertEqual(instance._job_id, expected.get('buildbucket_id'))
self.assertEqual(instance._bypass_skia_gold_functionality,
expected.get('bypass_skia_gold_functionality'))
def test_initializeSkiaGoldAttributes_unsetLocal(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {})
def test_initializeSkiaGoldAttributes_explicitLocal(self):
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': True})
def test_initializeSkiaGoldAttributes_explicitNonLocal(self):
args = createSkiaGoldArgs(local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': False})
def test_initializeSkiaGoldAttributes_explicitNoLuciAuth(self):
args = createSkiaGoldArgs(no_luci_auth=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {'no_luci_auth': True})
def test_initializeSkiaGoldAttributes_bypassExplicitTrue(self):
args = createSkiaGoldArgs(bypass_skia_gold_functionality=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {'bypass_skia_gold_functionality': True})
def test_initializeSkiaGoldAttributes_explicitGitRevision(self):
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {'git_revision': 'a'})
def test_initializeSkiaGoldAttributes_tryjobArgsIgnoredWithoutRevision(self):
args = createSkiaGoldArgs(gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(sgp, {})
def test_initializeSkiaGoldAttributes_tryjobArgs(self):
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.verifySkiaGoldProperties(
sgp, {
'git_revision': 'a',
'gerrit_issue': 1,
'gerrit_patchset': 2,
'buildbucket_id': 3
})
def test_initializeSkiaGoldAttributes_tryjobMissingPatchset(self):
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
buildbucket_id=3)
with self.assertRaises(RuntimeError):
skia_gold_properties.SkiaGoldProperties(args)
def test_initializeSkiaGoldAttributes_tryjobMissingBuildbucket(self):
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2)
with self.assertRaises(RuntimeError):
skia_gold_properties.SkiaGoldProperties(args)
class SkiaGoldPropertiesCalculationTest(unittest.TestCase):
"""Tests that SkiaGoldProperties properly calculates certain properties."""
def testLocalPixelTests_determineTrue(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
with mock.patch.dict(os.environ, {}, clear=True):
self.assertTrue(sgp.local_pixel_tests)
def testLocalPixelTests_determineFalse(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
with mock.patch.dict(os.environ, {'SWARMING_SERVER': ''}, clear=True):
self.assertFalse(sgp.local_pixel_tests)
def testIsTryjobRun_noIssue(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.assertFalse(sgp.IsTryjobRun())
def testIsTryjobRun_issue(self):
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.assertTrue(sgp.IsTryjobRun())
def testGetGitRevision_revisionSet(self):
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
self.assertEqual(sgp.git_revision, 'a')
def testGetGitRevision_findValidRevision(self):
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
with mock.patch(
'gpu_tests.skia_gold.skia_gold_properties._GetGitOriginMasterHeadSha1'
) as patched_head:
expected = 'a' * 40
patched_head.return_value = expected
self.assertEqual(sgp.git_revision, expected)
# Should be cached.
self.assertEqual(sgp._git_revision, expected)
def testGetGitRevision_noExplicitOnBot(self):
args = createSkiaGoldArgs(local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
with self.assertRaises(RuntimeError):
_ = sgp.git_revision
def testGetGitRevision_findEmptyRevision(self):
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
with mock.patch(
'gpu_tests.skia_gold.skia_gold_properties._GetGitOriginMasterHeadSha1'
) as patched_head:
patched_head.return_value = ''
with self.assertRaises(RuntimeError):
_ = sgp.git_revision
def testGetGitRevision_findMalformedRevision(self):
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
with mock.patch(
'gpu_tests.skia_gold.skia_gold_properties._GetGitOriginMasterHeadSha1'
) as patched_head:
patched_head.return_value = 'a' * 39
with self.assertRaises(RuntimeError):
_ = sgp.git_revision
if __name__ == '__main__':
unittest.main(verbosity=2)

412
content/test/gpu/gpu_tests/skia_gold/skia_gold_session.py Normal file

@ -0,0 +1,412 @@
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Class for interacting with the Skia Gold image diffing service.
This is based heavily off Android's Skia Gold implementation in
//build/android/pylib/utils/gold_utils.py. If you need to make a change to this
file, check to see if the same change needs to be made there.
"""
import logging
import os
import subprocess
import sys
import tempfile
from gpu_tests import path_util
GOLDCTL_BINARY = os.path.join(path_util.GetChromiumSrcDir(), 'tools',
'skia_goldctl')
if sys.platform == 'win32':
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'win', 'goldctl') + '.exe'
elif sys.platform == 'darwin':
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac', 'goldctl')
else:
GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'linux', 'goldctl')
class SkiaGoldSession(object):
class StatusCodes(object):
"""Status codes for RunComparison."""
SUCCESS = 0
AUTH_FAILURE = 1
INIT_FAILURE = 2
COMPARISON_FAILURE_REMOTE = 3
COMPARISON_FAILURE_LOCAL = 4
LOCAL_DIFF_FAILURE = 5
class ComparisonResults(object):
"""Struct-like object for storing results of an image comparison."""
def __init__(self):
self.triage_link = None
self.triage_link_omission_reason = None
self.local_diff_given_image = None
self.local_diff_closest_image = None
self.local_diff_diff_image = None
def __init__(self, working_dir, gold_properties, keys_file, corpus, instance):
"""A class to handle all aspects of an image comparison via Skia Gold.
A single SkiaGoldSession is valid for a single instance/corpus/keys_file
combination.
Args:
working_dir: The directory to store config files, etc.
gold_properties: A skia_gold_properties.SkiaGoldProperties instance for
the current test run.
keys_file: A path to a JSON file containing various comparison config data
such as corpus and debug information like the hardware/software
configuration the images will be produced on.
corpus: The corpus that images that will be compared belong to.
instance: The name of the Skia Gold instance to interact with.
"""
self._working_dir = working_dir
self._gold_properties = gold_properties
self._keys_file = keys_file
self._corpus = corpus
self._instance = instance
self._triage_link_file = tempfile.NamedTemporaryFile(suffix='.txt',
dir=working_dir,
delete=False).name
# A map of image name (string) to ComparisonResults for that image.
self._comparison_results = {}
self._authenticated = False
self._initialized = False
def RunComparison(self, name, png_file, use_luci=True):
"""Helper method to run all steps to compare a produced image.
Handles authentication, itnitialization, comparison, and, if necessary,
local diffing.
Args:
name: The name of the image being compared.
png_file: A path to a PNG file containing the image to be compared.
use_luci: If true, authentication will use the service account provided by
the LUCI context. If false, will attempt to use whatever is set up in
gsutil, which is only supported for local runs.
Returns:
A tuple (status, error). |status| is a value from
SkiaGoldSession.StatusCodes signifying the result of the comparison.
|error| is an error message describing the status if not successful.
"""
auth_rc, auth_stdout = self.Authenticate(use_luci=use_luci)
if auth_rc:
return self.StatusCodes.AUTH_FAILURE, auth_stdout
init_rc, init_stdout = self.Initialize()
if init_rc:
return self.StatusCodes.INIT_FAILURE, init_stdout
compare_rc, compare_stdout = self.Compare(name=name, png_file=png_file)
if not compare_rc:
return self.StatusCodes.SUCCESS, None
logging.error('Gold comparison failed: %s', compare_stdout)
if not self._gold_properties.local_pixel_tests:
return self.StatusCodes.COMPARISON_FAILURE_REMOTE, compare_stdout
diff_rc, diff_stdout = self.Diff(name=name, png_file=png_file)
if diff_rc:
return self.StatusCodes.LOCAL_DIFF_FAILURE, diff_stdout
return self.StatusCodes.COMPARISON_FAILURE_LOCAL, compare_stdout
def Authenticate(self, use_luci=True):
"""Authenticates with Skia Gold for this session.
Args:
use_luci: If true, authentication will use the service account provided
by the LUCI context. If false, will attempt to use whatever is set up
in gsutil, which is only supported for local runs.
Returns:
A tuple (return_code, output). |return_code| is the return code of the
authentication process. |output| is the stdout + stderr of the
authentication process.
"""
if self._authenticated:
return 0, None
if self._gold_properties.bypass_skia_gold_functionality:
logging.warning('Not actually authenticating with Gold due to '
'--bypass-skia-gold-functionality being present.')
return 0, None
auth_cmd = [GOLDCTL_BINARY, 'auth', '--work-dir', self._working_dir]
if use_luci:
auth_cmd.append('--luci')
elif not self._gold_properties.local_pixel_tests:
raise RuntimeError(
'Cannot authenticate to Skia Gold with use_luci=False unless running '
'local pixel tests')
rc, stdout = _RunCmdForRcAndOutput(auth_cmd)
if rc == 0:
self._authenticated = True
return rc, stdout
def Initialize(self):
"""Initializes the working directory if necessary.
This can technically be skipped if the same information is passed to the
command used for image comparison, but that is less efficient under the
hood. Doing it that way effectively requires an initialization for every
comparison (~250 ms) instead of once at the beginning.
Returns:
A tuple (return_code, output). |return_code| is the return code of the
initialization process. |output| is the stdout + stderr of the
initialization process.
"""
if self._initialized:
return 0, None
if self._gold_properties.bypass_skia_gold_functionality:
logging.warning('Not actually initializing Gold due to '
'--bypass-skia-gold-functionality being present.')
return 0, None
init_cmd = [
GOLDCTL_BINARY,
'imgtest',
'init',
'--passfail',
'--instance',
self._instance,
'--corpus',
self._corpus,
'--keys-file',
self._keys_file,
'--work-dir',
self._working_dir,
'--failure-file',
self._triage_link_file,
'--commit',
self._gold_properties.git_revision,
]
if self._gold_properties.IsTryjobRun():
init_cmd.extend([
'--issue',
str(self._gold_properties.issue),
'--patchset',
str(self._gold_properties.patchset),
'--jobid',
str(self._gold_properties.job_id),
'--crs',
str(self._gold_properties.code_review_system),
'--cis',
str(self._gold_properties.continuous_integration_system),
])
rc, stdout = _RunCmdForRcAndOutput(init_cmd)
if rc == 0:
self._initialized = True
return rc, stdout
def Compare(self, name, png_file):
"""Compares the given image to images known to Gold.
Triage links can later be retrieved using GetTriageLink().
Args:
name: The name of the image being compared.
png_file: A path to a PNG file containing the image to be compared.
Returns:
A tuple (return_code, output). |return_code| is the return code of the
comparison process. |output| is the stdout + stderr of the comparison
process.
"""
if self._gold_properties.bypass_skia_gold_functionality:
logging.warning('Not actually comparing with Gold due to '
'--bypass-skia-gold-functionality being present.')
return 0, None
compare_cmd = [
GOLDCTL_BINARY,
'imgtest',
'add',
'--test-name',
name,
'--png-file',
png_file,
'--work-dir',
self._working_dir,
]
if self._gold_properties.local_pixel_tests:
compare_cmd.append('--dryrun')
self._ClearTriageLinkFile()
rc, stdout = _RunCmdForRcAndOutput(compare_cmd)
self._comparison_results[name] = self.ComparisonResults()
if rc == 0:
self._comparison_results[name].triage_link_omission_reason = (
'Comparison succeeded, no triage link')
elif self._gold_properties.IsTryjobRun():
cl_triage_link = ('https://{instance}-gold.skia.org/search?'
'issue={issue}')
cl_triage_link = cl_triage_link.format(instance=self._instance,
issue=self._gold_properties.issue)
self._comparison_results[name].triage_link = cl_triage_link
else:
try:
with open(self._triage_link_file) as tlf:
triage_link = tlf.read().strip()
self._comparison_results[name].triage_link = triage_link
except IOError:
self._comparison_results[name].triage_link_omission_reason = (
'Failed to read triage link from file')
return rc, stdout
def Diff(self, name, png_file):
"""Performs a local image diff against the closest known positive in Gold.
This is used for running tests on a workstation, where uploading data to
Gold for ingestion is not allowed, and thus the web UI is not available.
Image links can later be retrieved using Get*ImageLink().
Args:
name: The name of the image being compared.
png_file: The path to a PNG file containing the image to be diffed.
Returns:
A tuple (return_code, output). |return_code| is the return code of the
diff process. |output| is the stdout + stderr of the diff process.
"""
# Instead of returning that everything is okay and putting in dummy links,
# just fail since this should only be called when running locally and
# --bypass-skia-gold-functionality is only meant for use on the bots.
if self._gold_properties.bypass_skia_gold_functionality:
raise RuntimeError(
'--bypass-skia-gold-functionality is not supported when running '
'tests locally.')
# We intentionally don't clean this up and don't put it in self._working_dir
# since we need it to stick around after the test completes so the user
# can look at its contents.
output_dir = tempfile.mkdtemp()
diff_cmd = [
GOLDCTL_BINARY,
'diff',
'--corpus',
self._corpus,
'--instance',
self._instance,
'--input',
png_file,
'--test',
name,
'--work-dir',
self._working_dir,
'--out-dir',
output_dir,
]
rc, stdout = _RunCmdForRcAndOutput(diff_cmd)
results = self._comparison_results.setdefault(name,
self.ComparisonResults())
# The directory should contain "input-<hash>.png", "closest-<hash>.png",
# and "diff.png".
for f in os.listdir(output_dir):
file_url = 'file://%s' % os.path.join(output_dir, f)
if f.startswith('input-'):
results.local_diff_given_image = file_url
elif f.startswith('closest-'):
results.local_diff_closest_image = file_url
elif f == 'diff.png':
results.local_diff_diff_image = file_url
return rc, stdout
def GetTriageLink(self, name):
"""Gets the triage link for the given image.
Args:
name: The name of the image to retrieve the triage link for.
Returns:
A string containing the triage link if it is available, or None if it is
not available for some reason. The reason can be retrieved using
GetTriageLinkOmissionReason.
"""
return self._comparison_results.get(name,
self.ComparisonResults()).triage_link
def GetTriageLinkOmissionReason(self, name):
"""Gets the reason why a triage link is not available for an image.
Args:
name: The name of the image whose triage link does not exist.
Returns:
A string containing the reason why a triage link is not available.
"""
if name not in self._comparison_results:
return 'No image comparison performed for %s' % name
results = self._comparison_results[name]
# This method should not be called if there is a valid triage link.
assert results.triage_link is None
if results.triage_link_omission_reason:
return results.triage_link_omission_reason
if results.local_diff_given_image:
return 'Gold only used to do a local image diff'
raise RuntimeError(
'Somehow have a ComparisonResults instance for %s that should not '
'exist' % name)
def GetGivenImageLink(self, name):
"""Gets the link to the given image used for local diffing.
Args:
name: The name of the image that was diffed.
Returns:
A string containing the link to where the image is saved, or None if it
does not exist.
"""
assert name in self._comparison_results
return self._comparison_results[name].local_diff_given_image
def GetClosestImageLink(self, name):
"""Gets the link to the closest known image used for local diffing.
Args:
name: The name of the image that was diffed.
Returns:
A string containing the link to where the image is saved, or None if it
does not exist.
"""
assert name in self._comparison_results
return self._comparison_results[name].local_diff_closest_image
def GetDiffImageLink(self, name):
"""Gets the link to the diff between the given and closest images.
Args:
name: The name of the image that was diffed.
Returns:
A string containing the link to where the image is saved, or None if it
does not exist.
"""
assert name in self._comparison_results
return self._comparison_results[name].local_diff_diff_image
def _ClearTriageLinkFile(self):
"""Clears the contents of the triage link file.
This should be done before every comparison since goldctl appends to the
file instead of overwriting its contents, which results in multiple triage
links getting concatenated together if there are multiple failures.
"""
open(self._triage_link_file, 'w').close()
def _RunCmdForRcAndOutput(cmd):
try:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return 0, output
except subprocess.CalledProcessError as e:
return e.returncode, e.output

71
content/test/gpu/gpu_tests/skia_gold/skia_gold_session_manager.py Normal file

@ -0,0 +1,71 @@
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Class for managing multiple SkiaGoldSessions.
This is vased heavily off Android's Skia Gold implementation in
//build/android/pylib/utils/gold_utils.py. If you need to make a change to this
file, check to see if the same change needs to be made there.
"""
import json
import tempfile
from gpu_tests.skia_gold import skia_gold_session
DEFAULT_INSTANCE = 'chrome-gpu'
class SkiaGoldSessionManager(object):
def __init__(self, working_dir, gold_properties):
"""Class to manage one or more skia_gold_session.SkiaGoldSessions.
A separate session is required for each instance/corpus/keys_file
combination, so this class will lazily create them as necessary.
Args:
working_dir: The working directory under which each individual
SkiaGoldSessions' working directory will be created.
gold_properties: A SkiaGoldProperties instance that will be used to create
any SkiaGoldSessions.
"""
self._working_dir = working_dir
self._gold_properties = gold_properties
self._sessions = {}
def GetSkiaGoldSession(self,
keys_dict,
corpus=None,
instance=DEFAULT_INSTANCE):
"""Gets a SkiaGoldSession for the given arguments.
Lazily creates one if necessary.
Args:
keys_dict: A dictionary containing various comparison config data such as
corpus and debug information like the hardware/software configuration
the image was produced on.
corpus: The corpus the session is for. If None, the corpus will be
determined using available information.
instance: The name of the Skia Gold instance to interact with.
"""
keys_string = json.dumps(keys_dict, sort_keys=True)
if corpus is None:
corpus = keys_dict.get('source_type', instance)
# Use the string representation of the keys JSON as a proxy for a hash since
# dicts themselves are not hashable.
session = self._sessions.setdefault(instance,
{}).setdefault(corpus, {}).setdefault(
keys_string, None)
if not session:
working_dir = tempfile.mkdtemp(dir=self._working_dir)
keys_file = tempfile.NamedTemporaryFile(suffix='.json',
dir=working_dir,
delete=False).name
with open(keys_file, 'w') as f:
json.dump(keys_dict, f)
session = skia_gold_session.SkiaGoldSession(working_dir,
self._gold_properties,
keys_file, corpus, instance)
self._sessions[instance][corpus][keys_string] = session
return session

119
content/test/gpu/gpu_tests/skia_gold/skia_gold_session_manager_unittest.py Executable file

@ -0,0 +1,119 @@
#!/usr/bin/env vpython
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#pylint: disable=protected-access
import os
import tempfile
import unittest
import mock
from gpu_tests.skia_gold import skia_gold_properties
from gpu_tests.skia_gold import skia_gold_session
from gpu_tests.skia_gold import skia_gold_session_manager
from gpu_tests.skia_gold import unittest_utils
from pyfakefs import fake_filesystem_unittest
createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs
class SkiaGoldSessionManagerGetSessionTest(fake_filesystem_unittest.TestCase):
"""Tests the functionality of SkiaGoldSessionManager.GetSkiaGoldSession."""
def setUp(self):
self.setUpPyfakefs()
self._working_dir = tempfile.mkdtemp()
def test_ArgsForwardedToSession(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance')
self.assertTrue(session._keys_file.startswith(self._working_dir))
self.assertEqual(session._corpus, 'corpus')
self.assertEqual(session._instance, 'instance')
# Make sure the session's working directory is a subdirectory of the
# manager's working directory.
self.assertEqual(os.path.dirname(session._working_dir), self._working_dir)
def test_corpusFromJson(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session = sgsm.GetSkiaGoldSession({'source_type': 'foobar'}, None,
'instance')
self.assertTrue(session._keys_file.startswith(self._working_dir))
self.assertEqual(session._corpus, 'foobar')
self.assertEqual(session._instance, 'instance')
def test_corpusDefaultsToInstance(self):
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session = sgsm.GetSkiaGoldSession({}, None, 'instance')
self.assertTrue(session._keys_file.startswith(self._working_dir))
self.assertEqual(session._corpus, 'instance')
self.assertEqual(session._instance, 'instance')
@mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__')
def test_matchingSessionReused(self, session_mock):
session_mock.return_value = None
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance')
session2 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance')
self.assertEqual(session1, session2)
# For some reason, session_mock.assert_called_once() always passes,
# so check the call count directly.
self.assertEqual(session_mock.call_count, 1)
@mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__')
def test_separateSessionsFromKeys(self, session_mock):
session_mock.return_value = None
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance')
session2 = sgsm.GetSkiaGoldSession({'something_different': 1}, 'corpus',
'instance')
self.assertNotEqual(session1, session2)
self.assertEqual(session_mock.call_count, 2)
@mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__')
def test_separateSessionsFromCorpus(self, session_mock):
session_mock.return_value = None
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session1 = sgsm.GetSkiaGoldSession({}, 'corpus1', 'instance')
session2 = sgsm.GetSkiaGoldSession({}, 'corpus2', 'instance')
self.assertNotEqual(session1, session2)
self.assertEqual(session_mock.call_count, 2)
@mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__')
def test_separateSessionsFromInstance(self, session_mock):
session_mock.return_value = None
args = createSkiaGoldArgs()
sgp = skia_gold_properties.SkiaGoldProperties(args)
self._working_dir = tempfile.mkdtemp()
sgsm = skia_gold_session_manager.SkiaGoldSessionManager(
self._working_dir, sgp)
session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance1')
session2 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance2')
self.assertNotEqual(session1, session2)
self.assertEqual(session_mock.call_count, 2)
if __name__ == '__main__':
unittest.main(verbosity=2)

635
content/test/gpu/gpu_tests/skia_gold/skia_gold_session_unittest.py Executable file

@ -0,0 +1,635 @@
#!/usr/bin/env vpython
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#pylint: disable=protected-access
import json
import os
import tempfile
import unittest
import mock
from gpu_tests.skia_gold import skia_gold_properties
from gpu_tests.skia_gold import skia_gold_session
from gpu_tests.skia_gold import unittest_utils
from pyfakefs import fake_filesystem_unittest
createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs
def assertArgWith(test, arg_list, arg, value):
i = arg_list.index(arg)
test.assertEqual(arg_list[i + 1], value)
class SkiaGoldSessionRunComparisonTest(fake_filesystem_unittest.TestCase):
"""Tests the functionality of SkiaGoldSession.RunComparison."""
def setUp(self):
self.setUpPyfakefs()
self._working_dir = tempfile.mkdtemp()
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate')
def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (0, None)
keys_file = os.path.join(self._working_dir, 'keys.json')
with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = skia_gold_session.SkiaGoldSession(self._working_dir, None,
keys_file, None, None)
status, _ = session.RunComparison(None, None, None)
self.assertEqual(status,
skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS)
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate')
def test_authFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (1, 'Auth failed')
session = skia_gold_session.SkiaGoldSession(self._working_dir, None, None,
None, None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(status,
skia_gold_session.SkiaGoldSession.StatusCodes.AUTH_FAILURE)
self.assertEqual(error, 'Auth failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 0)
self.assertEqual(compare_mock.call_count, 0)
self.assertEqual(diff_mock.call_count, 0)
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate')
def test_initFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (1, 'Init failed')
session = skia_gold_session.SkiaGoldSession(self._working_dir, None, None,
None, None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(status,
skia_gold_session.SkiaGoldSession.StatusCodes.INIT_FAILURE)
self.assertEqual(error, 'Init failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 0)
self.assertEqual(diff_mock.call_count, 0)
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate')
def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
args = createSkiaGoldArgs(local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
keys_file = os.path.join(self._working_dir, 'keys.json')
with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
keys_file, None, None)
status, error = session.RunComparison(None, None, None)
self.assertEqual(
status,
skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_REMOTE)
self.assertEqual(error, 'Compare failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 0)
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate')
def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock,
diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (0, None)
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
keys_file = os.path.join(self._working_dir, 'keys.json')
with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
keys_file, None, None)
status, error = session.RunComparison(None, None)
self.assertEqual(
status,
skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_LOCAL)
self.assertEqual(error, 'Compare failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 1)
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize')
@mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate')
def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock):
auth_mock.return_value = (0, None)
init_mock.return_value = (0, None)
compare_mock.return_value = (1, 'Compare failed')
diff_mock.return_value = (1, 'Diff failed')
args = createSkiaGoldArgs(local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
keys_file = os.path.join(self._working_dir, 'keys.json')
with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f:
json.dump({}, f)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
keys_file, None, None)
status, error = session.RunComparison(None, None)
self.assertEqual(
status,
skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE)
self.assertEqual(error, 'Diff failed')
self.assertEqual(auth_mock.call_count, 1)
self.assertEqual(init_mock.call_count, 1)
self.assertEqual(compare_mock.call_count, 1)
self.assertEqual(diff_mock.call_count, 1)
class SkiaGoldSessionAuthenticateTest(fake_filesystem_unittest.TestCase):
"""Tests the functionality of SkiaGoldSession.Authenticate."""
def setUp(self):
self.setUpPyfakefs()
self._working_dir = tempfile.mkdtemp()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandOutputReturned(self, cmd_mock):
cmd_mock.return_value = (1, 'Something bad :(')
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
rc, stdout = session.Authenticate()
self.assertEqual(cmd_mock.call_count, 1)
self.assertEqual(rc, 1)
self.assertEqual(stdout, 'Something bad :(')
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_bypassSkiaGoldFunctionality(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a',
bypass_skia_gold_functionality=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
rc, _ = session.Authenticate()
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_shortCircuitAlreadyAuthenticated(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session._authenticated = True
rc, _ = session.Authenticate()
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_successSetsShortCircuit(self, cmd_mock):
cmd_mock.return_value = (0, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
self.assertFalse(session._authenticated)
rc, _ = session.Authenticate()
self.assertEqual(rc, 0)
self.assertTrue(session._authenticated)
cmd_mock.assert_called_once()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_failureDoesNotSetShortCircuit(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
self.assertFalse(session._authenticated)
rc, _ = session.Authenticate()
self.assertEqual(rc, 1)
self.assertFalse(session._authenticated)
cmd_mock.assert_called_once()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandWithUseLuciTrue(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Authenticate(use_luci=True)
self.assertIn('--luci', cmd_mock.call_args[0][0])
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandWithUseLuciFalse(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Authenticate(use_luci=False)
self.assertNotIn('--luci', cmd_mock.call_args[0][0])
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandWithUseLuciFalseNotLocal(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
with self.assertRaises(RuntimeError):
session.Authenticate(use_luci=False)
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Authenticate()
call_args = cmd_mock.call_args[0][0]
self.assertIn('auth', call_args)
assertArgWith(self, call_args, '--work-dir', self._working_dir)
class SkiaGoldSessionInitializeTest(fake_filesystem_unittest.TestCase):
"""Tests the functionality of SkiaGoldSession.Initialize."""
def setUp(self):
self.setUpPyfakefs()
self._working_dir = tempfile.mkdtemp()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_bypassSkiaGoldFunctionality(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a',
bypass_skia_gold_functionality=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
rc, _ = session.Initialize()
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_shortCircuitAlreadyInitialized(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session._initialized = True
rc, _ = session.Initialize()
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_successSetsShortCircuit(self, cmd_mock):
cmd_mock.return_value = (0, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
self.assertFalse(session._initialized)
rc, _ = session.Initialize()
self.assertEqual(rc, 0)
self.assertTrue(session._initialized)
cmd_mock.assert_called_once()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_failureDoesNotSetShortCircuit(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
self.assertFalse(session._initialized)
rc, _ = session.Initialize()
self.assertEqual(rc, 1)
self.assertFalse(session._initialized)
cmd_mock.assert_called_once()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
'keys_file',
'corpus',
instance='instance')
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('init', call_args)
self.assertIn('--passfail', call_args)
assertArgWith(self, call_args, '--instance', 'instance')
assertArgWith(self, call_args, '--corpus', 'corpus')
assertArgWith(self, call_args, '--keys-file', 'keys_file')
assertArgWith(self, call_args, '--work-dir', self._working_dir)
assertArgWith(self, call_args, '--failure-file', session._triage_link_file)
assertArgWith(self, call_args, '--commit', 'a')
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandTryjobArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Initialize()
call_args = cmd_mock.call_args[0][0]
assertArgWith(self, call_args, '--issue', '1')
assertArgWith(self, call_args, '--patchset', '2')
assertArgWith(self, call_args, '--jobid', '3')
assertArgWith(self, call_args, '--crs', 'gerrit')
assertArgWith(self, call_args, '--cis', 'buildbucket')
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandTryjobArgsMissing(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Initialize()
call_args = cmd_mock.call_args[0][0]
self.assertNotIn('--issue', call_args)
self.assertNotIn('--patchset', call_args)
self.assertNotIn('--jobid', call_args)
self.assertNotIn('--crs', call_args)
self.assertNotIn('--cis', call_args)
class SkiaGoldSessionCompareTest(fake_filesystem_unittest.TestCase):
"""Tests the functionality of SkiaGoldSession.Compare."""
def setUp(self):
self.setUpPyfakefs()
self._working_dir = tempfile.mkdtemp()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandOutputReturned(self, cmd_mock):
cmd_mock.return_value = (1, 'Something bad :(')
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
rc, stdout = session.Compare(None, None)
self.assertEqual(cmd_mock.call_count, 1)
self.assertEqual(rc, 1)
self.assertEqual(stdout, 'Something bad :(')
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_bypassSkiaGoldFunctionality(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a',
bypass_skia_gold_functionality=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
rc, _ = session.Compare(None, None)
self.assertEqual(rc, 0)
cmd_mock.assert_not_called()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandWithLocalPixelTestsTrue(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Compare(None, None)
self.assertIn('--dryrun', cmd_mock.call_args[0][0])
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandWithLocalPixelTestsFalse(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
session.Compare(None, None)
self.assertNotIn('--dryrun', cmd_mock.call_args[0][0])
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
'keys_file',
'corpus',
instance='instance')
session.Compare('name', 'png_file')
call_args = cmd_mock.call_args[0][0]
self.assertIn('imgtest', call_args)
self.assertIn('add', call_args)
assertArgWith(self, call_args, '--test-name', 'name')
assertArgWith(self, call_args, '--png-file', 'png_file')
assertArgWith(self, call_args, '--work-dir', self._working_dir)
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_noLinkOnSuccess(self, cmd_mock):
cmd_mock.return_value = (0, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 0)
self.assertEqual(session._comparison_results['name'].triage_link, None)
self.assertNotEqual(
session._comparison_results['name'].triage_link_omission_reason, None)
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_clLinkOnTrybot(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a',
gerrit_issue=1,
gerrit_patchset=2,
buildbucket_id=3)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
self.assertNotEqual(session._comparison_results['name'].triage_link, None)
self.assertIn('issue=1', session._comparison_results['name'].triage_link)
self.assertEqual(
session._comparison_results['name'].triage_link_omission_reason, None)
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_individualLinkOnCi(self, cmd_mock):
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
def WriteTriageLinkFile(_):
with open(session._triage_link_file, 'w') as f:
f.write('foobar')
return (1, None)
cmd_mock.side_effect = WriteTriageLinkFile
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
self.assertNotEqual(session._comparison_results['name'].triage_link, None)
self.assertEqual(session._comparison_results['name'].triage_link, 'foobar')
self.assertEqual(
session._comparison_results['name'].triage_link_omission_reason, None)
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_validOmissionOnIoError(self, cmd_mock):
cmd_mock.return_value = (1, None)
args = createSkiaGoldArgs(git_revision='a')
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp,
'keys_file', None, None)
def DeleteTriageLinkFile(_):
os.remove(session._triage_link_file)
return (1, None)
cmd_mock.side_effect = DeleteTriageLinkFile
rc, _ = session.Compare('name', 'png_file')
self.assertEqual(rc, 1)
self.assertEqual(session._comparison_results['name'].triage_link, None)
self.assertNotEqual(
session._comparison_results['name'].triage_link_omission_reason, None)
self.assertIn(
'Failed to read',
session._comparison_results['name'].triage_link_omission_reason)
class SkiaGoldSessionDiffTest(fake_filesystem_unittest.TestCase):
"""Tests the functionality of SkiaGoldSession.Diff."""
def setUp(self):
self.setUpPyfakefs()
self._working_dir = tempfile.mkdtemp()
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandOutputReturned(self, cmd_mock):
cmd_mock.return_value = (1, 'Something bad :(')
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
rc, stdout = session.Diff(None, None)
self.assertEqual(cmd_mock.call_count, 1)
self.assertEqual(rc, 1)
self.assertEqual(stdout, 'Something bad :(')
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_bypassSkiaGoldFunctionality(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a',
bypass_skia_gold_functionality=True)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None,
None, None)
with self.assertRaises(RuntimeError):
session.Diff(None, None)
@mock.patch('gpu_tests.skia_gold.skia_gold_session._RunCmdForRcAndOutput')
def test_commandCommonArgs(self, cmd_mock):
cmd_mock.return_value = (None, None)
args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False)
sgp = skia_gold_properties.SkiaGoldProperties(args)
session = skia_gold_session.SkiaGoldSession(self._working_dir,
sgp,
None,
'corpus',
instance='instance')
session.Diff('name', 'png_file')
call_args = cmd_mock.call_args[0][0]
self.assertIn('diff', call_args)
assertArgWith(self, call_args, '--corpus', 'corpus')
assertArgWith(self, call_args, '--instance', 'instance')
assertArgWith(self, call_args, '--input', 'png_file')
assertArgWith(self, call_args, '--test', 'name')
assertArgWith(self, call_args, '--work-dir', self._working_dir)
i = call_args.index('--out-dir')
# The output directory should not be a subdirectory of the working
# directory.
self.assertNotIn(self._working_dir, call_args[i + 1])
class SkiaGoldSessionTriageLinkOmissionTest(unittest.TestCase):
"""Tests the functionality of SkiaGoldSession.GetTriageLinkOmissionReason."""
# Avoid having to bother with the working directory.
class FakeGoldSession(skia_gold_session.SkiaGoldSession):
def __init__(self): # pylint: disable=super-init-not-called
self._comparison_results = {
'foo': skia_gold_session.SkiaGoldSession.ComparisonResults(),
}
def test_noComparison(self):
session = self.FakeGoldSession()
session._comparison_results = {}
reason = session.GetTriageLinkOmissionReason('foo')
self.assertEqual(reason, 'No image comparison performed for foo')
def test_validReason(self):
session = self.FakeGoldSession()
session._comparison_results['foo'].triage_link_omission_reason = 'bar'
reason = session.GetTriageLinkOmissionReason('foo')
self.assertEqual(reason, 'bar')
def test_onlyLocal(self):
session = self.FakeGoldSession()
session._comparison_results['foo'].local_diff_given_image = 'bar'
reason = session.GetTriageLinkOmissionReason('foo')
self.assertEqual(reason, 'Gold only used to do a local image diff')
def test_onlyWithoutTriageLink(self):
session = self.FakeGoldSession()
session._comparison_results['foo'].triage_link = 'bar'
with self.assertRaises(AssertionError):
session.GetTriageLinkOmissionReason('foo')
def test_resultsShouldNotExist(self):
session = self.FakeGoldSession()
with self.assertRaises(RuntimeError):
session.GetTriageLinkOmissionReason('foo')
if __name__ == '__main__':
unittest.main(verbosity=2)

28
content/test/gpu/gpu_tests/skia_gold/unittest_utils.py Normal file

@ -0,0 +1,28 @@
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utility methods for Skia Gold functionality unittests."""
import collections
_SkiaGoldArgs = collections.namedtuple('_SkiaGoldArgs', [
'local_pixel_tests',
'no_luci_auth',
'git_revision',
'gerrit_issue',
'gerrit_patchset',
'buildbucket_id',
'bypass_skia_gold_functionality',
])
def createSkiaGoldArgs(local_pixel_tests=None,
no_luci_auth=None,
git_revision=None,
gerrit_issue=None,
gerrit_patchset=None,
buildbucket_id=None,
bypass_skia_gold_functionality=None):
return _SkiaGoldArgs(local_pixel_tests, no_luci_auth, git_revision,
gerrit_issue, gerrit_patchset, buildbucket_id,
bypass_skia_gold_functionality)

318
content/test/gpu/gpu_tests/skia_gold_integration_test_base.py

@ -3,12 +3,9 @@
# found in the LICENSE file.
from datetime import date
import json
import logging
import os
import re
import subprocess
from subprocess import CalledProcessError
import shutil
import sys
import tempfile
@ -16,6 +13,9 @@ import tempfile
from gpu_tests import gpu_integration_test
from gpu_tests import path_util
from gpu_tests import color_profile_manager
from gpu_tests.skia_gold import skia_gold_properties
from gpu_tests.skia_gold import skia_gold_session
from gpu_tests.skia_gold import skia_gold_session_manager
from py_utils import cloud_storage
@ -29,26 +29,10 @@ TEST_DATA_DIRS = [
os.path.join(path_util.GetChromiumSrcDir(), 'media/test/data'),
]
GOLDCTL_BIN = os.path.join(path_util.GetChromiumSrcDir(), 'tools',
'skia_goldctl')
if sys.platform == 'win32':
GOLDCTL_BIN = os.path.join(GOLDCTL_BIN, 'win', 'goldctl') + '.exe'
elif sys.platform == 'darwin':
GOLDCTL_BIN = os.path.join(GOLDCTL_BIN, 'mac', 'goldctl')
else:
GOLDCTL_BIN = os.path.join(GOLDCTL_BIN, 'linux', 'goldctl')
SKIA_GOLD_INSTANCE = 'chrome-gpu'
SKIA_GOLD_CORPUS = SKIA_GOLD_INSTANCE
# This is mainly used to determine if we need to run a subprocess through the
# shell - on Windows, finding executables via PATH doesn't work properly unless
# run through the shell.
def IsWin():
return sys.platform == 'win32'
class _ImageParameters(object):
def __init__(self):
# Parameters for cloud storage reference images.
@ -78,9 +62,8 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
_image_parameters = None
_skia_gold_temp_dir = None
_local_run = None
_git_revision = None
_skia_gold_session_manager = None
_skia_gold_properties = None
@classmethod
def SetParsedCommandLineOptions(cls, options):
@ -101,6 +84,21 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
cls.SetStaticServerDirs(TEST_DATA_DIRS)
cls._skia_gold_temp_dir = tempfile.mkdtemp()
@classmethod
def GetSkiaGoldProperties(cls):
if not cls._skia_gold_properties:
cls._skia_gold_properties = skia_gold_properties.SkiaGoldProperties(
cls.GetParsedCommandLineOptions())
return cls._skia_gold_properties
@classmethod
def GetSkiaGoldSessionManager(cls):
if not cls._skia_gold_session_manager:
cls._skia_gold_session_manager =\
skia_gold_session_manager.SkiaGoldSessionManager(
cls._skia_gold_temp_dir, cls.GetSkiaGoldProperties())
return cls._skia_gold_session_manager
@staticmethod
def _AddDefaultArgs(browser_args):
if not browser_args:
@ -164,16 +162,27 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
help='For Skia Gold integration. Always report that the test passed '
'even if the Skia Gold image comparison reported a failure, but '
'otherwise perform the same steps as usual.')
# Telemetry is *still* using optparse instead of argparse, so we can't have
# these two options in a mutually exclusive group.
parser.add_option(
'--local-run',
'--local-pixel-tests',
action='store_true',
default=None,
type=int,
help='Specifies to run the test harness in local run mode or not. When '
'run in local mode, uploading to Gold is disabled and links to '
'help with local debugging are output. Running in local mode also '
'implies --no-luci-auth. If left unset, the test harness will '
'attempt to detect whether it is running on a workstation or not '
'and set this option accordingly.')
'implies --no-luci-auth. If both this and --no-local-pixel-tests are '
'left unset, the test harness will attempt to detect whether it is '
'running on a workstation or not and set this option accordingly.')
parser.add_option(
'--no-local-pixel-tests',
action='store_false',
dest='local_pixel_tests',
help='Specifies to run the test harness in non-local (bot) mode. When '
'run in this mode, data is actually uploaded to Gold and triage links '
'arge generated. If both this and --local-pixel-tests are left unset, '
'the test harness will attempt to detect whether it is running on a '
'workstation or not and set this option accordingly.')
parser.add_option(
'--no-luci-auth',
action='store_true',
@ -250,11 +259,12 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
# bots, so kept around for future use.
@classmethod
def _UploadGoldErrorImageToCloudStorage(cls, image_name, screenshot):
revision = cls.GetSkiaGoldProperties().git_revision
machine_name = re.sub(r'\W+', '_',
cls.GetParsedCommandLineOptions().test_machine_name)
base_bucket = '%s/gold_failures' % (cls._error_image_cloud_storage_bucket)
image_name_with_revision_and_machine = '%s_%s_%s.png' % (
image_name, machine_name, cls._GetBuildRevision())
image_name, machine_name, revision)
cls._UploadBitmapToCloudStorage(
base_bucket,
image_name_with_revision_and_machine,
@ -340,26 +350,6 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
image_name = re.sub(r'(\.|/|-)', '_', image_name)
return image_name
def _GetBuildIdArgs(self):
# Get all the information that goldctl requires.
parsed_options = self.GetParsedCommandLineOptions()
build_id_args = [
'--commit',
self._GetBuildRevision(),
]
# If --gerrit-issue is passed, then we assume we're running on a trybot.
if parsed_options.gerrit_issue:
# yapf: disable
build_id_args += [
'--issue', parsed_options.gerrit_issue,
'--patchset', parsed_options.gerrit_patchset,
'--jobid', parsed_options.buildbucket_id,
'--crs', 'gerrit',
'--cis', 'buildbucket',
]
# yapf: enable
return build_id_args
def GetGoldJsonKeys(self, page):
"""Get all the JSON metadata that will be passed to golctl."""
img_params = self.GetImageParameters(page)
@ -400,17 +390,7 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
gpu_keys['ignore'] = '1'
return gpu_keys
# TODO(crbug.com/1076144): This is due for a refactor, likely similar to how
# the instrumentation tests handle it (see
# //build/android/pylib/utils/gold_utils.py), which will address the
# too-many-locals error.
# pylint: disable=too-many-locals
def _UploadTestResultToSkiaGold(self,
image_name,
screenshot,
page,
build_id_args=None):
def _UploadTestResultToSkiaGold(self, image_name, screenshot, page):
"""Compares the given image using Skia Gold and uploads the result.
No uploading is done if the test is being run in local run mode. Compares
@ -421,128 +401,56 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
image_name: the name of the image being checked.
screenshot: the image being checked as a Telemetry Bitmap.
page: the GPU PixelTestPage object for the test.
build_id_args: a list of build-identifying flags and values.
"""
if self.GetParsedCommandLineOptions().bypass_skia_gold_functionality:
logging.warning('Not actually comparing with Gold due to '
'--bypass-skia-gold-functionality being present.')
return
if not isinstance(build_id_args, list) or '--commit' not in build_id_args:
raise Exception('Requires build args to be specified, including --commit')
# Write screenshot to PNG file on local disk.
png_temp_file = tempfile.NamedTemporaryFile(
suffix='.png', dir=self._skia_gold_temp_dir).name
image_util.WritePngFile(screenshot, png_temp_file)
gpu_keys = self.GetGoldJsonKeys(page)
json_temp_file = tempfile.NamedTemporaryFile(
suffix='.json', dir=self._skia_gold_temp_dir).name
failure_file = tempfile.NamedTemporaryFile(
suffix='.txt', dir=self._skia_gold_temp_dir).name
with open(json_temp_file, 'w+') as f:
json.dump(gpu_keys, f)
gold_session = self.GetSkiaGoldSessionManager().GetSkiaGoldSession(gpu_keys)
gold_properties = self.GetSkiaGoldProperties()
use_luci = not (gold_properties.local_pixel_tests
or gold_properties.no_luci_auth)
# Figure out any extra args we need to pass to goldctl.
extra_imgtest_args = []
extra_auth_args = []
parsed_options = self.GetParsedCommandLineOptions()
if self._IsLocalRun():
extra_imgtest_args.append('--dryrun')
elif not parsed_options.no_luci_auth:
extra_auth_args = ['--luci']
status, error = gold_session.RunComparison(name=image_name,
png_file=png_temp_file,
use_luci=use_luci)
if not status:
return
# Run goldctl for a result.
try:
subprocess.check_output(
[GOLDCTL_BIN, 'auth', '--work-dir', self._skia_gold_temp_dir] +
extra_auth_args,
stderr=subprocess.STDOUT)
algorithm_args = page.matching_algorithm.GetCmdline()
if algorithm_args:
logging.info('Using non-exact matching algorithm %s for %s',
page.matching_algorithm.Name(), image_name)
# yapf: disable
cmd = ([
GOLDCTL_BIN,
'imgtest', 'add',
'--passfail',
'--test-name', image_name,
'--instance', SKIA_GOLD_INSTANCE,
'--keys-file', json_temp_file,
'--png-file', png_temp_file,
'--work-dir', self._skia_gold_temp_dir,
'--failure-file', failure_file
] + build_id_args + extra_imgtest_args + algorithm_args)
# yapf: enable
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
# TODO(skbug.com/10245): Remove this once the issue with auto-triaging
# inexactly matched images is fixed.
if 'VP9_YUY2' in image_name:
logging.error(output)
except CalledProcessError as e:
# We don't want to bother printing out triage links for local runs.
# Instead, we print out local filepaths for debugging. However, we want
# these to be at the bottom of the output so they're easier to find, so
# that is handled later.
if self._IsLocalRun():
pass
# The triage link for the image is output to the failure file, so report
# that if it's available so it shows up in Milo. If for whatever reason
# the file is not present or malformed, the triage link will still be
# present in the stdout of the goldctl command.
# If we're running on a trybot, instead generate a link to all results
# for the CL so that the user can visit a single page instead of
# clicking on multiple links on potentially multiple bots.
elif parsed_options.gerrit_issue:
cl_images = ('https://%s-gold.skia.org/search?'
'issue=%s&new_clstore=true' %
(SKIA_GOLD_INSTANCE, parsed_options.gerrit_issue))
self.artifacts.CreateLink('triage_link_for_entire_cl', cl_images)
status_codes = skia_gold_session.SkiaGoldSession.StatusCodes
if status == status_codes.AUTH_FAILURE:
logging.error('Gold authentication failed with output %s', error)
elif status == status_codes.INIT_FAILURE:
logging.error('Gold initialization failed with output %s', error)
elif status == status_codes.COMPARISON_FAILURE_REMOTE:
triage_link = gold_session.GetTriageLink(image_name)
if not triage_link:
logging.error('Failed to get triage link for %s, raw output: %s',
image_name, error)
logging.error('Reason for no triage link: %s',
gold_session.GetTriageLinkOmissionReason(image_name))
elif gold_properties.IsTryjobRun():
self.artifacts.CreateLink('triage_link_for_entire_cl', triage_link)
else:
try:
with open(failure_file, 'r') as ff:
self.artifacts.CreateLink('gold_triage_link', ff.read())
except Exception:
logging.error('Failed to read contents of goldctl failure file')
logging.error('goldctl failed with output: %s', e.output)
if self._IsLocalRun():
# Intentionally not cleaned up so the user can look at its contents.
diff_dir = tempfile.mkdtemp()
# yapf: disable
cmd = [
GOLDCTL_BIN,
'diff',
'--corpus', SKIA_GOLD_CORPUS,
'--instance', SKIA_GOLD_INSTANCE,
'--input', png_temp_file,
'--test', image_name,
'--work-dir', self._skia_gold_temp_dir,
'--out-dir', diff_dir,
]
# yapf: enable
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except CalledProcessError as e:
logging.error('Failed to generate diffs from Gold: %s', e)
# The directory should contain "input-<hash>.png", "closest-<hash>.png",
# and "diff.png".
for f in os.listdir(diff_dir):
filepath = os.path.join(diff_dir, f)
if f.startswith("input-"):
logging.error("Image produced by %s: file://%s", image_name,
filepath)
elif f.startswith("closest-"):
logging.error("Closest image for %s: file://%s", image_name,
filepath)
elif f == "diff.png":
logging.error("Diff image for %s: file://%s", image_name, filepath)
if self._ShouldReportGoldFailure(page):
raise Exception('goldctl command failed, see above for details')
# pylint: enable=too-many-locals
self.artifacts.CreateLink('gold_triage_link', triage_link)
elif status == status_codes.COMPARISON_FAILURE_LOCAL:
logging.error('Local comparison failed. Local diff files:')
_OutputLocalDiffFiles(gold_session, image_name)
elif status == status_codes.LOCAL_DIFF_FAILURE:
logging.error(
'Local comparison failed and an error occurred during diff '
'generation: %s', error)
# There might be some files, so try outputting them.
logging.error('Local diff files:')
_OutputLocalDiffFiles(gold_session, image_name)
else:
logging.error(
'Given unhandled SkiaGoldSession StatusCode %s with error %s', status,
error)
if self._ShouldReportGoldFailure(page):
raise Exception('goldctl command failed, see above for details')
def _ShouldReportGoldFailure(self, page):
"""Determines if a Gold failure should actually be surfaced.
@ -565,7 +473,7 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
return True
def _ValidateScreenshotSamplesWithSkiaGold(self, tab, page, screenshot,
device_pixel_ratio, build_id_args):
device_pixel_ratio):
"""Samples the given screenshot and verifies pixel color values.
In case any of the samples do not match the expected color, it raises
@ -576,7 +484,6 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
page: the GPU PixelTestPage object for the test.
screenshot: the screenshot of the test page as a Telemetry Bitmap.
device_pixel_ratio: the device pixel ratio for the test device as a float.
build_id_args: a list of build-identifying flags and values.
"""
try:
self._CompareScreenshotSamples(
@ -589,8 +496,7 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
# We want to report the screenshot comparison failure, not any failures
# related to Gold.
try:
self._UploadTestResultToSkiaGold(
image_name, screenshot, page, build_id_args=build_id_args)
self._UploadTestResultToSkiaGold(image_name, screenshot, page)
except Exception as gold_exception:
logging.error(str(gold_exception))
# TODO(https://crbug.com/1043129): Switch this to just "raise" once these
@ -600,50 +506,6 @@ class SkiaGoldIntegrationTestBase(gpu_integration_test.GpuIntegrationTest):
# _CompareScreenshotSamples. See https://stackoverflow.com/q/28698622.
raise comparison_exception
@classmethod
def _IsLocalRun(cls):
"""Returns whether the test is running on a local workstation or not."""
# Do nothing if we've already determine whether we're in local mode or not.
if cls._local_run is not None:
pass
# Use the --local-run value if it's been set.
elif cls.GetParsedCommandLineOptions().local_run is not None:
cls._local_run = cls.GetParsedCommandLineOptions().local_run
# Look for the presence of the SWARMING_SERVER environment variable as a
# heuristic to determine whether we're running on a workstation or a bot.
# This should always be set on swarming, but would be strange to be set on
# a workstation.
cls._local_run = 'SWARMING_SERVER' not in os.environ
if cls._local_run:
logging.warning(
'Automatically determined that test is running on a workstation')
else:
logging.warning('Automatically determined that test is running on a bot')
return cls._local_run
@classmethod
def _GetBuildRevision(cls):
"""Returns the current git master revision being tested."""
# Do nothing if we've already determined the git revision.
if cls._git_revision is not None:
pass
# use the --git-revision value if it's been set.
elif cls.GetParsedCommandLineOptions().git_revision:
cls._git_revision = cls.GetParsedCommandLineOptions().git_revision
# Try to determine what revision we're on using git.
else:
try:
cls._git_revision = subprocess.check_output(
['git', 'rev-parse', 'origin/master'],
shell=IsWin(),
cwd=path_util.GetChromiumSrcDir()).strip()
logging.warning('Automatically determined git revision to be %s',
cls._git_revision)
except subprocess.CalledProcessError:
raise Exception('--git-revision not passed, and unable to '
'determine revision using git')
return cls._git_revision
@classmethod
def GenerateGpuTests(cls, options):
del options
@ -702,6 +564,24 @@ def _StripAngleRevisionFromDriver(img_params):
img_params.driver_version = '.'.join(kept_parts)
def _OutputLocalDiffFiles(gold_session, image_name):
"""Logs the local diff image files from the given SkiaGoldSession.
Args:
gold_session: A skia_gold_session.SkiaGoldSession instance to pull files
from.
image_name: A string containing the name of the image/test that was
compared.
"""
given_file = gold_session.GetGivenImageLink(image_name)
closest_file = gold_session.GetClosestImageLink(image_name)
diff_file = gold_session.GetDiffImageLink(image_name)
failure_message = 'Unable to retrieve link'
logging.error('Generated image: %s', given_file or failure_message)
logging.error('Closest image: %s', closest_file or failure_message)
logging.error('Diff image: %s', diff_file or failure_message)
def load_tests(loader, tests, pattern):
del loader, tests, pattern # Unused.
return gpu_integration_test.LoadAllTestsInModule(sys.modules[__name__])

14
docs/gpu/gpu_testing.md

@ -278,11 +278,11 @@ being tested. This *should* mean that the pixel tests will automatically work
when run locally. However, if the local run detection code fails for some
reason, you can manually pass some flags to force the same behavior:
In order to get around the local run issues, simply pass the `--local-run=1`
flag to the tests. This will disable uploading, but otherwise go through the
same steps as a test normally would. Each test will also print out `file://`
URLs to the produced image, the closest image for the test known to Gold, and
the diff between the two.
In order to get around the local run issues, simply pass the
`--local-pixel-tests` flag to the tests. This will disable uploading, but
otherwise go through the same steps as a test normally would. Each test will
also print out `file://` URLs to the produced image, the closest image for the
test known to Gold, and the diff between the two.
Because the image produced by the test locally is likely slightly different from
any of the approved images in Gold, local test runs are likely to fail during
@ -292,7 +292,7 @@ comparison. When using `--no-skia-gold-failure`, you'll also need to pass the
`--passthrough` flag in order to actually see the link output.
Example usage:
`run_gpu_integration_test.py pixel --no-skia-gold-failure --local-run=1
`run_gpu_integration_test.py pixel --no-skia-gold-failure --local-pixel-tests
--passthrough`
If, for some reason, the local run code is unable to determine what the git
@ -300,7 +300,7 @@ revision is, simply pass `--git-revision aabbccdd`. Note that `aabbccdd` must
be replaced with an actual Chromium src revision (typically whatever revision
origin/master is currently synced to) in order for the tests to work. This can
be done automatically using:
``run_gpu_integration_test.py pixel --no-skia-gold-failure --local-run
``run_gpu_integration_test.py pixel --no-skia-gold-failure --local-pixel-tests
--passthrough --git-revision `git rev-parse origin/master` ``
## Running Binaries from the Bots Locally