Add GPU inexact matching optimization script

Adds a script to find optimal values for inexact matching in the GPU
pixel tests and enables inexact matching on several problematic tests
that could benefit from it.

Bug: 1074130
Change-Id: If4597bbda1a94fdc4dd28f87ca59ea86489c2ac1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2173415
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
Cr-Commit-Position: refs/heads/master@{#764424}

content/test/gpu/gold_inexact_matching/base_parameter_optimizer.py

@@ -0,0 +1,435 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import glob
+import itertools
+import io
+import logging
+import multiprocessing
+import os
+from PIL import Image
+import requests
+import shutil
+import subprocess
+import sys
+import tempfile
+
+import parameter_set
+
+CHROMIUM_SRC_DIR = os.path.realpath(
+    os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
+GOLDCTL_PATHS = [
+    os.path.join(CHROMIUM_SRC_DIR, 'tools', 'skia_goldctl', 'linux', 'goldctl'),
+    os.path.join(CHROMIUM_SRC_DIR, 'tools', 'skia_goldctl', 'mac', 'goldctl'),
+    os.path.join(CHROMIUM_SRC_DIR, 'tools', 'skia_goldctl', 'win',
+                 'goldctl.exe'),
+]
+
+
+class BaseParameterOptimizer(object):
+  """Abstract base class for running a parameter optimization for a test."""
+  MIN_EDGE_THRESHOLD = 0
+  MAX_EDGE_THRESHOLD = 255
+  MIN_MAX_DIFF = 0
+  MIN_DELTA_THRESHOLD = 0
+  # 4 channels, ranging from 0-255 each.
+  MAX_DELTA_THRESHOLD = 255 * 4
+
+  def __init__(self, args, test_name):
+    """
+    Args:
+      args: The parse arguments from an argparse.ArgumentParser.
+      test_name: The name of the test to optimize.
+    """
+    self._args = args
+    self._test_name = test_name
+    self._goldctl_binary = None
+    self._working_dir = None
+    self._expectations = None
+    self._gold_url = 'https://%s-gold.skia.org' % args.gold_instance
+    # A map of strings, denoting a resolution or trace, to an iterable of
+    # strings, denoting images that are that dimension or belong to that
+    # trace.
+    self._images = {}
+    self._VerifyArgs()
+
+  @classmethod
+  def AddArguments(cls, parser):
+    """Add optimizer-specific arguments to the parser.
+
+    Args:
+      parser: An argparse.ArgumentParser instance.
+
+    Returns:
+      A 3-tuple (common_group, sobel_group, fuzzy_group). All three are
+      argument groups of |parser| corresponding to arguments for any sort of
+      inexact matching algorithm, arguments specific to Sobel filter matching,
+      and arguments specific to fuzzy matching.
+    """
+    common_group = parser.add_argument_group('Common Arguments')
+    common_group.add_argument(
+        '--test',
+        required=True,
+        action='append',
+        dest='test_names',
+        help='The name of a test to find parameter values for, as reported in '
+        'the Skia Gold UI. Can be passed multiple times to run optimizations '
+        'for multiple tests.')
+    common_group.add_argument(
+        '--gold-instance',
+        default='chrome-gpu',
+        help='The Skia Gold instance to interact with.')
+    common_group.add_argument(
+        '--corpus',
+        default='chrome-gpu',
+        help='The corpus within the instance to interact with.')
+    common_group.add_argument(
+        '--target-success-percent',
+        default=100,
+        type=float,
+        help='The percentage of comparisons that need to succeed in order for '
+        'a set of parameters to be considered good.')
+    common_group.add_argument(
+        '--no-cleanup',
+        action='store_true',
+        default=False,
+        help="Don't clean up the temporary files left behind by the "
+        'optimization process.')
+    common_group.add_argument(
+        '--group-images-by-resolution',
+        action='store_true',
+        default=False,
+        help='Group images for comparison based on resolution instead of by '
+        'Gold trace. This will likely add some noise, as some comparisons will '
+        'be made that Gold would not consider, but this has the benefit of '
+        'optimizing over all historical data instead of only over data in '
+        'the past several hundred commits. Note that this will likely '
+        'result in a significantly longer runtime.')
+
+    sobel_group = parser.add_argument_group(
+        'Sobel Arguments',
+        'To disable Sobel functionality, set both min and max edge thresholds '
+        'to 255.')
+    sobel_group.add_argument(
+        '--min-edge-threshold',
+        default=10,
+        type=int,
+        help='The minimum value to consider for the Sobel edge threshold. '
+        'Lower values result in more of the image being blacked out before '
+        'comparison.')
+    sobel_group.add_argument(
+        '--max-edge-threshold',
+        default=255,
+        type=int,
+        help='The maximum value to consider for the Sobel edge threshold. '
+        'Higher values result in less of the image being blacked out before '
+        'comparison.')
+
+    fuzzy_group = parser.add_argument_group(
+        'Fuzzy Arguments',
+        'To disable Fuzzy functionality, set min/max for both parameters to 0')
+    fuzzy_group.add_argument(
+        '--min-max-different-pixels',
+        dest='min_max_diff',
+        default=0,
+        type=int,
+        help='The minimum value to consider for the maximum number of '
+        'different pixels. Lower values result in less fuzzy comparisons being '
+        'allowed.')
+    fuzzy_group.add_argument(
+        '--max-max-different-pixels',
+        dest='max_max_diff',
+        default=50,
+        type=int,
+        help='The maximum value to consider for the maximum number of '
+        'different pixels. Higher values result in more fuzzy comparisons '
+        'being allowed.')
+    fuzzy_group.add_argument(
+        '--min-delta-threshold',
+        default=0,
+        type=int,
+        help='The minimum value to consider for the per-channel delta sum '
+        'threshold. Lower values result in less fuzzy comparisons being '
+        'allowed.')
+    fuzzy_group.add_argument(
+        '--max-delta-threshold',
+        default=30,
+        type=int,
+        help='The maximum value to consider for the per-channel delta sum '
+        'threshold. Higher values result in more fuzzy comparisons being '
+        'allowed.')
+
+    return common_group, sobel_group, fuzzy_group
+
+  def _VerifyArgs(self):
+    """Verifies that the provided arguments are valid for an optimizer."""
+    assert self._args.target_success_percent > 0
+    assert self._args.target_success_percent <= 100
+
+    assert self._args.min_edge_threshold >= self.MIN_EDGE_THRESHOLD
+    assert self._args.max_edge_threshold <= self.MAX_EDGE_THRESHOLD
+    assert self._args.min_edge_threshold <= self._args.max_edge_threshold
+
+    assert self._args.min_max_diff >= self.MIN_MAX_DIFF
+    assert self._args.min_max_diff <= self._args.max_max_diff
+    assert self._args.min_delta_threshold >= self.MIN_DELTA_THRESHOLD
+    assert self._args.max_delta_threshold <= self.MAX_DELTA_THRESHOLD
+    assert self._args.min_delta_threshold <= self._args.max_delta_threshold
+
+  def RunOptimization(self):
+    """Runs an optimization for whatever test and parameters were supplied.
+
+    The results should be printed to stdout when they are available.
+    """
+    self._working_dir = tempfile.mkdtemp()
+    try:
+      self._DownloadData()
+
+      # Do a preliminary test to make sure that the most permissive
+      # parameters can succeed.
+      logging.info('Verifying initial parameters')
+      success, num_pixels, max_delta = self._RunComparisonForParameters(
+          self._GetMostPermissiveParameters())
+      if not success:
+        raise RuntimeError(
+            'Most permissive parameters did not result in a comparison '
+            'success. Try loosening parameters or lowering target success '
+            'percent. Max differing pixels: %d, max delta: %s' % (num_pixels,
+                                                                  max_delta))
+
+      self._RunOptimizationImpl()
+
+    finally:
+      if not self._args.no_cleanup:
+        shutil.rmtree(self._working_dir)
+        # Cleanup files left behind by "goldctl match"
+        for f in glob.iglob(os.path.join(tempfile.gettempdir(), 'goldctl-*')):
+          shutil.rmtree(f)
+
+  def _RunOptimizationImpl(self):
+    """Runs the algorithm-specific optimization code for an optimizer."""
+    raise NotImplementedError()
+
+  def _GetMostPermissiveParameters(self):
+    return parameter_set.ParameterSet(self._args.max_max_diff,
+                                      self._args.max_delta_threshold,
+                                      self._args.min_edge_threshold)
+
+  def _DownloadData(self):
+    """Downloads all the necessary data for a test."""
+    assert self._working_dir
+    logging.info('Downloading images')
+    if self._args.group_images_by_resolution:
+      self._DownloadExpectations('%s/json/expectations' % self._gold_url)
+      self._DownloadImagesForResolutionGrouping()
+    else:
+      self._DownloadExpectations(
+          '%s/json/debug/digestsbytestname/%s/%s' %
+          (self._gold_url, self._args.corpus, self._test_name))
+      self._DownloadImagesForTraceGrouping()
+    for grouping, digests in self._images.iteritems():
+      logging.info('Found %d images for group %s', len(digests), grouping)
+      logging.debug('Digests: %r', digests)
+
+  def _DownloadExpectations(self, url):
+    """Downloads the expectation JSON from Gold into memory."""
+    logging.info('Downloading expectations JSON')
+    r = requests.get(url)
+    assert r.status_code == 200
+    self._expectations = r.json()
+
+  def _DownloadImagesForResolutionGrouping(self):
+    """Downloads all the positive images for a test to disk.
+
+    Images are grouped by resolution.
+    """
+    assert self._expectations
+    test_expectations = self._expectations.get('master', {}).get(
+        self._test_name, {})
+    positive_digests = [
+        digest for digest, val in test_expectations.items() if val == 1
+    ]
+    if not positive_digests:
+      raise RuntimeError('Failed to find any positive digests for test %s',
+                         self._test_name)
+    for digest in positive_digests:
+      content = self._DownloadImageWithDigest(digest)
+      image = Image.open(io.BytesIO(content))
+      self._images.setdefault('%dx%d' % (image.size[0], image.size[1]),
+                              []).append(digest)
+
+  def _DownloadImagesForTraceGrouping(self):
+    """Download all recent positive images for a test to disk.
+
+    Images are grouped by Skia Gold trace ID, i.e. each hardware/software
+    combination is a separate group.
+    """
+    assert self._expectations
+    # The downloaded trace data maps a trace ID (string) to a list of digests.
+    # The digests can be empty (which we don't care about) or duplicated, so
+    # convert to a set and filter out the empty strings.
+    filtered_traces = {}
+    for trace, digests in self._expectations.iteritems():
+      filtered_digests = set(digests)
+      filtered_digests.discard('')
+      if not filtered_digests:
+        logging.warning(
+            'Failed to find any positive digests for test %s and trace %s. '
+            'This is likely due to the trace being old.', self._test_name,
+            trace)
+      filtered_traces[trace] = filtered_digests
+      for digest in filtered_digests:
+        self._DownloadImageWithDigest(digest)
+    self._images = filtered_traces
+
+  def _DownloadImageWithDigest(self, digest):
+    """Downloads an image with the given digest and saves it to disk.
+
+    Args:
+      digest: The md5 digest of the image to download.
+
+    Returns:
+      A copy of the image content that was written to disk as bytes.
+    """
+    logging.debug('Downloading image %s.png', digest)
+    r = requests.get('%s/img/images/%s.png' % (self._gold_url, digest))
+    assert r.status_code == 200
+    with open(self._GetImagePath(digest), 'wb') as outfile:
+      outfile.write(r.content)
+    return r.content
+
+  def _GetImagePath(self, digest):
+    """Gets a filepath to an image based on digest.
+
+    Args:
+      digest: The md5 digest of the image, as provided by Gold.
+
+    Returns:
+      A string containing a filepath to where the image should be on disk.
+    """
+    return os.path.join(self._working_dir, '%s.png' % digest)
+
+  def _GetGoldctlBinary(self):
+    """Gets the filepath to the goldctl binary to use.
+
+    Returns:
+      A string containing a filepath to the goldctl binary to use.
+    """
+    if not self._goldctl_binary:
+      for path in GOLDCTL_PATHS:
+        if os.path.isfile(path):
+          self._goldctl_binary = path
+          break
+      if not self._goldctl_binary:
+        raise RuntimeError(
+            'Could not find goldctl binary. Checked %s' % GOLDCTL_PATHS)
+    return self._goldctl_binary
+
+  def _RunComparisonForParameters(self, parameters):
+    """Runs a comparison for all image combinations using some parameters.
+
+    Args:
+      parameters: A parameter_set.ParameterSet instance containing parameters to
+          use.
+
+    Returns:
+      A 3-tuple (success, num_pixels, max_diff). |success| is a boolean
+      denoting whether enough comparisons succeeded to meet the desired success
+      percentage. |num_pixels| is an int denoting the maximum number of pixels
+      that did not match across all comparisons. |max_delta| is the maximum
+      per-channel delta sum across all comparisons.
+    """
+    logging.debug('Running comparison for parameters: %s', parameters)
+    num_attempts = 0
+    num_successes = 0
+    max_num_pixels = -1
+    max_max_delta = -1
+
+    process_pool = multiprocessing.Pool()
+    for resolution, digest_list in self._images.iteritems():
+      logging.debug('Resolution/trace: %s, digests: %s', resolution,
+                    digest_list)
+      cmds = [
+          self._GenerateComparisonCmd(l, r, parameters)
+          for (l, r) in itertools.combinations(digest_list, 2)
+      ]
+      results = process_pool.map(RunCommandAndExtractData, cmds)
+      for (success, num_pixels, max_delta) in results:
+        num_attempts += 1
+        if success:
+          num_successes += 1
+        max_num_pixels = max(num_pixels, max_num_pixels)
+        max_max_delta = max(max_delta, max_max_delta)
+
+    # This could potentially happen if run on a test where there's only one
+    # positive image per resolution/trace.
+    if num_attempts == 0:
+      num_attempts = 1
+      num_successes = 1
+    success_percent = float(num_successes) * 100 / num_attempts
+    logging.debug('Success percent: %s', success_percent)
+    logging.debug('target success percent: %s',
+                  self._args.target_success_percent)
+    successful = success_percent >= self._args.target_success_percent
+    logging.debug(
+        'Successful: %s, Max different pixels: %d, Max per-channel delta sum: '
+        '%d', successful, max_num_pixels, max_max_delta)
+    return successful, max_num_pixels, max_max_delta
+
+  def _GenerateComparisonCmd(self, left_digest, right_digest, parameters):
+    """Generates a comparison command for the given arguments.
+
+    The returned command can be passed directly to a subprocess call.
+
+    Args:
+      left_digest: The first/left image digest to compare.
+      right_digest: The second/right image digest to compare.
+      parameters: A parameter_set.ParameterSet instance containing the
+          parameters to use for image comparison.
+
+    Returns:
+      A list of strings specifying a goldctl command to compare |left_digest|
+      to |right_digest| using the parameters in |parameters|.
+    """
+    cmd = [
+        self._GetGoldctlBinary(),
+        'match',
+        self._GetImagePath(left_digest),
+        self._GetImagePath(right_digest),
+        '--algorithm',
+        'sobel',
+    ] + parameters.AsList()
+    return cmd
+
+
+def RunCommandAndExtractData(cmd):
+  """Runs a comparison command and extracts data from it.
+
+  This is outside of the parameter optimizers because it is meant to be run via
+  multiprocessing.Pool.map(), which does not play nice with class methods since
+  they can't be easily pickled.
+
+  Args:
+    cmd: A list of strings containing the command to run.
+
+  Returns:
+    A 3-tuple (success, num_pixels, max_delta). |success| is a boolean denoting
+    whether the comparison succeeded or not. |num_pixels| is an int denoting
+    the number of pixels that did not match. |max_delta| is the maximum
+    per-channel delta sum in the comparison.
+  """
+  output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+  success = False
+  num_pixels = 0
+  max_delta = 0
+  for line in output.splitlines():
+    if 'Images match.' in line:
+      success = True
+    if 'Number of different pixels' in line:
+      num_pixels = int(line.split(':')[1])
+    if 'Maximum per-channel delta sum' in line:
+      max_delta = int(line.split(':')[1])
+  logging.debug('Result for %r: success: %s, num_pixels: %d, max_delta: %d',
+                cmd, success, num_pixels, max_delta)
+  return success, num_pixels, max_delta

content/test/gpu/gold_inexact_matching/binary_search_parameter_optimizer.py

@@ -0,0 +1,97 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+
+import base_parameter_optimizer as base_optimizer
+import parameter_set
+
+
+class BinarySearchParameterOptimizer(base_optimizer.BaseParameterOptimizer):
+  """A ParameterOptimizer for use with a single changing parameter.
+
+  The ideal optimizer if only one parameter needs to be varied, e.g. finding
+  the best Sobel edge threshold to use when not using any additional fuzzy
+  diffing.
+  """
+  UNLOCKED_PARAM_MAX_DIFF = 1
+  UNLOCKED_PARAM_DELTA_THRESHOLD = 2
+  UNLOCKED_PARAM_EDGE_THRESHOLD = 3
+
+  def __init__(self, args, test_name):
+    self._unlocked_parameter = None
+    super(BinarySearchParameterOptimizer, self).__init__(args, test_name)
+
+  def _VerifyArgs(self):
+    super(BinarySearchParameterOptimizer, self)._VerifyArgs()
+
+    max_diff_locked = self._args.max_max_diff == self._args.min_max_diff
+    delta_threshold_locked = (
+        self._args.max_delta_threshold == self._args.min_delta_threshold)
+    edge_threshold_locked = (
+        self._args.max_edge_threshold == self._args.min_delta_threshold)
+
+    if not ((max_diff_locked ^ delta_threshold_locked) or
+            (delta_threshold_locked ^ edge_threshold_locked)):
+      raise RuntimeError(
+          'Binary search optimization requires all but one parameter to be '
+          'locked (min == max).')
+
+    if not max_diff_locked:
+      self._unlocked_parameter = self.UNLOCKED_PARAM_MAX_DIFF
+    elif not delta_threshold_locked:
+      self._unlocked_parameter = self.UNLOCKED_PARAM_DELTA_THRESHOLD
+    else:
+      self._unlocked_parameter = self.UNLOCKED_PARAM_EDGE_THRESHOLD
+
+  def _RunOptimizationImpl(self):
+    known_good, known_bad = self._GetStartingValues()
+    while (abs(known_good - known_bad)) > 1:
+      midpoint = (known_good + known_bad) / 2
+      parameters = self._CreateParameterSet(midpoint)
+      success, num_pixels, max_diff = self._RunComparisonForParameters(
+          parameters)
+      if success:
+        logging.info('Found good parameters %s', parameters)
+        known_good = midpoint
+      else:
+        logging.info('Found bad parameters %s', parameters)
+        known_bad = midpoint
+    print 'Found optimal parameters: %s' % parameters
+
+  def _GetStartingValues(self):
+    """Gets the initial good/bad values for the binary search.
+
+    Returns:
+      A tuple (known_good, assumed_bad). |known_good| is a value that is known
+      to make the comparison succeed. |assumed_bad| is a value that is expected
+      to make the comparison fail, although it has not necessarily been tested
+      yet.
+    """
+    if self._unlocked_parameter == self.UNLOCKED_PARAM_MAX_DIFF:
+      return self._args.max_max_diff, self._args.min_max_diff
+    elif self._unlocked_parameter == self.UNLOCKED_PARAM_DELTA_THRESHOLD:
+      return self._args.max_delta_threshold, self._args.min_delta_threshold
+    else:
+      return self._args.min_edge_threshold, self._args.max_edge_threshold
+
+  def _CreateParameterSet(self, value):
+    """Creates a parameter_set.ParameterSet to test.
+
+    Args:
+      value: The value to set the variable parameter to.
+
+    Returns:
+      A parameter_set.ParameterSet with the variable parameter set to |value|
+      and the other parameters set to their fixed values.
+    """
+    if self._unlocked_parameter == self.UNLOCKED_PARAM_MAX_DIFF:
+      return parameter_set.ParameterSet(value, self._args.min_delta_threshold,
+                                        self._args.min_edge_threshold)
+    elif self._unlocked_parameter == self.UNLOCKED_PARAM_DELTA_THRESHOLD:
+      return parameter_set.ParameterSet(self._args.min_max_diff, value,
+                                        self._args.min_edge_threshold)
+    else:
+      return parameter_set.ParameterSet(self._args.min_max_diff,
+                                        self._args.min_delta_threshold, value)

content/test/gpu/gold_inexact_matching/brute_force_parameter_optimizer.py

@@ -0,0 +1,46 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+
+import iterative_parameter_optimizer as iterative_optimizer
+import parameter_set
+
+
+class BruteForceParameterOptimizer(
+    iterative_optimizer.IterativeParameterOptimizer):
+  """A ParameterOptimizer for use with any number of changing parameters.
+
+  VERY slow, but provides the most complete information when trying to
+  optimize multiple parameters.
+  """
+
+  def _VerifyArgs(self):
+    # range/xrange(x, y) returns the range in [x, y), so adjust by 1 to make
+    # the range inclusive.
+    # We go from max to min instead of min to max for this parameter, so
+    # decrease the min by 1 instead of increasing the max by 1.
+    self._args.min_edge_threshold -= 1
+    self._args.max_max_diff += 1
+    self._args.max_delta_threshold += 1
+
+  def _RunOptimizationImpl(self):
+    should_continue = True
+    # Look for the minimum max_delta that results in a successful comparison
+    # for each possible edge_threshold/max_diff combination.
+    for edge_threshold in xrange(self._args.max_edge_threshold,
+                                 self._args.min_edge_threshold,
+                                 -1 * self._args.edge_threshold_step):
+      for max_diff in xrange(self._args.min_max_diff, self._args.max_max_diff,
+                             self._args.max_diff_step):
+        for max_delta in xrange(self._args.min_delta_threshold,
+                                self._args.max_delta_threshold,
+                                self._args.delta_threshold_step):
+          parameters = parameter_set.ParameterSet(max_diff, max_delta,
+                                                  edge_threshold)
+          success, _, _ = self._RunComparisonForParameters(parameters)
+          if success:
+            print 'Found good parameters %s' % parameters
+            break
+          logging.info('Found bad parameters %s', parameters)

content/test/gpu/gold_inexact_matching/determine_gold_inexact_parameters.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env vpython
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import logging
+import sys
+
+import base_parameter_optimizer as base_optimizer
+import binary_search_parameter_optimizer as binary_optimizer
+import brute_force_parameter_optimizer as brute_optimizer
+import local_minima_parameter_optimizer as local_optimizer
+import optimizer_set
+"""Script to find suitable values for Skia Gold inexact matching.
+
+Inexact matching in Skia Gold has three tunable parameters:
+  1. The max number of differing pixels.
+  2. The max delta for any single pixel.
+  3. The threshold for a Sobel filter.
+
+Ideally, we use the following hierarchy of comparison approaches:
+  1. Exact matching.
+  2. Exact matching after a Sobel filter is applied.
+  3. Fuzzy matching after a Sobel filter is applied.
+
+However, there may be cases where only using a Sobel filter requires masking a
+very large amount of the image compared to Sobel + very conservative fuzzy
+matching.
+
+Even if such cases are not hit, the process of determining good values for the
+parameters is quite tedious since it requires downloading images from Gold and
+manually running multiple calls to `goldctl match`.
+
+This script attempts to remedy both issues by handling all of the trial and
+error and suggesting potential parameter values for the user to choose from.
+"""
+
+
+def CreateArgumentParser():
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  script_parser = parser.add_argument_group('Script Arguments')
+  script_parser.add_argument(
+      '-v',
+      '--verbose',
+      dest='verbose_count',
+      default=0,
+      action='count',
+      help='Verbose level (multiple times for more')
+
+  subparsers = parser.add_subparsers(help='Optimization algorithm')
+
+  binary_parser = subparsers.add_parser(
+      'binary_search',
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+      help='Perform a binary search to optimize a single parameter. The best '
+      'option if you only want to tune one parameter.')
+  binary_parser.set_defaults(
+      clazz=binary_optimizer.BinarySearchParameterOptimizer)
+  binary_optimizer.BinarySearchParameterOptimizer.AddArguments(binary_parser)
+
+  local_parser = subparsers.add_parser(
+      'local_minima',
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+      help='Perform a BFS to find local minima using weights for each '
+      'parameter. Slower than binary searching, but supports an arbitrary '
+      'number of parameters.')
+  local_parser.set_defaults(clazz=local_optimizer.LocalMinimaParameterOptimizer)
+  local_optimizer.LocalMinimaParameterOptimizer.AddArguments(local_parser)
+
+  brute_parser = subparsers.add_parser(
+      'brute_force',
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+      help='Brute force all possible combinations. VERY, VERY slow, but can '
+      'potentially find better values than local_minima.')
+  brute_parser.set_defaults(clazz=brute_optimizer.BruteForceParameterOptimizer)
+  brute_optimizer.BruteForceParameterOptimizer.AddArguments(brute_parser)
+
+  return parser
+
+
+def SetLoggingVerbosity(args):
+  logger = logging.getLogger()
+  if args.verbose_count == 0:
+    logger.setLevel(logging.WARNING)
+  elif args.verbose_count == 1:
+    logger.setLevel(logging.INFO)
+  else:
+    logger.setLevel(logging.DEBUG)
+
+
+def main():
+  parser = CreateArgumentParser()
+  args = parser.parse_args()
+  SetLoggingVerbosity(args)
+  optimizer = optimizer_set.OptimizerSet(args, args.clazz)
+  optimizer.RunOptimization()
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

content/test/gpu/gold_inexact_matching/iterative_parameter_optimizer.py

@@ -0,0 +1,51 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import base_parameter_optimizer as base_optimizer
+
+
+class IterativeParameterOptimizer(base_optimizer.BaseParameterOptimizer):
+  """Abstract ParameterOptimizer class for running an iterative algorithm."""
+
+  MIN_EDGE_THRESHOLD_STEP = 0
+  MAX_EDGE_THRESHOLD_STEP = (
+      base_optimizer.BaseParameterOptimizer.MAX_EDGE_THRESHOLD)
+  MIN_MAX_DIFF_STEP = MIN_DELTA_THRESHOLD_STEP = 0
+  MAX_DELTA_THRESHOLD_STEP = (
+      base_optimizer.BaseParameterOptimizer.MAX_DELTA_THRESHOLD)
+
+  @classmethod
+  def AddArguments(cls, parser):
+    common_group, sobel_group, fuzzy_group = super(IterativeParameterOptimizer,
+                                                   cls).AddArguments(parser)
+
+    sobel_group.add_argument(
+        '--edge-threshold-step',
+        default=10,
+        type=int,
+        help='The amount to change the Sobel edge threshold on each iteration.')
+
+    fuzzy_group.add_argument(
+        '--max-diff-step',
+        default=10,
+        type=int,
+        help='The amount to change the fuzzy diff maximum number of different '
+        'pixels on each iteration.')
+    fuzzy_group.add_argument(
+        '--delta-threshold-step',
+        default=5,
+        type=int,
+        help='The amount to change the fuzzy diff per-channel delta sum '
+        'threshold on each iteration.')
+
+    return common_group, sobel_group, fuzzy_group
+
+  def _VerifyArgs(self):
+    super(IterativeParameterOptimizer, self)._VerifyArgs()
+
+    assert self._args.edge_threshold_step >= self.MIN_EDGE_THRESHOLD_STEP
+    assert self._args.edge_threshold_step <= self.MAX_EDGE_THRESHOLD_STEP
+
+    assert self._args.max_diff_step >= self.MIN_MAX_DIFF_STEP
+    assert self._args.delta_threshold_step >= self.MIN_DELTA_THRESHOLD_STEP

content/test/gpu/gold_inexact_matching/local_minima_parameter_optimizer.py

@@ -0,0 +1,143 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import itertools
+import logging
+import sys
+
+import iterative_parameter_optimizer as iterative_optimizer
+import parameter_set
+
+
+class LocalMinimaParameterOptimizer(
+    iterative_optimizer.IterativeParameterOptimizer):
+  """A ParameterOptimizer to find local minima.
+
+  Works on any number of variable parameters and is faster than brute
+  forcing, but not guaranteed to find all interesting parameter combinations.
+  """
+  MIN_EDGE_THRESHOLD_WEIGHT = 0
+  MIN_MAX_DIFF_WEIGHT = MIN_DELTA_THRESHOLD_WEIGHT = 0
+
+  @classmethod
+  def AddArguments(cls, parser):
+    common_group, sobel_group, fuzzy_group = super(
+        LocalMinimaParameterOptimizer, cls).AddArguments(parser)
+
+    common_group.add_argument(
+        '--use-bfs',
+        action='store_true',
+        default=False,
+        help='Use a breadth-first search instead of a depth-first search. This '
+        'will likely be significantly slower, but is more likely to find '
+        'multiple local minima with the same weight.')
+
+    sobel_group.add_argument(
+        '--edge-threshold-weight',
+        default=1,
+        type=int,
+        help='The weight associated with the edge threshold. Higher values '
+        'will penalize a more permissive parameter value more harshly.')
+
+    fuzzy_group.add_argument(
+        '--max-diff-weight',
+        default=3,
+        type=int,
+        help='The weight associated with the maximum number of different '
+        'pixels. Higher values will penalize a more permissive parameter value '
+        'more harshly.')
+    fuzzy_group.add_argument(
+        '--delta-threshold-weight',
+        default=10,
+        type=int,
+        help='The weight associated with the per-channel delta sum. Higher '
+        'values will penalize a more permissive parameter value more harshly.')
+
+    return common_group, sobel_group, fuzzy_group
+
+  def _VerifyArgs(self):
+    super(LocalMinimaParameterOptimizer, self)._VerifyArgs()
+
+    assert self._args.edge_threshold_weight >= self.MIN_EDGE_THRESHOLD_WEIGHT
+
+    assert self._args.max_diff_weight >= self.MIN_MAX_DIFF_WEIGHT
+    assert self._args.delta_threshold_weight >= self.MIN_DELTA_THRESHOLD_WEIGHT
+
+  def _RunOptimizationImpl(self):
+    visited_parameters = set()
+    to_visit = collections.deque()
+    smallest_weight = sys.maxsize
+    smallest_parameters = []
+
+    to_visit.append(self._GetMostPermissiveParameters())
+    # Do a search, only considering adjacent parameters if:
+    # 1. Their weight is less than or equal to the smallest found weight.
+    # 2. They haven't been visited already.
+    # 3. The current parameters result in a successful comparison.
+    while len(to_visit):
+      current_parameters = None
+      if self._args.use_bfs:
+        current_parameters = to_visit.popleft()
+      else:
+        current_parameters = to_visit.pop()
+      weight = self._GetWeight(current_parameters)
+      if weight > smallest_weight:
+        continue
+      if current_parameters in visited_parameters:
+        continue
+      visited_parameters.add(current_parameters)
+      success, _, _ = self._RunComparisonForParameters(current_parameters)
+      if success:
+        for adjacent in self._AdjacentParameters(current_parameters):
+          to_visit.append(adjacent)
+        if smallest_weight == weight:
+          logging.info('Found additional smallest parameter %s',
+                       current_parameters)
+          smallest_parameters.append(current_parameters)
+        else:
+          logging.info('Found new smallest parameter with weight %d: %s',
+                       weight, current_parameters)
+          smallest_weight = weight
+          smallest_parameters = [current_parameters]
+    print 'Found %d parameter(s) with the smallest weight:' % len(
+        smallest_parameters)
+    for p in smallest_parameters:
+      print p
+
+  def _AdjacentParameters(self, starting_parameters):
+    max_diff = starting_parameters.max_diff
+    delta_threshold = starting_parameters.delta_threshold
+    edge_threshold = starting_parameters.edge_threshold
+
+    max_diff_step = self._args.max_diff_step
+    delta_threshold_step = self._args.delta_threshold_step
+    edge_threshold_step = self._args.edge_threshold_step
+
+    max_diffs = [
+        max(self._args.min_max_diff, max_diff - max_diff_step), max_diff,
+        min(self._args.max_max_diff, max_diff + max_diff_step)
+    ]
+    delta_thresholds = [
+        max(self._args.min_delta_threshold,
+            delta_threshold - delta_threshold_step), delta_threshold,
+        min(self._args.max_delta_threshold,
+            delta_threshold + delta_threshold_step)
+    ]
+    edge_thresholds = [
+        max(self._args.min_edge_threshold,
+            edge_threshold - edge_threshold_step), edge_threshold,
+        min(self._args.max_edge_threshold, edge_threshold + edge_threshold_step)
+    ]
+    for combo in itertools.product(max_diffs, delta_thresholds,
+                                   edge_thresholds):
+      adjacent = parameter_set.ParameterSet(combo[0], combo[1], combo[2])
+      if adjacent != starting_parameters:
+        yield adjacent
+
+  def _GetWeight(self, parameters):
+    return (parameters.max_diff * self._args.max_diff_weight +
+            parameters.delta_threshold * self._args.delta_threshold_weight +
+            (self.MAX_EDGE_THRESHOLD - parameters.edge_threshold) *
+            self._args.edge_threshold_weight)

content/test/gpu/gold_inexact_matching/optimizer_set.py

@@ -0,0 +1,23 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+class OptimizerSet(object):
+  """Class to run a ParameterOptimizer for multiple tests."""
+
+  def __init__(self, args, optimizer_class):
+    """
+    Args:
+      args: The parse arguments from an argparse.ArgumentParser.
+      optimizer_class: The optimizer class to use for the optimization.
+    """
+    self._args = args
+    self._optimizer_class = optimizer_class
+
+  def RunOptimization(self):
+    test_names = set(self._args.test_names)
+    for name in test_names:
+      print 'Running optimization for test %s' % name
+      optimizer = self._optimizer_class(self._args, name)
+      optimizer.RunOptimization()

content/test/gpu/gold_inexact_matching/parameter_set.py

@@ -0,0 +1,50 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+class ParameterSet(object):
+  """Struct-like object for holding parameters for an iteration."""
+
+  def __init__(self, max_diff, delta_threshold, edge_threshold):
+    """
+    Args:
+      max_diff: The maximum number of pixels that are allowed to differ.
+      delta_threshold: The maximum per-channel delta sum that is allowed.
+      edge_threshold: The threshold for what is considered an edge for a
+          Sobel filter.
+    """
+    self.max_diff = max_diff
+    self.delta_threshold = delta_threshold
+    self.edge_threshold = edge_threshold
+
+  def AsList(self):
+    """Returns the object's data in list format.
+
+    The returned object is suitable for appending to a "goldctl match" command
+    in order to compare using the parameters stored within the object.
+
+    Returns:
+      A list of strings.
+    """
+    return [
+        '--parameter',
+        'fuzzy_max_different_pixels:%d' % self.max_diff,
+        '--parameter',
+        'fuzzy_pixel_delta_threshold:%d' % self.delta_threshold,
+        '--parameter',
+        'sobel_edge_threshold:%d' % self.edge_threshold,
+    ]
+
+  def __str__(self):
+    return ('Max different pixels: %d, Max per-channel delta sum: %d, Sobel '
+            'edge threshold: %d') % (self.max_diff, self.delta_threshold,
+                                     self.edge_threshold)
+
+  def __eq__(self, other):
+    return (self.max_diff == other.max_diff
+            and self.delta_threshold == other.delta_threshold
+            and self.edge_threshold == other.edge_threshold)
+
+  def __hash__(self):
+    return hash((self.max_diff, self.delta_threshold, self.edge_threshold))

content/test/gpu/gpu_tests/pixel_test_pages.py

@@ -220,8 +220,7 @@ class PixelTestPages(object):
             matching_algorithm=algo.SobelMatchingAlgorithm(
                 max_different_pixels=0,
                 pixel_delta_threshold=0,
-                edge_threshold=100,
-            )),
+                edge_threshold=100)),
         PixelTestPage(
             'pixel_webgl_aa_alpha.html',
             base_name + '_WebGLGreenTriangle_AA_Alpha',
@@ -910,7 +909,11 @@ class PixelTestPages(object):
             'pixel_precision_rounded_corner.html',
             base_name + '_PrecisionRoundedCorner',
             test_rect=[0, 0, 400, 400],
-            browser_args=browser_args)
+            browser_args=browser_args,
+            matching_algorithm=algo.SobelMatchingAlgorithm(
+                max_different_pixels=10,
+                pixel_delta_threshold=30,
+                edge_threshold=100)),
     ]
 
   # Pages that should be run with off-thread paint worklet flags.
@@ -1253,13 +1256,21 @@ class PixelTestPages(object):
         PixelTestPage(
             'filter_effects.html',
             base_name + '_CSSFilterEffects',
-            test_rect=[0, 0, 300, 300]),
+            test_rect=[0, 0, 300, 300],
+            matching_algorithm=algo.SobelMatchingAlgorithm(
+                max_different_pixels=10,
+                pixel_delta_threshold=5,
+                edge_threshold=245)),
         PixelTestPage(
             'filter_effects.html',
             base_name + '_CSSFilterEffects_NoOverlays',
             test_rect=[0, 0, 300, 300],
             tolerance=10,
-            browser_args=no_overlays_args),
+            browser_args=no_overlays_args,
+            matching_algorithm=algo.SobelMatchingAlgorithm(
+                max_different_pixels=240,
+                pixel_delta_threshold=5,
+                edge_threshold=250)),
 
         # Test WebGL's premultipliedAlpha:false without the CA compositor.
         PixelTestPage(

docs/gpu/gpu_pixel_testing_with_gold.md

@@ -172,6 +172,35 @@ the triage link for the problematic image.
 
 [skia crbug]: https://bugs.chromium.org/p/skia
 
+## Inexact Matching
+
+By default, Gold uses exact matching with support for multiple baselines per
+test. This works well for most of the GPU tests, but there are a handful of
+tests such as `Pixel_CSS3DBlueBox` that are prone to noise which causes them to
+need additional triaging at times.
+
+For cases like this, using inexact matching can help, as it allows a comparison
+to pass if there are only minor differences between the produced image and a
+known-good image. Images that pass in this way will be automatically approved
+in Gold, so there is still a record of exactly what was produced.
+
+To enable this functionality, simply add a `matching_algorithm` field to the
+`PixelTestPage` definition for the test (see other uses of this in the file for
+concrete examples).
+
+In order to determine which values to use, you can use the script located at
+`//content/test/gpu/gold_inexact_matching/determine_gold_inexact_parameters.py`.
+
+More complete documentation can be found in the `--help` output of the script,
+but in general:
+* Use the `binary_search` optimization algorithm if you only want to vary
+    a single parameter, e.g. you only want to use a Sobel filter.
+* Use the `local_minima` optimization algorithm if you want to vary multiple
+    parameters, such as using fuzzy diffing + a Sobel filter together.
+* The default boundaries and weights generally work and give good results, but
+    you may need to tune them to better suit your particular test, e.g.
+    increasing the maximum number of differing pixels if your image is large.
+
 ## Working On Gold
 
 ### Modifying Gold And goldctl