# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import unittest
from auto_bisect import source_control as source_control_module
# Special import necessary because filename contains dash characters.
bisect_perf_module = __import__('bisect-perf-regression')
class BisectPerfRegressionTest(unittest.TestCase):
"""Test case for other functions and classes in bisect-perf-regression.py."""
def _AssertConfidence(self, score, bad_values, good_values):
"""Checks whether the given sets of values have a given confidence score.
The score represents our confidence that the two sets of values wouldn't
be as different as they are just by chance; that is, that some real change
occurred between the two sets of values.
Args:
score: Expected confidence score.
bad_values: First list of numbers.
good_values: Second list of numbers.
"""
# ConfidenceScore takes a list of lists but these lists are flattened
# inside the function.
confidence = bisect_perf_module.ConfidenceScore(
[[v] for v in bad_values],
[[v] for v in good_values])
self.assertEqual(score, confidence)
def testConfidenceScore_ZeroConfidence(self):
# The good and bad sets contain the same values, so the confidence that
# they're different should be zero.
self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4])
def testConfidenceScore_MediumConfidence(self):
self._AssertConfidence(80.0, [0, 1, 1, 1, 2, 2], [1, 1, 1, 3, 3, 4])
def testConfidenceScore_HighConfidence(self):
self._AssertConfidence(95.0, [0, 1, 1, 1, 2, 2], [1, 2, 2, 3, 3, 4])
def testConfidenceScore_VeryHighConfidence(self):
# Confidence is high if the two sets of values have no internal variance.
self._AssertConfidence(99.9, [1, 1, 1, 1], [1.2, 1.2, 1.2, 1.2])
self._AssertConfidence(99.9, [1, 1, 1, 1], [1.01, 1.01, 1.01, 1.01])
def testConfidenceScore_UnbalancedSampleSize(self):
# The second set of numbers only contains one number, so confidence is 0.
self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2], [1.4])
def testConfidenceScore_EmptySample(self):
# Confidence is zero if either or both samples are empty.
self._AssertConfidence(0.0, [], [])
self._AssertConfidence(0.0, [], [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3])
self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3], [])
def testConfidenceScore_FunctionalTestResults(self):
self._AssertConfidence(80.0, [1, 1, 0, 1, 1, 1, 0, 1], [0, 0, 1, 0, 1, 0])
self._AssertConfidence(99.9, [1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0])
def testParseDEPSStringManually(self):
"""Tests DEPS parsing."""
deps_file_contents = """
vars = {
'ffmpeg_hash':
'@ac4a9f31fe2610bd146857bbd55d7a260003a888',
'webkit_url':
'https://chromium.googlesource.com/chromium/blink.git',
'git_url':
'https://chromium.googlesource.com',
'webkit_rev':
'@e01ac0a267d1017288bc67fa3c366b10469d8a24',
'angle_revision':
'74697cf2064c0a2c0d7e1b1b28db439286766a05'
}"""
# Should only expect SVN/git revisions to come through, and URLs should be
# filtered out.
expected_vars_dict = {
'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
}
# Testing private function.
# pylint: disable=W0212
vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually(
deps_file_contents)
self.assertEqual(vars_dict, expected_vars_dict)
def _AssertParseResult(self, expected_values, result_string):
"""Asserts some values are parsed from a RESULT line."""
results_template = ('RESULT other_chart: other_trace= 123 count\n'
'RESULT my_chart: my_trace= %(value)s\n')
results = results_template % {'value': result_string}
metric = ['my_chart', 'my_trace']
# Testing private function.
# pylint: disable=W0212
values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results)
self.assertEqual(expected_values, values)
def testTryParseResultValuesFromOutput_WithSingleValue(self):
"""Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
self._AssertParseResult([66.88], '66.88 kb')
self._AssertParseResult([66.88], '66.88 ')
self._AssertParseResult([-66.88], '-66.88 kb')
self._AssertParseResult([66], '66 kb')
self._AssertParseResult([0.66], '.66 kb')
self._AssertParseResult([], '. kb')
self._AssertParseResult([], 'aaa kb')
def testTryParseResultValuesFromOutput_WithMultiValue(self):
"""Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""
self._AssertParseResult([66.88], '[66.88] kb')
self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')
self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')
self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')
self._AssertParseResult([-66, 99], '[-66,99] kb')
self._AssertParseResult([-66, 99], '[-66,99,] kb')
self._AssertParseResult([-66, 0.99], '[-66,.99] kb')
self._AssertParseResult([], '[] kb')
self._AssertParseResult([], '[-66,abc] kb')
def testTryParseResultValuesFromOutputWithMeanStd(self):
"""Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""
self._AssertParseResult([33.22], '{33.22, 3.6} kb')
self._AssertParseResult([33.22], '{33.22, 3.6} kb')
self._AssertParseResult([33.22], '{33.22,3.6}kb')
self._AssertParseResult([33.22], '{33.22,3.6} kb')
self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')
self._AssertParseResult([-33.22], '{-33.22,3.6}kb')
self._AssertParseResult([22], '{22,6}kb')
self._AssertParseResult([.22], '{.22,6}kb')
self._AssertParseResult([], '{.22,6, 44}kb')
self._AssertParseResult([], '{}kb')
self._AssertParseResult([], '{XYZ}kb')
def _AssertCompatibleCommand(
self, expected_command, original_command, revision, target_platform):
"""Tests the modification of the command that might be done.
This modification to the command is done in order to get a Telemetry
command that works; before some revisions, the browser name that Telemetry
expects is different in some cases, but we want it to work anyway.
Specifically, only for android:
After r276628, only android-chrome-shell works.
Prior to r274857, only android-chromium-testshell works.
In the range [274857, 276628], both work.
"""
bisect_options = bisect_perf_module.BisectOptions()
bisect_options.output_buildbot_annotations = None
source_control = source_control_module.DetermineAndCreateSourceControl(
bisect_options)
bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
source_control, bisect_options)
bisect_instance.opts.target_platform = target_platform
git_revision = bisect_instance.source_control.ResolveToRevision(
revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100)
depot = 'chromium'
command = bisect_instance.GetCompatibleCommand(
original_command, git_revision, depot)
self.assertEqual(expected_command, command)
def testGetCompatibleCommand_ChangeToTestShell(self):
# For revisions <= r274857, only android-chromium-testshell is used.
self._AssertCompatibleCommand(
'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
274857, 'android')
def testGetCompatibleCommand_ChangeToShell(self):
# For revisions >= r276728, only android-chrome-shell can be used.
self._AssertCompatibleCommand(
'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
276628, 'android')
def testGetCompatibleCommand_NoChange(self):
# For revisions < r276728, android-chromium-testshell can be used.
self._AssertCompatibleCommand(
'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
274858, 'android')
# For revisions > r274857, android-chrome-shell can be used.
self._AssertCompatibleCommand(
'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
274858, 'android')
def testGetCompatibleCommand_NonAndroidPlatform(self):
# In most cases, there's no need to change Telemetry command.
# For revisions >= r276728, only android-chrome-shell can be used.
self._AssertCompatibleCommand(
'tools/perf/run_benchmark -v --browser=release foo',
'tools/perf/run_benchmark -v --browser=release foo',
276628, 'chromium')
# This method doesn't reference self; it fails if an error is thrown.
# pylint: disable=R0201
def testDryRun(self):
"""Does a dry run of the bisect script.
This serves as a smoke test to catch errors in the basic execution of the
script.
"""
options_dict = {
'debug_ignore_build': True,
'debug_ignore_sync': True,
'debug_ignore_perf_test': True,
'command': 'fake_command',
'metric': 'fake/metric',
'good_revision': 280000,
'bad_revision': 280005,
}
bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict)
source_control = source_control_module.DetermineAndCreateSourceControl(
bisect_options)
bisect_instance = bisect_perf_module.BisectPerformanceMetrics(
source_control, bisect_options)
results = bisect_instance.Run(bisect_options.command,
bisect_options.bad_revision,
bisect_options.good_revision,
bisect_options.metric)
bisect_instance.FormatAndPrintResults(results)
if __name__ == '__main__':
unittest.main()