Reland "wpt: Integrate Results Sink with the wpt test harness"
We need to pass the artifacts dictionary within an Artifacts instance to the Result constructor. Bug: 1166359 Change-Id: I0aadc30909fb571816caff82c72f182cea7476df Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2968667 Reviewed-by: Weizhong Xia <weizhong@google.com> Commit-Queue: Rakib Hasan <rmhasan@google.com> Cr-Commit-Position: refs/heads/master@{#893969}
This commit is contained in:

committed by
Chromium LUCI CQ

parent
cbfea205e4
commit
892c515592
testing/scripts
@ -6,24 +6,35 @@ import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import six
|
||||
|
||||
import common
|
||||
|
||||
BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools')
|
||||
WEB_TESTS_DIR = os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests')
|
||||
EXTERNAL_WPT_TESTS_DIR = os.path.join(WEB_TESTS_DIR, 'external', 'wpt')
|
||||
CATAPULT_DIR = os.path.join(common.SRC_DIR, 'third_party', 'catapult')
|
||||
LAYOUT_TEST_RESULTS_SUBDIR = 'layout-test-results'
|
||||
TYP_DIR = os.path.join(CATAPULT_DIR, 'third_party', 'typ')
|
||||
WEB_TESTS_DIR = os.path.normpath(
|
||||
os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests'))
|
||||
EXTERNAL_WPT_TESTS_DIR = os.path.join(WEB_TESTS_DIR, 'external', 'wpt')
|
||||
|
||||
if BLINK_TOOLS_DIR not in sys.path:
|
||||
sys.path.append(BLINK_TOOLS_DIR)
|
||||
|
||||
if TYP_DIR not in sys.path:
|
||||
sys.path.append(TYP_DIR)
|
||||
|
||||
from blinkpy.common.host import Host
|
||||
from blinkpy.common.html_diff import html_diff
|
||||
from blinkpy.common.system.filesystem import FileSystem
|
||||
from blinkpy.common.unified_diff import unified_diff
|
||||
from blinkpy.web_tests.models import test_failures
|
||||
from typ.artifacts import Artifacts
|
||||
from typ.json_results import Result
|
||||
from typ.result_sink import ResultSinkReporter
|
||||
|
||||
|
||||
class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
|
||||
"""The base class for script adapters that use wptrunner to execute web
|
||||
@ -41,6 +52,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
|
||||
# Path to the output of the test run. Comes from the args passed to the
|
||||
# run, parsed after this constructor. Can be overwritten by tests.
|
||||
self.wpt_output = None
|
||||
self.sink = ResultSinkReporter()
|
||||
|
||||
def generate_test_output_args(self, output):
|
||||
return ['--log-chromium', output]
|
||||
@ -228,6 +240,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
|
||||
if artifact_subpath:
|
||||
root_node["artifacts"]["crash_log"] = [artifact_subpath]
|
||||
|
||||
self._add_result_to_sink(path_so_far, root_node)
|
||||
return
|
||||
|
||||
# We're not at a leaf node, continue traversing the trie.
|
||||
@ -239,6 +252,43 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
|
||||
self._process_test_leaves(results_dir, delim, root_node[key],
|
||||
new_path)
|
||||
|
||||
def _add_result_to_sink(self, test_name, result_node):
|
||||
"""Add's test results to results sink
|
||||
|
||||
Args:
|
||||
test_name: Name of the test to add to results sink.
|
||||
result_node: Dictionary containing the actual result, expected result
|
||||
and an artifacts dictionary.
|
||||
"""
|
||||
|
||||
artifacts = Artifacts(output_dir=self.wpt_output,
|
||||
host=self.sink.host,
|
||||
artifacts_base_dir=LAYOUT_TEST_RESULTS_SUBDIR)
|
||||
|
||||
assert len(result_node['actual'].split()) == 1, (
|
||||
('There should be only one result, however test %s has the '
|
||||
'following results "%s"') % (test_name, result_node['actual']))
|
||||
unexpected = result_node['actual'] not in result_node['expected']
|
||||
|
||||
for artifact_name, path in result_node.get('artifacts', {}).items():
|
||||
artifacts.AddArtifact(artifact_name, path)
|
||||
|
||||
result = Result(name=test_name,
|
||||
actual=result_node['actual'],
|
||||
started=time.time() - result_node.get('time', 0),
|
||||
took=result_node.get('time', 0),
|
||||
worker=0,
|
||||
expected=set(result_node['expected'].split()),
|
||||
unexpected=unexpected,
|
||||
artifacts=artifacts.artifacts)
|
||||
|
||||
index = test_name.find('?')
|
||||
test_path = test_name[:index] if index != -1 else test_name
|
||||
|
||||
self.sink.report_individual_test_result(
|
||||
test_name, result, LAYOUT_TEST_RESULTS_SUBDIR,
|
||||
None, os.path.join(WEB_TESTS_DIR, test_path))
|
||||
|
||||
def _maybe_write_expected_output(self, results_dir, test_name):
|
||||
"""Attempts to create an expected output artifact for the test.
|
||||
|
||||
|
@ -25,6 +25,25 @@ from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
|
||||
OUTPUT_JSON_FILENAME = "out.json"
|
||||
|
||||
|
||||
class MockResultSink(object):
|
||||
|
||||
def __init__(self):
|
||||
self.sink_requests = []
|
||||
self.host = MockHost()
|
||||
|
||||
def report_individual_test_result(self, test_name, result,
|
||||
artifacts_sub_dir,
|
||||
expectation, test_path):
|
||||
del artifacts_sub_dir
|
||||
assert not expectation, 'expectation parameter should always be None'
|
||||
self.sink_requests.append(
|
||||
{'test': test_name,
|
||||
'test_path': test_path,
|
||||
'result': {'actual': result.actual,
|
||||
'expected': result.expected,
|
||||
'unexpected': result.unexpected,
|
||||
'artifacts': result.artifacts}})
|
||||
|
||||
class BaseWptScriptAdapterTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.host = MockHost()
|
||||
@ -72,6 +91,7 @@ class BaseWptScriptAdapterTest(unittest.TestCase):
|
||||
"results-viewer-body")
|
||||
self.wpt_adapter = BaseWptScriptAdapter(self.host)
|
||||
self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME
|
||||
self.wpt_adapter.sink = MockResultSink()
|
||||
|
||||
def _create_json_output(self, json_dict):
|
||||
"""Writing some json output for processing."""
|
||||
@ -82,6 +102,103 @@ class BaseWptScriptAdapterTest(unittest.TestCase):
|
||||
"""Loads the json output after post-processing."""
|
||||
return json.loads(self.host.filesystem.read_text_file(filename))
|
||||
|
||||
def test_result_sink_for_test_expected_result(self):
|
||||
json_dict = {
|
||||
'tests': {
|
||||
'fail': {
|
||||
'test.html?variant1': {
|
||||
'expected': 'PASS FAIL',
|
||||
'actual': 'FAIL',
|
||||
'artifacts': {
|
||||
'wpt_actual_status': ['OK'],
|
||||
'wpt_actual_metadata': ['test.html actual text'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
'path_delimiter': os.path.sep,
|
||||
}
|
||||
test_abs_path = os.path.join(WEB_TESTS_DIR,
|
||||
'external/wpt/fail/test.html')
|
||||
self._create_json_output(json_dict)
|
||||
self.wpt_adapter.do_post_test_run_tasks()
|
||||
|
||||
baseline_artifacts = {'wpt_actual_status': [['OK']],
|
||||
'actual_text': [
|
||||
[('layout-test-results/external'
|
||||
'/wpt/fail/test_variant1-actual.txt')]]}
|
||||
self.assertEquals(self.wpt_adapter.sink.sink_requests,
|
||||
[{'test': 'external/wpt/fail/test.html?variant1',
|
||||
'test_path': test_abs_path,
|
||||
'result': {'actual': 'FAIL',
|
||||
'expected': set(['PASS', 'FAIL']),
|
||||
'unexpected': False,
|
||||
'artifacts': baseline_artifacts}}])
|
||||
|
||||
def test_result_sink_for_test_variant(self):
|
||||
json_dict = {
|
||||
'tests': {
|
||||
'fail': {
|
||||
'test.html?variant1': {
|
||||
'expected': 'PASS',
|
||||
'actual': 'FAIL',
|
||||
'artifacts': {
|
||||
'wpt_actual_status': ['OK'],
|
||||
'wpt_actual_metadata': ['test.html actual text'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
'path_delimiter': os.path.sep,
|
||||
}
|
||||
test_abs_path = os.path.join(WEB_TESTS_DIR,
|
||||
'external/wpt/fail/test.html')
|
||||
self._create_json_output(json_dict)
|
||||
self.wpt_adapter.do_post_test_run_tasks()
|
||||
baseline_artifacts = {'wpt_actual_status': [['OK']],
|
||||
'actual_text': [
|
||||
[('layout-test-results/external'
|
||||
'/wpt/fail/test_variant1-actual.txt')]]}
|
||||
self.assertEquals(self.wpt_adapter.sink.sink_requests,
|
||||
[{'test': 'external/wpt/fail/test.html?variant1',
|
||||
'test_path': test_abs_path,
|
||||
'result': {'actual': 'FAIL',
|
||||
'expected': set(['PASS']),
|
||||
'unexpected': True,
|
||||
'artifacts': baseline_artifacts}}])
|
||||
|
||||
def test_result_sink_artifacts(self):
|
||||
json_dict = {
|
||||
'tests': {
|
||||
'fail': {
|
||||
'test.html': {
|
||||
'expected': 'PASS',
|
||||
'actual': 'FAIL',
|
||||
'artifacts': {
|
||||
'wpt_actual_status': ['OK'],
|
||||
'wpt_actual_metadata': ['test.html actual text'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
'path_delimiter': os.path.sep,
|
||||
}
|
||||
self._create_json_output(json_dict)
|
||||
self.wpt_adapter.do_post_test_run_tasks()
|
||||
test_abs_path = os.path.join(WEB_TESTS_DIR,
|
||||
'external/wpt/fail/test.html')
|
||||
baseline_artifacts = {'wpt_actual_status': [['OK']],
|
||||
'actual_text': [
|
||||
[('layout-test-results/external'
|
||||
'/wpt/fail/test-actual.txt')]]}
|
||||
self.assertEquals(self.wpt_adapter.sink.sink_requests,
|
||||
[{'test': 'external/wpt/fail/test.html',
|
||||
'test_path': test_abs_path,
|
||||
'result': {'actual': 'FAIL',
|
||||
'expected': set(['PASS']),
|
||||
'unexpected': True,
|
||||
'artifacts': baseline_artifacts}}])
|
||||
|
||||
def test_write_jsons(self):
|
||||
# Ensure that various JSONs are written to the correct location.
|
||||
|
||||
|
Reference in New Issue
Block a user