0

Reland "wpt: Integrate Results Sink with the wpt test harness"

We need to pass the artifacts dictionary within an Artifacts
instance to the Result constructor.

Bug: 1166359
Change-Id: I0aadc30909fb571816caff82c72f182cea7476df
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2968667
Reviewed-by: Weizhong Xia <weizhong@google.com>
Commit-Queue: Rakib Hasan <rmhasan@google.com>
Cr-Commit-Position: refs/heads/master@{#893969}
This commit is contained in:
Rakib M. Hasan
2021-06-18 21:43:02 +00:00
committed by Chromium LUCI CQ
parent cbfea205e4
commit 892c515592
2 changed files with 169 additions and 2 deletions

@ -6,24 +6,35 @@ import base64
import json
import os
import sys
import time
import six
import common
BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools')
WEB_TESTS_DIR = os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests')
EXTERNAL_WPT_TESTS_DIR = os.path.join(WEB_TESTS_DIR, 'external', 'wpt')
CATAPULT_DIR = os.path.join(common.SRC_DIR, 'third_party', 'catapult')
LAYOUT_TEST_RESULTS_SUBDIR = 'layout-test-results'
TYP_DIR = os.path.join(CATAPULT_DIR, 'third_party', 'typ')
WEB_TESTS_DIR = os.path.normpath(
os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests'))
EXTERNAL_WPT_TESTS_DIR = os.path.join(WEB_TESTS_DIR, 'external', 'wpt')
if BLINK_TOOLS_DIR not in sys.path:
sys.path.append(BLINK_TOOLS_DIR)
if TYP_DIR not in sys.path:
sys.path.append(TYP_DIR)
from blinkpy.common.host import Host
from blinkpy.common.html_diff import html_diff
from blinkpy.common.system.filesystem import FileSystem
from blinkpy.common.unified_diff import unified_diff
from blinkpy.web_tests.models import test_failures
from typ.artifacts import Artifacts
from typ.json_results import Result
from typ.result_sink import ResultSinkReporter
class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
"""The base class for script adapters that use wptrunner to execute web
@ -41,6 +52,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
# Path to the output of the test run. Comes from the args passed to the
# run, parsed after this constructor. Can be overwritten by tests.
self.wpt_output = None
self.sink = ResultSinkReporter()
def generate_test_output_args(self, output):
return ['--log-chromium', output]
@ -228,6 +240,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
if artifact_subpath:
root_node["artifacts"]["crash_log"] = [artifact_subpath]
self._add_result_to_sink(path_so_far, root_node)
return
# We're not at a leaf node, continue traversing the trie.
@ -239,6 +252,43 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
self._process_test_leaves(results_dir, delim, root_node[key],
new_path)
def _add_result_to_sink(self, test_name, result_node):
"""Add's test results to results sink
Args:
test_name: Name of the test to add to results sink.
result_node: Dictionary containing the actual result, expected result
and an artifacts dictionary.
"""
artifacts = Artifacts(output_dir=self.wpt_output,
host=self.sink.host,
artifacts_base_dir=LAYOUT_TEST_RESULTS_SUBDIR)
assert len(result_node['actual'].split()) == 1, (
('There should be only one result, however test %s has the '
'following results "%s"') % (test_name, result_node['actual']))
unexpected = result_node['actual'] not in result_node['expected']
for artifact_name, path in result_node.get('artifacts', {}).items():
artifacts.AddArtifact(artifact_name, path)
result = Result(name=test_name,
actual=result_node['actual'],
started=time.time() - result_node.get('time', 0),
took=result_node.get('time', 0),
worker=0,
expected=set(result_node['expected'].split()),
unexpected=unexpected,
artifacts=artifacts.artifacts)
index = test_name.find('?')
test_path = test_name[:index] if index != -1 else test_name
self.sink.report_individual_test_result(
test_name, result, LAYOUT_TEST_RESULTS_SUBDIR,
None, os.path.join(WEB_TESTS_DIR, test_path))
def _maybe_write_expected_output(self, results_dir, test_name):
"""Attempts to create an expected output artifact for the test.

@ -25,6 +25,25 @@ from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
OUTPUT_JSON_FILENAME = "out.json"
class MockResultSink(object):
def __init__(self):
self.sink_requests = []
self.host = MockHost()
def report_individual_test_result(self, test_name, result,
artifacts_sub_dir,
expectation, test_path):
del artifacts_sub_dir
assert not expectation, 'expectation parameter should always be None'
self.sink_requests.append(
{'test': test_name,
'test_path': test_path,
'result': {'actual': result.actual,
'expected': result.expected,
'unexpected': result.unexpected,
'artifacts': result.artifacts}})
class BaseWptScriptAdapterTest(unittest.TestCase):
def setUp(self):
self.host = MockHost()
@ -72,6 +91,7 @@ class BaseWptScriptAdapterTest(unittest.TestCase):
"results-viewer-body")
self.wpt_adapter = BaseWptScriptAdapter(self.host)
self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME
self.wpt_adapter.sink = MockResultSink()
def _create_json_output(self, json_dict):
"""Writing some json output for processing."""
@ -82,6 +102,103 @@ class BaseWptScriptAdapterTest(unittest.TestCase):
"""Loads the json output after post-processing."""
return json.loads(self.host.filesystem.read_text_file(filename))
def test_result_sink_for_test_expected_result(self):
json_dict = {
'tests': {
'fail': {
'test.html?variant1': {
'expected': 'PASS FAIL',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
},
'path_delimiter': os.path.sep,
}
test_abs_path = os.path.join(WEB_TESTS_DIR,
'external/wpt/fail/test.html')
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
baseline_artifacts = {'wpt_actual_status': [['OK']],
'actual_text': [
[('layout-test-results/external'
'/wpt/fail/test_variant1-actual.txt')]]}
self.assertEquals(self.wpt_adapter.sink.sink_requests,
[{'test': 'external/wpt/fail/test.html?variant1',
'test_path': test_abs_path,
'result': {'actual': 'FAIL',
'expected': set(['PASS', 'FAIL']),
'unexpected': False,
'artifacts': baseline_artifacts}}])
def test_result_sink_for_test_variant(self):
json_dict = {
'tests': {
'fail': {
'test.html?variant1': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
},
'path_delimiter': os.path.sep,
}
test_abs_path = os.path.join(WEB_TESTS_DIR,
'external/wpt/fail/test.html')
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
baseline_artifacts = {'wpt_actual_status': [['OK']],
'actual_text': [
[('layout-test-results/external'
'/wpt/fail/test_variant1-actual.txt')]]}
self.assertEquals(self.wpt_adapter.sink.sink_requests,
[{'test': 'external/wpt/fail/test.html?variant1',
'test_path': test_abs_path,
'result': {'actual': 'FAIL',
'expected': set(['PASS']),
'unexpected': True,
'artifacts': baseline_artifacts}}])
def test_result_sink_artifacts(self):
json_dict = {
'tests': {
'fail': {
'test.html': {
'expected': 'PASS',
'actual': 'FAIL',
'artifacts': {
'wpt_actual_status': ['OK'],
'wpt_actual_metadata': ['test.html actual text'],
},
},
},
},
'path_delimiter': os.path.sep,
}
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
test_abs_path = os.path.join(WEB_TESTS_DIR,
'external/wpt/fail/test.html')
baseline_artifacts = {'wpt_actual_status': [['OK']],
'actual_text': [
[('layout-test-results/external'
'/wpt/fail/test-actual.txt')]]}
self.assertEquals(self.wpt_adapter.sink.sink_requests,
[{'test': 'external/wpt/fail/test.html',
'test_path': test_abs_path,
'result': {'actual': 'FAIL',
'expected': set(['PASS']),
'unexpected': True,
'artifacts': baseline_artifacts}}])
def test_write_jsons(self):
# Ensure that various JSONs are written to the correct location.