Add a Tast mode to cros VM test runner.
Tast is a new suite of tests. It's invoked via "local_test_runner" in the VM. You pass it a set of test names and it'll exercise some test cases. We shouldn't use much more than the browser flavored ones. Bug: 876587 Change-Id: I1d6ae9b6b5a929daadb6c12751cb0cd632a24d95 Reviewed-on: https://chromium-review.googlesource.com/1237673 Commit-Queue: Ben Pastene <bpastene@chromium.org> Reviewed-by: John Budorick <jbudorick@chromium.org> Cr-Commit-Position: refs/heads/master@{#594075}
This commit is contained in:
Ben Pastene
@ -29,6 +29,8 @@ sys.path.insert(
|
||||
0, os.path.join(CHROMIUM_SRC_PATH, 'tools', 'swarming_client', 'utils'))
|
||||
import subprocess42 # pylint: disable=import-error
|
||||
|
||||
DEFAULT_CROS_CACHE = os.path.abspath(os.path.join(
|
||||
CHROMIUM_SRC_PATH, 'build', 'cros_cache'))
|
||||
CHROMITE_PATH = os.path.abspath(os.path.join(
|
||||
CHROMIUM_SRC_PATH, 'third_party', 'chromite'))
|
||||
CROS_RUN_VM_TEST_PATH = os.path.abspath(os.path.join(
|
||||
@ -47,7 +49,6 @@ class RemoteTest(object):
|
||||
def __init__(self, args, unknown_args):
|
||||
self._additional_args = unknown_args
|
||||
self._path_to_outdir = args.path_to_outdir
|
||||
self._test_exe = args.test_exe
|
||||
self._test_launcher_summary_output = args.test_launcher_summary_output
|
||||
self._vm_logs_dir = args.vm_logs_dir
|
||||
|
||||
@ -67,6 +68,10 @@ class RemoteTest(object):
|
||||
'--results-dest-dir', args.vm_logs_dir,
|
||||
]
|
||||
|
||||
@property
|
||||
def suite_name(self):
|
||||
raise NotImplementedError('Child classes need to define suite name.')
|
||||
|
||||
@property
|
||||
def vm_test_cmd(self):
|
||||
return self._vm_test_cmd
|
||||
@ -119,7 +124,45 @@ class RemoteTest(object):
|
||||
return test_proc.returncode
|
||||
|
||||
def post_run(self, return_code):
|
||||
raise NotImplementedError()
|
||||
# Create a simple json results file for a test run. The results will contain
|
||||
# only one test (suite_name), and will either be a PASS or FAIL depending on
|
||||
# return_code.
|
||||
if self._test_launcher_summary_output:
|
||||
result = (base_test_result.ResultType.FAIL if return_code else
|
||||
base_test_result.ResultType.PASS)
|
||||
suite_result = base_test_result.BaseTestResult(self.suite_name, result)
|
||||
run_results = base_test_result.TestRunResults()
|
||||
run_results.AddResult(suite_result)
|
||||
with open(self._test_launcher_summary_output, 'w') as f:
|
||||
json.dump(json_results.GenerateResultsDict([run_results]), f)
|
||||
|
||||
|
||||
class TastTest(RemoteTest):
|
||||
|
||||
def __init__(self, args, unknown_args):
|
||||
super(TastTest, self).__init__(args, unknown_args)
|
||||
|
||||
self._suite_name = args.suite_name
|
||||
self._tests = args.tests
|
||||
|
||||
@property
|
||||
def suite_name(self):
|
||||
return self._suite_name
|
||||
|
||||
def build_test_command(self):
|
||||
if self._additional_args:
|
||||
raise TestFormatError(
|
||||
'Tast tests should not have additional args: %s' % (
|
||||
self._additional_args))
|
||||
|
||||
self._vm_test_cmd += [
|
||||
'--deploy',
|
||||
'--build-dir', os.path.relpath(self._path_to_outdir, CHROMIUM_SRC_PATH),
|
||||
'--cmd',
|
||||
'--',
|
||||
'local_test_runner',
|
||||
]
|
||||
self._vm_test_cmd.extend(self._tests)
|
||||
|
||||
|
||||
class GTestTest(RemoteTest):
|
||||
@ -133,6 +176,7 @@ class GTestTest(RemoteTest):
|
||||
def __init__(self, args, unknown_args):
|
||||
super(GTestTest, self).__init__(args, unknown_args)
|
||||
|
||||
self._test_exe = args.test_exe
|
||||
self._runtime_deps_path = args.runtime_deps_path
|
||||
self._vpython_dir = args.vpython_dir
|
||||
|
||||
@ -141,6 +185,10 @@ class GTestTest(RemoteTest):
|
||||
|
||||
self._on_vm_script = None
|
||||
|
||||
@property
|
||||
def suite_name(self):
|
||||
return self._test_exe
|
||||
|
||||
def build_test_command(self):
|
||||
# To keep things easy for us, ensure both types of output locations are
|
||||
# the same.
|
||||
@ -284,6 +332,10 @@ class BrowserSanityTest(RemoteTest):
|
||||
self._retries = 2
|
||||
self._timeout = 300
|
||||
|
||||
@property
|
||||
def suite_name(self):
|
||||
return SANITY_TEST_TARGET
|
||||
|
||||
def build_test_command(self):
|
||||
if '--gtest_filter=%s' % SANITY_TEST_TARGET in self._additional_args:
|
||||
logging.info(
|
||||
@ -317,20 +369,6 @@ class BrowserSanityTest(RemoteTest):
|
||||
self._test_env['PATH'] = (
|
||||
self._test_env['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin'))
|
||||
|
||||
def post_run(self, return_code):
|
||||
# Create a simple json results file for the sanity test if needed. The
|
||||
# results will contain only one test (SANITY_TEST_TARGET), and will
|
||||
# either be a PASS or FAIL depending on the return code of cros_run_vm_test.
|
||||
if self._test_launcher_summary_output:
|
||||
result = (base_test_result.ResultType.FAIL if return_code else
|
||||
base_test_result.ResultType.PASS)
|
||||
sanity_test_result = base_test_result.BaseTestResult(
|
||||
SANITY_TEST_TARGET, result)
|
||||
run_results = base_test_result.TestRunResults()
|
||||
run_results.AddResult(sanity_test_result)
|
||||
with open(self._test_launcher_summary_output, 'w') as f:
|
||||
json.dump(json_results.GenerateResultsDict([run_results]), f)
|
||||
|
||||
|
||||
def vm_test(args, unknown_args):
|
||||
# cros_run_vm_test has trouble with relative paths that go up directories,
|
||||
@ -340,7 +378,9 @@ def vm_test(args, unknown_args):
|
||||
# pylint: disable=redefined-variable-type
|
||||
# TODO: Remove the above when depot_tool's pylint is updated to include the
|
||||
# fix to https://github.com/PyCQA/pylint/issues/710.
|
||||
if args.test_exe == SANITY_TEST_TARGET:
|
||||
if args.test_type == 'tast':
|
||||
test = TastTest(args, unknown_args)
|
||||
elif args.test_exe == SANITY_TEST_TARGET:
|
||||
test = BrowserSanityTest(args, unknown_args)
|
||||
else:
|
||||
test = GTestTest(args, unknown_args)
|
||||
@ -402,6 +442,30 @@ def host_cmd(args, unknown_args):
|
||||
cros_run_vm_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=test_env)
|
||||
|
||||
|
||||
def add_common_args(parser):
|
||||
parser.add_argument(
|
||||
'--cros-cache', type=str, default=DEFAULT_CROS_CACHE,
|
||||
help='Path to cros cache.')
|
||||
parser.add_argument(
|
||||
'--path-to-outdir', type=str, required=True,
|
||||
help='Path to output directory, all of whose contents will be '
|
||||
'deployed to the device.')
|
||||
parser.add_argument(
|
||||
'--runtime-deps-path', type=str,
|
||||
help='Runtime data dependency file from GN.')
|
||||
parser.add_argument(
|
||||
'--vpython-dir', type=str,
|
||||
help='Location on host of a directory containing a vpython binary to '
|
||||
'deploy to the VM before the test starts. The location of this '
|
||||
'dir will be added onto PATH in the VM. WARNING: The arch of the '
|
||||
'VM might not match the arch of the host, so avoid using '
|
||||
'"${platform}" when downloading vpython via CIPD.')
|
||||
parser.add_argument(
|
||||
'--vm-logs-dir', type=str,
|
||||
help='Will copy everything under /var/log/ from the VM after the test '
|
||||
'into the specified dir.')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--verbose', '-v', action='store_true')
|
||||
@ -416,7 +480,8 @@ def main():
|
||||
'"--". Hostname and port for the VM will be 127.0.0.1:9222.')
|
||||
host_cmd_parser.set_defaults(func=host_cmd)
|
||||
host_cmd_parser.add_argument(
|
||||
'--cros-cache', type=str, required=True, help='Path to cros cache.')
|
||||
'--cros-cache', type=str, default=DEFAULT_CROS_CACHE,
|
||||
help='Path to cros cache.')
|
||||
host_cmd_parser.add_argument(
|
||||
'--path-to-outdir', type=os.path.realpath,
|
||||
help='Path to output directory, all of whose contents will be deployed '
|
||||
@ -426,14 +491,13 @@ def main():
|
||||
help='Will deploy a locally built Chrome binary to the VM before running '
|
||||
'the host-cmd.')
|
||||
host_cmd_parser.add_argument('cmd', nargs=argparse.REMAINDER)
|
||||
# VM-side test args.
|
||||
vm_test_parser = subparsers.add_parser(
|
||||
# GTest args.
|
||||
# TODO(bpastene): Rename 'vm-test' arg to 'gtest'.
|
||||
gtest_parser = subparsers.add_parser(
|
||||
'vm-test',
|
||||
help='Runs a vm-side gtest.')
|
||||
vm_test_parser.set_defaults(func=vm_test)
|
||||
vm_test_parser.add_argument(
|
||||
'--cros-cache', type=str, required=True, help='Path to cros cache.')
|
||||
vm_test_parser.add_argument(
|
||||
gtest_parser.set_defaults(func=vm_test)
|
||||
gtest_parser.add_argument(
|
||||
'--test-exe', type=str, required=True,
|
||||
help='Path to test executable to run inside VM. If the value is '
|
||||
'%s, the sanity test that ships with the VM '
|
||||
@ -444,39 +508,39 @@ def main():
|
||||
|
||||
# GTest args. Some are passed down to the test binary in the VM. Others are
|
||||
# parsed here since they might need tweaking or special handling.
|
||||
vm_test_parser.add_argument(
|
||||
gtest_parser.add_argument(
|
||||
'--test-launcher-summary-output', type=str,
|
||||
help='When set, will pass the same option down to the test and retrieve '
|
||||
'its result file at the specified location.')
|
||||
# Shard args are parsed here since we might also specify them via env vars.
|
||||
vm_test_parser.add_argument(
|
||||
gtest_parser.add_argument(
|
||||
'--test-launcher-shard-index',
|
||||
type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0),
|
||||
help='Index of the external shard to run.')
|
||||
vm_test_parser.add_argument(
|
||||
gtest_parser.add_argument(
|
||||
'--test-launcher-total-shards',
|
||||
type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
|
||||
help='Total number of external shards.')
|
||||
|
||||
# Misc args.
|
||||
vm_test_parser.add_argument(
|
||||
'--path-to-outdir', type=str, required=True,
|
||||
help='Path to output directory, all of whose contents will be deployed '
|
||||
'to the device.')
|
||||
vm_test_parser.add_argument(
|
||||
'--runtime-deps-path', type=str,
|
||||
help='Runtime data dependency file from GN.')
|
||||
vm_test_parser.add_argument(
|
||||
'--vpython-dir', type=str,
|
||||
help='Location on host of a directory containing a vpython binary to '
|
||||
'deploy to the VM before the test starts. The location of this dir '
|
||||
'will be added onto PATH in the VM. WARNING: The arch of the VM '
|
||||
'might not match the arch of the host, so avoid using "${platform}" '
|
||||
'when downloading vpython via CIPD.')
|
||||
vm_test_parser.add_argument(
|
||||
'--vm-logs-dir', type=str,
|
||||
help='Will copy everything under /var/log/ from the VM after the test '
|
||||
'into the specified dir.')
|
||||
# Tast test args.
|
||||
tast_test_parser = subparsers.add_parser(
|
||||
'tast',
|
||||
help='Runs a vm-side set of Tast tests.')
|
||||
tast_test_parser.set_defaults(func=vm_test)
|
||||
tast_test_parser.add_argument(
|
||||
'--suite-name', type=str, required=True,
|
||||
help='Name to apply to the set of Tast tests to run. This has no effect '
|
||||
'on what is executed, but is used mainly for test results reporting '
|
||||
'and tracking (eg: flakiness dashboard).')
|
||||
tast_test_parser.add_argument(
|
||||
'--test-launcher-summary-output', type=str,
|
||||
help='Generates a simple GTest-style JSON result file for the test run.')
|
||||
tast_test_parser.add_argument(
|
||||
'--test', '-t', action='append', dest='tests', required=True,
|
||||
help='A Tast test to run in the VM (eg: "ui.ChromeLogin").')
|
||||
|
||||
add_common_args(gtest_parser)
|
||||
add_common_args(tast_test_parser)
|
||||
args, unknown_args = parser.parse_known_args()
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARN)
|
||||
|
||||
Reference in New Issue
Block a user