Allow modification of multiple cmdline params with ParameterizedCommandLine.

This is needed for Page Controller or other UI-Automator based tests that can't
use the CommandLineFlags mechanism.

Bug: 1009529
Change-Id: If0364f37c2fe3f49e2111c2ed505fd4ed9392e6c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1860144
Commit-Queue: Andrew Luo <aluo@chromium.org>
Reviewed-by: Sky Malice <skym@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#710197}

base/BUILD.gn

@@ -3450,6 +3450,8 @@ if (is_android) {
       "test/android/javatests/src/org/chromium/base/test/params/ParameterizedRunnerDelegateFactory.java",
       "test/android/javatests/src/org/chromium/base/test/params/ParameterProvider.java",
       "test/android/javatests/src/org/chromium/base/test/params/ParameterSet.java",
+      "test/android/javatests/src/org/chromium/base/test/params/ParameterizedCommandLineFlags.java",
+      "test/android/javatests/src/org/chromium/base/test/params/SkipCommandLineParameterization.java",
       "test/android/javatests/src/org/chromium/base/test/task/SchedulerTestHelpers.java",
       "test/android/javatests/src/org/chromium/base/test/task/ThreadPoolTestHelpers.java",
       "test/android/javatests/src/org/chromium/base/test/util/AdvancedMockContext.java",
@@ -3483,8 +3485,6 @@ if (is_android) {
       "test/android/javatests/src/org/chromium/base/test/util/TimeoutTimer.java",
       "test/android/javatests/src/org/chromium/base/test/util/UserActionTester.java",
       "test/android/javatests/src/org/chromium/base/test/util/UrlUtils.java",
-      "test/android/javatests/src/org/chromium/base/test/util/parameter/CommandLineParameter.java",
-      "test/android/javatests/src/org/chromium/base/test/util/parameter/SkipCommandLineParameterization.java",
     ]
     annotation_processor_deps = [ "//base/android/jni_generator:jni_processor" ]
   }

base/test/android/javatests/src/org/chromium/base/test/TestListInstrumentationRunListener.java

@@ -7,6 +7,7 @@ package org.chromium.base.test;
 import android.support.test.internal.runner.listener.InstrumentationRunListener;
 
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.runner.Description;
 import org.junit.runner.notification.Failure;
@@ -22,7 +23,6 @@ import java.lang.annotation.Annotation;
 import java.lang.reflect.Array;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
@@ -66,19 +66,22 @@ public class TestListInstrumentationRunListener extends InstrumentationRunListen
     }
 
     /**
-     * Store the test method description to a Map at the beginning of a test run.
+     * Store the test method description to a Map at the beginning of a test
+     * run.
      */
     @Override
     public void testStarted(Description desc) throws Exception {
-        // BaseJUnit4ClassRunner only fires testFinished(), so a call to testStarted means a
+        // BaseJUnit4ClassRunner only fires testFinished(), so a call to
-        // different runner is active, and the test is actually being executed rather than just
+        // testStarted means a different runner is active, and the test is
-        // listed.
+        // actually being executed rather than just listed.
-        throw new InitializationError("All tests must use @RunWith(BaseJUnit4ClassRunner.class) or "
+        throw new InitializationError("All tests must use"
-                + "a subclass thereof. Found that this test does not: " + desc.getTestClass());
+                + " @RunWith(BaseJUnit4ClassRunner.class) or a subclass thereof."
+                + " Found that this test does not: " + desc.getTestClass());
     }
 
     /**
-     * Create a JSONArray with all the test class JSONObjects and save it to listed output path.
+     * Create a JSONArray with all the test class JSONObjects and save it to
+     * listed output path.
      */
     public void saveTestsToJson(String outputPath) throws IOException {
         if (mFirstFailure != null) {
@@ -105,7 +108,8 @@ public class TestListInstrumentationRunListener extends InstrumentationRunListen
     }
 
     /**
-     * Create a JSONObject that represent a collection of anntations.
+     * Make a JSONObject dictionary out of annotations, keyed by the
+     * Annotation types' simple java names.
      *
      * For example, for the following group of annotations for ExampleClass
      * <code>
@@ -125,80 +129,58 @@ public class TestListInstrumentationRunListener extends InstrumentationRunListen
      * }
      * </code>
      *
-     * The method accomplish this by though through each annotation and reflectively call the
+     * The method accomplish this by though through each annotation and
-     * annotation's method to get the element value, with exceptions to methods like "equals()"
+     * reflectively call the annotation's method to get the element value,
-     * or "hashCode".
+     * with exceptions to methods like "equals()" or "hashCode".
      */
     static JSONObject getAnnotationJSON(Collection<Annotation> annotations)
-            throws Exception {
+            throws IllegalAccessException, InvocationTargetException, JSONException {
-        JSONObject annotationsJsons = new JSONObject();
+        JSONObject result = new JSONObject();
         for (Annotation a : annotations) {
-            JSONObject elementJsonObject = new JSONObject();
+            JSONObject aJSON = (JSONObject) asJSON(a);
-            for (Method method : a.annotationType().getMethods()) {
+            String aType = aJSON.keys().next();
+            result.put(aType, aJSON.get(aType));
+        }
+        return result;
+    }
+
+    /**
+     * Recursively serialize an Annotation or an Annotation field value to
+     * a JSON compatible type.
+     */
+    private static Object asJSON(Object obj)
+            throws IllegalAccessException, InvocationTargetException, JSONException {
+        // Use instanceof to determine if it is an Annotation.
+        // obj.getClass().isAnnotation() doesn't work as expected because
+        // obj.getClass() returns a proxy class.
+        if (obj instanceof Annotation) {
+            Class<? extends Annotation> annotationType = ((Annotation) obj).annotationType();
+            JSONObject json = new JSONObject();
+            for (Method method : annotationType.getMethods()) {
                 if (SKIP_METHODS.contains(method.getName())) {
                     continue;
                 }
-                try {
+                json.put(method.getName(), asJSON(method.invoke(obj)));
-                    Object value = method.invoke(a);
-                    if (value == null) {
-                        elementJsonObject.put(method.getName(), null);
-                    } else if (value.getClass().isArray()) {
-                        Class<?> componentClass = value.getClass().getComponentType();
-                        // Arrays of primitives can't be cast to Object arrays, so we have to
-                        // special case them and manually make a copy.
-                        // This could be done more cleanly with something like
-                        // Arrays.stream(value).boxed().toArray(Integer[]::new), but that requires
-                        // a minimum SDK level of 24 to use.
-                        Object[] arrayValue = componentClass.isPrimitive()
-                                ? copyPrimitiveArrayToObjectArray(value)
-                                : ((Object[]) value);
-                        elementJsonObject.put(
-                                method.getName(), new JSONArray(Arrays.asList(arrayValue)));
-                    } else {
-                        elementJsonObject.put(method.getName(), value.toString());
-                    }
-                } catch (IllegalArgumentException e) {
-                }
             }
-            annotationsJsons.put(a.annotationType().getSimpleName(), elementJsonObject);
+            JSONObject outerJson = new JSONObject();
-        }
+            // If proguard is enabled and InnerClasses attribute is not kept,
-        return annotationsJsons;
+            // then getCanonicalName() will return Outer$Inner instead of
-    }
+            // Outer.Inner.  So just use getName().
-
+            outerJson.put(annotationType.getName().replaceFirst(
-    private static Object[] copyPrimitiveArrayToObjectArray(Object primitiveArray)
+                                  annotationType.getPackage().getName() + ".", ""),
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException,
+                    json);
-                   ClassCastException {
+            return outerJson;
-        Class<?> primitiveClass = primitiveArray.getClass();
-        Class<?> componentClass = primitiveClass.getComponentType();
-        Class<?> wrapperClass = null;
-        if (componentClass == Boolean.TYPE) {
-            wrapperClass = Boolean.class;
-        } else if (componentClass == Byte.TYPE) {
-            wrapperClass = Byte.class;
-        } else if (componentClass == Character.TYPE) {
-            wrapperClass = Character.class;
-        } else if (componentClass == Double.TYPE) {
-            wrapperClass = Double.class;
-        } else if (componentClass == Float.TYPE) {
-            wrapperClass = Float.class;
-        } else if (componentClass == Integer.TYPE) {
-            wrapperClass = Integer.class;
-        } else if (componentClass == Long.TYPE) {
-            wrapperClass = Long.class;
-        } else if (componentClass == Short.TYPE) {
-            wrapperClass = Short.class;
         } else {
-            // This should only be void since there are 8 primitives + void, but we can't support
+            Class<?> clazz = obj.getClass();
-            // void.
+            if (clazz.isArray()) {
-            throw new ClassCastException(
+                JSONArray jarr = new JSONArray();
-                    "Cannot cast a primitive void array to Object void array.");
+                for (int i = 0; i < Array.getLength(obj); i++) {
+                    jarr.put(asJSON(Array.get(obj, i)));
+                }
+                return jarr;
+            } else {
+                return obj;
+            }
         }
-        Method converterMethod = wrapperClass.getMethod("valueOf", componentClass);
-        ArrayList<Object> arrayValue = new ArrayList<Object>();
-        for (int i = 0; i < Array.getLength(primitiveClass.cast(primitiveArray)); i++) {
-            arrayValue.add(
-                    converterMethod.invoke(Array.get(primitiveClass.cast(primitiveArray), i)));
-        }
-        return arrayValue.toArray();
     }
 }

base/test/android/javatests/src/org/chromium/base/test/params/ParameterizedCommandLineFlags.java

@@ -0,0 +1,59 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.base.test.params;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to parameterize flags via the command-line flags file.
+ *
+ * For example, the following annotation would run the test 3 times:
+ *
+ * 1st: No flags.
+ * 2nd: --enable-features=one --one-params=p1
+ * 3rd: --enable-features=two --two-params=p2
+ *
+ * <code>
+ * @ParameterizedCommandLineFlags({
+ *     @Switches(),
+ *     @Switches({"enable-features=one", "one-params=p1"}),
+ *     @Switches({"enable-features=two", "two-params=p2"})
+ * })
+ * </code>
+ */
+
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface ParameterizedCommandLineFlags {
+    /**
+     * Annotation to set commnad line flags in the command-line flags file
+     * for JUnit4 instrumentation tests.
+     *
+     * E.g. if you add the following annotation to your test class:
+     *
+     * <code>
+     * @ParameterizedCommandLineFlags.Switches({"FLAG_A", "FLAG_B"})
+     * public class MyTestClass
+     * </code>
+     *
+     * The test harness would run the test once with with --FLAG_A --FLAG_B.
+     *
+     * If you want to have the method run multiple times with different sets of
+     * parameters, see {@link ParameterizedCommandLineFlags}.
+     */
+    @Inherited
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target({ElementType.METHOD, ElementType.TYPE})
+    public @interface Switches {
+        String[] value() default {};
+    }
+
+    Switches[] value() default {};
+}

base/test/android/javatests/src/org/chromium/base/test/params/SkipCommandLineParameterization.java

@@ -1,7 +1,7 @@
 // Copyright 2017 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-package org.chromium.base.test.util.parameter;
+package org.chromium.base.test.params;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Inherited;

base/test/android/javatests/src/org/chromium/base/test/util/parameter/CommandLineParameter.java

@@ -1,32 +0,0 @@
-// Copyright 2015 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package org.chromium.base.test.util.parameter;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Inherited;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * The annotation for parametering CommandLineFlags in JUnit3 instrumentation tests.
- *
- * E.g. if you add the following annotation to your test class:
- *
- * <code>
- * @CommandLineParameter({"", FLAG_A, FLAG_B})
- * public class MyTestClass
- * </code>
- *
- * The test harness would run the test 3 times with each of the flag added to commandline
- * file.
- */
-
-@Inherited
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.METHOD, ElementType.TYPE})
-public @interface CommandLineParameter {
-    String[] value() default {};
-}

base/test/android/junit/src/org/chromium/base/test/TestListInstrumentationRunListenerTest.java

@@ -4,9 +4,6 @@
 
 package org.chromium.base.test;
 
-import static org.chromium.base.test.TestListInstrumentationRunListener.getAnnotationJSON;
-import static org.chromium.base.test.TestListInstrumentationRunListener.getTestMethodJSON;
-
 import org.json.JSONObject;
 import org.junit.Assert;
 import org.junit.Test;
@@ -14,6 +11,8 @@ import org.junit.runner.Description;
 import org.junit.runner.RunWith;
 import org.robolectric.annotation.Config;
 
+import org.chromium.base.test.params.ParameterizedCommandLineFlags;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags.Switches;
 import org.chromium.base.test.util.CommandLineFlags;
 
 import java.util.Arrays;
@@ -36,20 +35,40 @@ public class TestListInstrumentationRunListenerTest {
     private static class ChildClass extends ParentClass {
     }
 
+    private static class Groups {
+        // clang-format off
+        @ParameterizedCommandLineFlags({
+            @Switches({"c1", "c2"}),
+            @Switches({"c3", "c4"}),
+        })
+        public void testA() {}
+        // clang-format on
+        @ParameterizedCommandLineFlags
+        public void testB() {}
+    }
+
+    private String makeJSON(String... lines) {
+        StringBuilder builder = new StringBuilder();
+        for (String line : lines) {
+            builder.append(line);
+        }
+        return builder.toString().replaceAll("\\s", "").replaceAll("'", "\"");
+    }
+
     @Test
     public void testGetTestMethodJSON_testA() throws Throwable {
         Description desc = Description.createTestDescription(
                 ParentClass.class, "testA",
                 ParentClass.class.getMethod("testA").getAnnotations());
-        JSONObject json = getTestMethodJSON(desc);
+        JSONObject json = TestListInstrumentationRunListener.getTestMethodJSON(desc);
-        String expectedJsonString =
+        // clang-format off
-                "{"
+        String expectedJsonString = makeJSON(
-                + "'method': 'testA',"
+            "{",
-                + "'annotations': {}"
+            " 'method': 'testA',",
-                + "}";
+            " 'annotations': {}",
-        expectedJsonString = expectedJsonString
+            "}"
-            .replaceAll("\\s", "")
+        );
-            .replaceAll("'", "\"");
+        // clang-format on
         Assert.assertEquals(expectedJsonString, json.toString());
     }
 
@@ -58,19 +77,19 @@ public class TestListInstrumentationRunListenerTest {
         Description desc = Description.createTestDescription(
                 ParentClass.class, "testB",
                 ParentClass.class.getMethod("testB").getAnnotations());
-        JSONObject json = getTestMethodJSON(desc);
+        JSONObject json = TestListInstrumentationRunListener.getTestMethodJSON(desc);
-        String expectedJsonString =
+        // clang-format off
-                "{"
+        String expectedJsonString = makeJSON(
-                + "'method': 'testB',"
+            "{",
-                + "'annotations': {"
+            " 'method': 'testB',",
-                + "  'Add': {"
+            " 'annotations': {",
-                + "    'value': ['world']"
+            "  'CommandLineFlags$Add': {",
-                + "    }"
+            "   'value': ['world']",
-                + "  }"
+            "  }",
-                + "}";
+            " }",
-        expectedJsonString = expectedJsonString
+            "}"
-            .replaceAll("\\s", "")
+        );
-            .replaceAll("'", "\"");
+        // clang-format on
         Assert.assertEquals(expectedJsonString, json.toString());
     }
 
@@ -80,39 +99,105 @@ public class TestListInstrumentationRunListenerTest {
         Description desc = Description.createTestDescription(
                 ChildClass.class, "testB",
                 ChildClass.class.getMethod("testB").getAnnotations());
-        JSONObject json = getTestMethodJSON(desc);
+        JSONObject json = TestListInstrumentationRunListener.getTestMethodJSON(desc);
-        String expectedJsonString =
+        // clang-format off
-                "{"
+        String expectedJsonString = makeJSON(
-                + "'method': 'testB',"
+            "{",
-                + "'annotations': {"
+            " 'method': 'testB',",
-                + "  'Add': {"
+            " 'annotations': {",
-                + "    'value': ['world']"
+            "   'CommandLineFlags$Add': {",
-                + "    }"
+            "    'value': ['world']",
-                + "  }"
+            "   }",
-                + "}";
+            "  }",
-        expectedJsonString = expectedJsonString
+            "}"
-            .replaceAll("\\s", "")
+        );
-            .replaceAll("'", "\"");
+        // clang-format on
         Assert.assertEquals(expectedJsonString, json.toString());
     }
 
     @Test
     public void testGetAnnotationJSONForParentClass() throws Throwable {
-        JSONObject json = getAnnotationJSON(Arrays.asList(ParentClass.class.getAnnotations()));
+        JSONObject json = TestListInstrumentationRunListener.getAnnotationJSON(
-        String expectedJsonString = "{'Add':{'value':['hello']}}";
+                Arrays.asList(ParentClass.class.getAnnotations()));
-        expectedJsonString = expectedJsonString
+        // clang-format off
-            .replaceAll("\\s", "")
+        String expectedJsonString = makeJSON(
-            .replaceAll("'", "\"");
+            "{",
+            " 'CommandLineFlags$Add': {",
+            "  'value': ['hello']",
+            " }",
+            "}"
+        );
+        // clang-format on
         Assert.assertEquals(expectedJsonString, json.toString());
     }
 
     @Test
     public void testGetAnnotationJSONForChildClass() throws Throwable {
-        JSONObject json = getAnnotationJSON(Arrays.asList(ChildClass.class.getAnnotations()));
+        JSONObject json = TestListInstrumentationRunListener.getAnnotationJSON(
-        String expectedJsonString = "{'Add':{'value':['hello']},'Remove':{'value':['hello']}}";
+                Arrays.asList(ChildClass.class.getAnnotations()));
-        expectedJsonString = expectedJsonString
+        // clang-format off
-            .replaceAll("\\s", "")
+        String expectedJsonString = makeJSON(
-            .replaceAll("'", "\"");
+            "{",
+            " 'CommandLineFlags$Add': {",
+            "  'value': ['hello']",
+            " },",
+            " 'CommandLineFlags$Remove': {",
+            "  'value': ['hello']",
+            " }",
+            "}"
+        );
+        // clang-format on
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+
+    @Test
+    public void testGetTestMethodJSONGroup_testA() throws Throwable {
+        Description desc = Description.createTestDescription(
+                Groups.class, "testA", Groups.class.getMethod("testA").getAnnotations());
+        JSONObject json = TestListInstrumentationRunListener.getTestMethodJSON(desc);
+        // clang-format off
+        String expectedJsonString = makeJSON(
+            "{",
+            " 'method': 'testA',",
+            " 'annotations': {",
+            "  'ParameterizedCommandLineFlags': {",
+            "   'value': [",
+            "    {",
+            "     'ParameterizedCommandLineFlags$Switches': {",
+            "      'value': ['c1','c2']",
+            "     }",
+            "    },",
+            "    {",
+            "     'ParameterizedCommandLineFlags$Switches': {",
+            "      'value': ['c3','c4']",
+            "     }",
+            "    }",
+            "   ]",
+            "  }",
+            " }",
+            "}"
+        );
+        // clang-format on
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+
+    @Test
+    public void testGetTestMethodJSONGroup_testB() throws Throwable {
+        Description desc = Description.createTestDescription(
+                Groups.class, "testB", Groups.class.getMethod("testB").getAnnotations());
+        JSONObject json = TestListInstrumentationRunListener.getTestMethodJSON(desc);
+        // clang-format off
+        String expectedJsonString = makeJSON(
+            "{",
+            " 'method': 'testB',",
+            " 'annotations': {",
+            "  'ParameterizedCommandLineFlags': {",
+            "   'value': []",
+            "  }",
+            " }",
+            "}"
+        );
+        // clang-format on
         Assert.assertEquals(expectedJsonString, json.toString());
     }
 }

build/android/pylib/instrumentation/instrumentation_test_instance.py

@@ -54,7 +54,9 @@ _TEST_LIST_JUNIT4_RUNNERS = [
     'org.chromium.base.test.BaseChromiumAndroidJUnitRunner']
 
 _SKIP_PARAMETERIZATION = 'SkipCommandLineParameterization'
-_COMMANDLINE_PARAMETERIZATION = 'CommandLineParameter'
+_PARAMETERIZED_COMMAND_LINE_FLAGS = 'ParameterizedCommandLineFlags'
+_PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES = (
+    'ParameterizedCommandLineFlags$Switches')
 _NATIVE_CRASH_RE = re.compile('(process|native) crash', re.IGNORECASE)
 _PICKLE_FORMAT_VERSION = 12
 
@@ -66,6 +68,12 @@ class MissingSizeAnnotationError(test_exception.TestException):
         ', '.join('@' + a for a in _VALID_ANNOTATIONS))
 
 
+class CommandLineParameterizationException(test_exception.TestException):
+
+  def __init__(self, msg):
+    super(CommandLineParameterizationException, self).__init__(msg)
+
+
 class TestListPickleException(test_exception.TestException):
   pass
 
@@ -901,18 +909,51 @@ class InstrumentationTestInstance(test_instance.TestInstance):
     return inflated_tests
 
   def _ParameterizeTestsWithFlags(self, tests):
+
+    def _checkParameterization(annotations):
+      types = [
+          _PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES,
+          _PARAMETERIZED_COMMAND_LINE_FLAGS,
+      ]
+      if types[0] in annotations and types[1] in annotations:
+        raise CommandLineParameterizationException(
+            'Multiple command-line parameterization types: {}.'.format(
+                ', '.join(types)))
+
+    def _switchesToFlags(switches):
+      return ['--{}'.format(s) for s in switches if s]
+
+    def _annotationToSwitches(clazz, methods):
+      if clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES:
+        return [methods['value']]
+      elif clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS:
+        list_of_switches = []
+        for annotation in methods['value']:
+          for clazz, methods in annotation.iteritems():
+            list_of_switches += _annotationToSwitches(clazz, methods)
+        return list_of_switches
+      else:
+        return []
+
+    def _setTestFlags(test, flags):
+      if flags:
+        test['flags'] = flags
+      elif 'flags' in test:
+        del test['flags']
+
     new_tests = []
     for t in tests:
       annotations = t['annotations']
-      parameters = None
+      list_of_switches = []
-      if (annotations.get(_COMMANDLINE_PARAMETERIZATION)
+      _checkParameterization(annotations)
-          and _SKIP_PARAMETERIZATION not in annotations):
+      if _SKIP_PARAMETERIZATION not in annotations:
-        parameters = annotations[_COMMANDLINE_PARAMETERIZATION]['value']
+        for clazz, methods in annotations.iteritems():
-      if parameters:
+          list_of_switches += _annotationToSwitches(clazz, methods)
-        t['flags'] = [parameters[0]]
+      if list_of_switches:
-        for p in parameters[1:]:
+        _setTestFlags(t, _switchesToFlags(list_of_switches[0]))
+        for p in list_of_switches[1:]:
           parameterized_t = copy.copy(t)
-          parameterized_t['flags'] = ['--%s' % p]
+          _setTestFlags(parameterized_t, _switchesToFlags(p))
           new_tests.append(parameterized_t)
     return tests + new_tests
 

build/android/pylib/instrumentation/instrumentation_test_instance_test.py

@@ -860,119 +860,320 @@ class InstrumentationTestInstanceTest(unittest.TestCase):
     self.assertEqual(1, len(results))
     self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType())
 
-  def testCommandLineParameterization(self):
+  def testParameterizedCommandLineFlagsSwitches(self):
     o = self.createTestInstance()
-    raw_tests = [
+    raw_tests = [{
-      {
+        'annotations': {
-        'annotations': {'CommandLineParameter': {
+            'ParameterizedCommandLineFlags$Switches': {
-          'value': ['', 'enable-features=abc']}},
+                'value': ['enable-features=abc', 'enable-features=def']
-        'class': 'org.chromium.test.SampleTest',
+            }
-        'superclass': 'java.lang.Object',
+        },
+        'class':
+        'org.chromium.test.SampleTest',
+        'superclass':
+        'java.lang.Object',
         'methods': [
-          {
+            {
-            'annotations': {'SmallTest': None},
+                'annotations': {
-            'method': 'testMethod1',
+                    'SmallTest': None
-          },
+                },
-          {
+                'method': 'testMethod1',
-            'annotations': {'MediumTest': None},
+            },
-            'method': 'testMethod2',
+            {
-          },
+                'annotations': {
+                    'MediumTest': None,
+                    'ParameterizedCommandLineFlags$Switches': {
+                        'value': ['enable-features=ghi', 'enable-features=jkl']
+                    },
+                },
+                'method': 'testMethod2',
+            },
+            {
+                'annotations': {
+                    'MediumTest': None,
+                    'ParameterizedCommandLineFlags$Switches': {
+                        'value': []
+                    },
+                },
+                'method': 'testMethod3',
+            },
+            {
+                'annotations': {
+                    'MediumTest': None,
+                    'SkipCommandLineParameterization': None,
+                },
+                'method': 'testMethod4',
+            },
         ],
-      }
+    }]
-    ]
 
     expected_tests = [
         {
-          'annotations': {
+            'annotations': {},
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
+            'class': 'org.chromium.test.SampleTest',
-            'SmallTest': None},
+            'flags': ['--enable-features=abc', '--enable-features=def'],
-          'class': 'org.chromium.test.SampleTest',
+            'is_junit4': True,
-          'flags': [''],
+            'method': 'testMethod1'
-          'is_junit4': True,
+        },
-          'method': 'testMethod1'},
         {
-          'annotations': {
+            'annotations': {},
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
+            'class': 'org.chromium.test.SampleTest',
-            'MediumTest': None},
+            'flags': ['--enable-features=ghi', '--enable-features=jkl'],
-          'class': 'org.chromium.test.SampleTest',
+            'is_junit4': True,
-          'flags': [''],
+            'method': 'testMethod2'
-          'is_junit4': True,
+        },
-          'method': 'testMethod2'},
         {
-          'annotations': {
+            'annotations': {},
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
+            'class': 'org.chromium.test.SampleTest',
-            'SmallTest': None},
+            'is_junit4': True,
-          'class': 'org.chromium.test.SampleTest',
+            'method': 'testMethod3'
-          'flags': ['--enable-features=abc'],
+        },
-          'is_junit4': True,
-          'method': 'testMethod1'},
         {
-          'annotations': {
+            'annotations': {},
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
+            'class': 'org.chromium.test.SampleTest',
-             'MediumTest': None},
+            'is_junit4': True,
-          'class': 'org.chromium.test.SampleTest',
+            'method': 'testMethod4'
-          'flags': ['--enable-features=abc'],
+        },
-          'is_junit4': True,
+    ]
-          'method': 'testMethod2'}]
+    for i in range(4):
+      expected_tests[i]['annotations'].update(raw_tests[0]['annotations'])
+      expected_tests[i]['annotations'].update(
+          raw_tests[0]['methods'][i]['annotations'])
 
     o._test_jar = 'path/to/test.jar'
     o._junit4_runner_class = 'J4Runner'
     actual_tests = o.ProcessRawTests(raw_tests)
     self.assertEquals(actual_tests, expected_tests)
 
-  def testCommandLineParameterization_skipped(self):
+  def testParameterizedCommandLineFlags(self):
+    o = self.createTestInstance()
+    raw_tests = [{
+        'annotations': {
+            'ParameterizedCommandLineFlags': {
+                'value': [
+                    {
+                        'ParameterizedCommandLineFlags$Switches': {
+                            'value': [
+                                'enable-features=abc',
+                                'force-fieldtrials=trial/group'
+                            ],
+                        }
+                    },
+                    {
+                        'ParameterizedCommandLineFlags$Switches': {
+                            'value': [
+                                'enable-features=abc2',
+                                'force-fieldtrials=trial/group2'
+                            ],
+                        }
+                    },
+                ],
+            },
+        },
+        'class':
+        'org.chromium.test.SampleTest',
+        'superclass':
+        'java.lang.Object',
+        'methods': [
+            {
+                'annotations': {
+                    'SmallTest': None
+                },
+                'method': 'testMethod1',
+            },
+            {
+                'annotations': {
+                    'MediumTest': None,
+                    'ParameterizedCommandLineFlags': {
+                        'value': [{
+                            'ParameterizedCommandLineFlags$Switches': {
+                                'value': ['enable-features=def']
+                            }
+                        }],
+                    },
+                },
+                'method': 'testMethod2',
+            },
+            {
+                'annotations': {
+                    'MediumTest': None,
+                    'ParameterizedCommandLineFlags': {
+                        'value': [],
+                    },
+                },
+                'method': 'testMethod3',
+            },
+            {
+                'annotations': {
+                    'MediumTest': None,
+                    'SkipCommandLineParameterization': None,
+                },
+                'method': 'testMethod4',
+            },
+        ],
+    }]
+
+    expected_tests = [
+        {
+            'annotations': {},
+            'class': 'org.chromium.test.SampleTest',
+            'flags':
+            ['--enable-features=abc', '--force-fieldtrials=trial/group'],
+            'is_junit4': True,
+            'method': 'testMethod1'
+        },
+        {
+            'annotations': {},
+            'class': 'org.chromium.test.SampleTest',
+            'flags': ['--enable-features=def'],
+            'is_junit4': True,
+            'method': 'testMethod2'
+        },
+        {
+            'annotations': {},
+            'class': 'org.chromium.test.SampleTest',
+            'is_junit4': True,
+            'method': 'testMethod3'
+        },
+        {
+            'annotations': {},
+            'class': 'org.chromium.test.SampleTest',
+            'is_junit4': True,
+            'method': 'testMethod4'
+        },
+        {
+            'annotations': {},
+            'class':
+            'org.chromium.test.SampleTest',
+            'flags': [
+                '--enable-features=abc2',
+                '--force-fieldtrials=trial/group2',
+            ],
+            'is_junit4':
+            True,
+            'method':
+            'testMethod1'
+        },
+    ]
+    for i in range(4):
+      expected_tests[i]['annotations'].update(raw_tests[0]['annotations'])
+      expected_tests[i]['annotations'].update(
+          raw_tests[0]['methods'][i]['annotations'])
+    expected_tests[4]['annotations'].update(raw_tests[0]['annotations'])
+    expected_tests[4]['annotations'].update(
+        raw_tests[0]['methods'][0]['annotations'])
+
+    o._test_jar = 'path/to/test.jar'
+    o._junit4_runner_class = 'J4Runner'
+    actual_tests = o.ProcessRawTests(raw_tests)
+    self.assertEquals(actual_tests, expected_tests)
+
+  def testDifferentCommandLineParameterizations(self):
+    o = self.createTestInstance()
+    raw_tests = [{
+        'annotations': {},
+        'class':
+        'org.chromium.test.SampleTest',
+        'superclass':
+        'java.lang.Object',
+        'methods': [
+            {
+                'annotations': {
+                    'SmallTest': None,
+                    'ParameterizedCommandLineFlags': {
+                        'value': [
+                            {
+                                'ParameterizedCommandLineFlags$Switches': {
+                                    'value': ['a1', 'a2'],
+                                }
+                            },
+                        ],
+                    },
+                },
+                'method': 'testMethod2',
+            },
+            {
+                'annotations': {
+                    'SmallTest': None,
+                    'ParameterizedCommandLineFlags$Switches': {
+                        'value': ['b1', 'b2'],
+                    },
+                },
+                'method': 'testMethod3',
+            },
+        ],
+    }]
+
+    expected_tests = [
+        {
+            'annotations': {},
+            'class': 'org.chromium.test.SampleTest',
+            'flags': ['--a1', '--a2'],
+            'is_junit4': True,
+            'method': 'testMethod2'
+        },
+        {
+            'annotations': {},
+            'class': 'org.chromium.test.SampleTest',
+            'flags': ['--b1', '--b2'],
+            'is_junit4': True,
+            'method': 'testMethod3'
+        },
+    ]
+    for i in range(2):
+      expected_tests[i]['annotations'].update(
+          raw_tests[0]['methods'][i]['annotations'])
+
+    o._test_jar = 'path/to/test.jar'
+    o._junit4_runner_class = 'J4Runner'
+    actual_tests = o.ProcessRawTests(raw_tests)
+    self.assertEquals(actual_tests, expected_tests)
+
+  def testMultipleCommandLineParameterizations_raises(self):
     o = self.createTestInstance()
     raw_tests = [
-      {
+        {
-        'annotations': {'CommandLineParameter': {
-          'value': ['', 'enable-features=abc']}},
-        'class': 'org.chromium.test.SampleTest',
-        'superclass': 'java.lang.Object',
-        'methods': [
-          {
             'annotations': {
-              'SmallTest': None,
+                'ParameterizedCommandLineFlags': {
-              'SkipCommandLineParameterization': None},
+                    'value': [
-            'method': 'testMethod1',
+                        {
-          },
+                            'ParameterizedCommandLineFlags$Switches': {
-          {
+                                'value': [
-            'annotations': {'MediumTest': None},
+                                    'enable-features=abc',
-            'method': 'testMethod2',
+                                    'force-fieldtrials=trial/group',
-          },
+                                ],
-        ],
+                            }
-      }
+                        },
+                    ],
+                },
+            },
+            'class':
+            'org.chromium.test.SampleTest',
+            'superclass':
+            'java.lang.Object',
+            'methods': [
+                {
+                    'annotations': {
+                        'SmallTest': None,
+                        'ParameterizedCommandLineFlags$Switches': {
+                            'value': [
+                                'enable-features=abc',
+                                'force-fieldtrials=trial/group',
+                            ],
+                        },
+                    },
+                    'method': 'testMethod1',
+                },
+            ],
+        },
     ]
 
-    expected_tests = [
-        {
-          'annotations': {
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
-            'SkipCommandLineParameterization': None,
-            'SmallTest': None},
-          'class': 'org.chromium.test.SampleTest',
-          'is_junit4': True,
-          'method': 'testMethod1'},
-        {
-          'annotations': {
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
-            'MediumTest': None},
-          'class': 'org.chromium.test.SampleTest',
-          'flags': [''],
-          'is_junit4': True,
-          'method': 'testMethod2'},
-        {
-          'annotations': {
-            'CommandLineParameter': {'value': ['', 'enable-features=abc']},
-             'MediumTest': None},
-          'class': 'org.chromium.test.SampleTest',
-          'flags': ['--enable-features=abc'],
-          'is_junit4': True,
-          'method': 'testMethod2'}]
-
     o._test_jar = 'path/to/test.jar'
     o._junit4_runner_class = 'J4Runner'
-    actual_tests = o.ProcessRawTests(raw_tests)
+    self.assertRaises(
-    self.assertEquals(actual_tests, expected_tests)
+        instrumentation_test_instance.CommandLineParameterizationException,
+        o.ProcessRawTests, [raw_tests[0]])
+
 
 if __name__ == '__main__':
   unittest.main(verbosity=2)

chrome/android/javatests/src/org/chromium/chrome/browser/interstitials/LookalikeInterstitialTest.java

@@ -15,8 +15,9 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.chromium.base.test.params.ParameterizedCommandLineFlags;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags.Switches;
 import org.chromium.base.test.util.CommandLineFlags;
-import org.chromium.base.test.util.parameter.CommandLineParameter;
 import org.chromium.chrome.browser.ChromeFeatureList;
 import org.chromium.chrome.browser.ChromeSwitches;
 import org.chromium.chrome.browser.tab.Tab;
@@ -32,8 +33,12 @@ import org.chromium.net.test.EmbeddedTestServer;
 @MediumTest
 @CommandLineFlags.Add({ChromeSwitches.DISABLE_FIRST_RUN_EXPERIENCE,
         ContentSwitches.HOST_RESOLVER_RULES + "=MAP * 127.0.0.1"})
-@CommandLineParameter(
+// clang-format off
-        {"", "enable-features=" + ChromeFeatureList.LOOKALIKE_NAVIGATION_URL_SUGGESTIONS_UI})
+@ParameterizedCommandLineFlags({
+  @Switches(),
+  @Switches("enable-features=" + ChromeFeatureList.LOOKALIKE_NAVIGATION_URL_SUGGESTIONS_UI),
+})
+// clang-format on
 public class LookalikeInterstitialTest {
     private static final String INTERSTITIAL_TITLE_PREFIX = "Continue to ";
 

chrome/android/javatests/src/org/chromium/chrome/browser/omnibox/OmniboxTest.java

@@ -26,14 +26,15 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.chromium.base.test.params.ParameterizedCommandLineFlags;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags.Switches;
+import org.chromium.base.test.params.SkipCommandLineParameterization;
 import org.chromium.base.test.util.CallbackHelper;
 import org.chromium.base.test.util.CommandLineFlags;
 import org.chromium.base.test.util.DisabledTest;
 import org.chromium.base.test.util.EnormousTest;
 import org.chromium.base.test.util.Feature;
 import org.chromium.base.test.util.RetryOnFailure;
-import org.chromium.base.test.util.parameter.CommandLineParameter;
-import org.chromium.base.test.util.parameter.SkipCommandLineParameterization;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.ChromeActivity;
 import org.chromium.chrome.browser.ChromeFeatureList;
@@ -76,12 +77,17 @@ import java.util.concurrent.atomic.AtomicInteger;
 /**
  * Tests of the Omnibox.
  *
- * TODO(yolandyan): Replace the CommandLineParameter with new JUnit4 parameterized
+ * TODO(yolandyan): Replace the ParameterizedCommandLineFlags with new JUnit4
- * framework once it supports Test Rule Parameterization
+ * parameterized framework once it supports Test Rule Parameterization.
  */
 @RunWith(ChromeJUnit4ClassRunner.class)
 @CommandLineFlags.Add({ChromeSwitches.DISABLE_FIRST_RUN_EXPERIENCE})
-@CommandLineParameter({"", "disable-features=" + ChromeFeatureList.SPANNABLE_INLINE_AUTOCOMPLETE})
+// clang-format off
+@ParameterizedCommandLineFlags({
+  @Switches(),
+  @Switches("disable-features=" + ChromeFeatureList.SPANNABLE_INLINE_AUTOCOMPLETE),
+})
+// clang-format on
 @SuppressLint("SetTextI18n")
 public class OmniboxTest {
     @Rule

chrome/android/javatests/src/org/chromium/chrome/browser/omnibox/UrlBarIntegrationTest.java

@@ -18,10 +18,11 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.chromium.base.test.params.ParameterizedCommandLineFlags;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags.Switches;
 import org.chromium.base.test.util.CommandLineFlags;
 import org.chromium.base.test.util.Feature;
 import org.chromium.base.test.util.RetryOnFailure;
-import org.chromium.base.test.util.parameter.CommandLineParameter;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.ChromeFeatureList;
 import org.chromium.chrome.browser.ChromeSwitches;
@@ -42,7 +43,12 @@ import java.util.concurrent.Callable;
  */
 @RunWith(ChromeJUnit4ClassRunner.class)
 @CommandLineFlags.Add({ChromeSwitches.DISABLE_FIRST_RUN_EXPERIENCE})
-@CommandLineParameter({"", "disable-features=" + ChromeFeatureList.SPANNABLE_INLINE_AUTOCOMPLETE})
+// clang-format off
+@ParameterizedCommandLineFlags({
+  @Switches(),
+  @Switches("disable-features=" + ChromeFeatureList.SPANNABLE_INLINE_AUTOCOMPLETE),
+})
+// clang-format on
 public class UrlBarIntegrationTest {
     // 9000+ chars of goodness
     private static final String HUGE_URL =

chrome/android/javatests/src/org/chromium/chrome/browser/ssl/CaptivePortalTest.java

@@ -18,8 +18,9 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.chromium.base.metrics.RecordHistogram;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags.Switches;
 import org.chromium.base.test.util.CommandLineFlags;
-import org.chromium.base.test.util.parameter.CommandLineParameter;
 import org.chromium.chrome.browser.ChromeFeatureList;
 import org.chromium.chrome.browser.ChromeSwitches;
 import org.chromium.chrome.browser.tab.Tab;
@@ -43,7 +44,12 @@ import java.util.concurrent.Callable;
 @RunWith(ChromeJUnit4ClassRunner.class)
 @MediumTest
 @CommandLineFlags.Add({ChromeSwitches.DISABLE_FIRST_RUN_EXPERIENCE})
-@CommandLineParameter({"", "enable-features=" + ChromeFeatureList.CAPTIVE_PORTAL_CERTIFICATE_LIST})
+// clang-format off
+@ParameterizedCommandLineFlags({
+  @Switches(),
+  @Switches("enable-features=" + ChromeFeatureList.CAPTIVE_PORTAL_CERTIFICATE_LIST),
+})
+// clang-format on
 public class CaptivePortalTest {
     private static final String CAPTIVE_PORTAL_INTERSTITIAL_TITLE_PREFIX = "Connect to";
     private static final String SSL_INTERSTITIAL_TITLE = "Privacy error";

chrome/test/android/javatests/src/org/chromium/chrome/test/pagecontroller/tests/FirstRunControllerTest.java

@@ -15,7 +15,10 @@ import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 
 import org.chromium.base.test.BaseJUnit4ClassRunner;
+import org.chromium.base.test.params.ParameterizedCommandLineFlags.Switches;
+import org.chromium.chrome.browser.ChromeSwitches;
 import org.chromium.chrome.test.pagecontroller.controllers.first_run.TOSController;
+import org.chromium.chrome.test.pagecontroller.controllers.ntp.NewTabPageController;
 import org.chromium.chrome.test.pagecontroller.rules.ChromeUiApplicationTestRule;
 import org.chromium.chrome.test.pagecontroller.rules.ChromeUiAutomatorTestRule;
 
@@ -31,16 +34,23 @@ public class FirstRunControllerTest {
     @Rule
     public final TestRule mChain = RuleChain.outerRule(mChromeUiRule).around(mUiAutomatorRule);
 
-    private TOSController mTOSController;
-
     @Before
     public void setUp() {
         mChromeUiRule.launchApplication();
-        mTOSController = TOSController.getInstance();
     }
 
     @Test
     public void testFirstRunIsShown() {
-        Assert.assertTrue("TOS page should be shown", mTOSController.isCurrentPageThis());
+        Assert.assertTrue("TOS page should be shown.",
+                          TOSController.getInstance().isCurrentPageThis());
+    }
+
+    @Switches(ChromeSwitches.DISABLE_FIRST_RUN_EXPERIENCE)
+    @Test
+    public void testDisableFre() {
+        Assert.assertTrue(ChromeSwitches.DISABLE_FIRST_RUN_EXPERIENCE + " should work.",
+                          NewTabPageController.getInstance().isCurrentPageThis());
+        Assert.assertFalse("TOS Page should not be detected.",
+                          TOSController.getInstance().isCurrentPageThis());
     }
 }

testing/buildbot/gn_isolate_map.pyl

@@ -614,6 +614,9 @@
     "type": "junit_test"
   },
   "chrome_java_test_pagecontroller_tests": {
+    "args": [
+      "--use-apk-under-test-flags-file",
+    ],
     "label": "//chrome/test/android:chrome_java_test_pagecontroller_tests",
     "type": "console_test_launcher"
   },