Autofill: Send the field types to Autofill service

Send the server and computed types to autofill service when the
response arrives after the autofill starts.

Add test to cover this case.

Bug: 1151542
Change-Id: I91ffab1f0bd3ce62fb768ed690e90e2094c4d86d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2595792
Commit-Queue: Michael Bai <michaelbai@chromium.org>
Reviewed-by: Dominic Battré <battre@chromium.org>
Cr-Commit-Position: refs/heads/master@{#838178}

android_webview/javatests/src/org/chromium/android_webview/test/AwAutofillTest.java

@@ -17,6 +17,7 @@ import android.graphics.Rect;
 import android.os.Build;
 import android.os.Build.VERSION;
 import android.os.Bundle;
+import android.os.IBinder;
 import android.os.LocaleList;
 import android.os.Parcel;
 import android.os.SystemClock;
@@ -54,11 +55,13 @@ import org.chromium.base.test.util.Feature;
 import org.chromium.base.test.util.FlakyTest;
 import org.chromium.base.test.util.MetricsUtils;
 import org.chromium.base.test.util.MinAndroidSdkLevel;
+import org.chromium.components.autofill.AutofillHintsServiceTestHelper;
 import org.chromium.components.autofill.AutofillManagerWrapper;
 import org.chromium.components.autofill.AutofillPopup;
 import org.chromium.components.autofill.AutofillProvider;
 import org.chromium.components.autofill.AutofillProviderTestHelper;
 import org.chromium.components.autofill.AutofillProviderUMA;
+import org.chromium.components.autofill_public.ViewType;
 import org.chromium.components.embedder_support.util.WebResourceResponseInfo;
 import org.chromium.content_public.browser.UiThreadTaskTraits;
 import org.chromium.content_public.browser.test.util.DOMUtils;
@@ -2112,6 +2115,10 @@ public class AwAutofillTest {
                         "crowdsourcing-autofill-hints"));
         assertEquals("HTML_TYPE_EMAIL",
                 viewStructure.getChild(1).getHtmlInfo().getAttribute("computed-autofill-hints"));
+
+        // Binder will not be set if the prediction already arrives.
+        IBinder binder = viewStructure.getExtras().getBinder("AUTOFILL_HINTS_SERVICE");
+        assertNull(binder);
     }
 
     @Test
@@ -2130,6 +2137,68 @@ public class AwAutofillTest {
         executeJavaScriptAndWaitForResult("document.getElementById('text1').select();");
         dispatchDownAndUpKeyEvents(KeyEvent.KEYCODE_A);
 
+        cnt += waitForCallbackAndVerifyTypes(cnt,
+                new Integer[] {AUTOFILL_CANCEL, AUTOFILL_VIEW_ENTERED, AUTOFILL_SESSION_STARTED,
+                        AUTOFILL_VALUE_CHANGED});
+
+        invokeOnProvideAutoFillVirtualStructure();
+        TestViewStructure viewStructure = mTestValues.testViewStructure;
+        assertNotNull(viewStructure);
+        assertEquals(2, viewStructure.getChildCount());
+        assertEquals("NO_SERVER_DATA",
+                viewStructure.getChild(0).getHtmlInfo().getAttribute(
+                        "crowdsourcing-autofill-hints"));
+        assertEquals("UNKNOWN_TYPE",
+                viewStructure.getChild(0).getHtmlInfo().getAttribute("computed-autofill-hints"));
+        assertEquals("NO_SERVER_DATA",
+                viewStructure.getChild(1).getHtmlInfo().getAttribute(
+                        "crowdsourcing-autofill-hints"));
+        assertEquals("HTML_TYPE_EMAIL",
+                viewStructure.getChild(1).getHtmlInfo().getAttribute("computed-autofill-hints"));
+
+        IBinder binder = viewStructure.getExtras().getBinder("AUTOFILL_HINTS_SERVICE");
+        assertNotNull(binder);
+        AutofillHintsServiceTestHelper autofillHintsServiceTestHelper =
+                new AutofillHintsServiceTestHelper();
+        autofillHintsServiceTestHelper.registerViewTypeService(binder);
+
+        TestThreadUtils.runOnUiThreadBlocking(
+                ()
+                        -> AutofillProviderTestHelper
+                                   .simulateMainFrameAutofillServerResponseForTesting(
+                                           mAwContents.getWebContents(),
+                                           new String[] {"text1", "text2"},
+                                           new int[] {/*USERNAME, EMAIL_ADDRESS*/ 86, 9}));
+
+        cnt += waitForCallbackAndVerifyTypes(cnt, new Integer[] {AUTOFILL_QUERY_DONE});
+        assertTrue(mTestAutofillManagerWrapper.isQuerySucceed());
+        autofillHintsServiceTestHelper.waitForCallbackInvoked();
+        List<ViewType> viewTypes = autofillHintsServiceTestHelper.getViewTypes();
+        assertEquals(2, viewTypes.size());
+        assertEquals(viewStructure.getChild(0).getAutofillId(), viewTypes.get(0).mAutofillId);
+        assertEquals("USERNAME", viewTypes.get(0).mServerType);
+        assertEquals("USERNAME", viewTypes.get(0).mComputedType);
+        assertEquals(viewStructure.getChild(1).getAutofillId(), viewTypes.get(1).mAutofillId);
+        assertEquals("EMAIL_ADDRESS", viewTypes.get(1).mServerType);
+        assertEquals("HTML_TYPE_EMAIL", viewTypes.get(1).mComputedType);
+    }
+
+    @Test
+    @SmallTest
+    @Feature({"AndroidWebView"})
+    @CommandLineFlags.Add({"enable-features=AndroidAutofillQueryServerFieldTypes"})
+    public void testServerPredictionArrivesBeforeCallbackRegistered() throws Throwable {
+        final String data = "<html><head></head><body><form action='a.html' name='formname'>"
+                + "<input type='text' id='text1' name='username'>"
+                + "<input type='text' name='email' id='text2' autocomplete='email'/>"
+                + "</form></body></html>";
+        final String url = mWebServer.setResponse(FILE, data, null);
+        loadUrlSync(url);
+
+        int cnt = 0;
+        executeJavaScriptAndWaitForResult("document.getElementById('text1').select();");
+        dispatchDownAndUpKeyEvents(KeyEvent.KEYCODE_A);
+
         cnt += waitForCallbackAndVerifyTypes(cnt,
                 new Integer[] {AUTOFILL_CANCEL, AUTOFILL_VIEW_ENTERED, AUTOFILL_SESSION_STARTED,
                         AUTOFILL_VALUE_CHANGED});
@@ -2160,7 +2229,20 @@ public class AwAutofillTest {
         cnt += waitForCallbackAndVerifyTypes(cnt, new Integer[] {AUTOFILL_QUERY_DONE});
         assertTrue(mTestAutofillManagerWrapper.isQuerySucceed());
 
-        // TODO(crbug.com/1151542): Verify the field types once they are sent from service.
+        IBinder binder = viewStructure.getExtras().getBinder("AUTOFILL_HINTS_SERVICE");
+        assertNotNull(binder);
+        AutofillHintsServiceTestHelper autofillHintsServiceTestHelper =
+                new AutofillHintsServiceTestHelper();
+        autofillHintsServiceTestHelper.registerViewTypeService(binder);
+        autofillHintsServiceTestHelper.waitForCallbackInvoked();
+        List<ViewType> viewTypes = autofillHintsServiceTestHelper.getViewTypes();
+        assertEquals(2, viewTypes.size());
+        assertEquals(viewStructure.getChild(0).getAutofillId(), viewTypes.get(0).mAutofillId);
+        assertEquals("USERNAME", viewTypes.get(0).mServerType);
+        assertEquals("USERNAME", viewTypes.get(0).mComputedType);
+        assertEquals(viewStructure.getChild(1).getAutofillId(), viewTypes.get(1).mAutofillId);
+        assertEquals("EMAIL_ADDRESS", viewTypes.get(1).mServerType);
+        assertEquals("HTML_TYPE_EMAIL", viewTypes.get(1).mComputedType);
     }
 
     @Test
@@ -2193,8 +2275,8 @@ public class AwAutofillTest {
         assertNull(viewStructure.getChild(1).getHtmlInfo().getAttribute(
                 "crowdsourcing-autofill-hints"));
         assertNull(viewStructure.getChild(1).getHtmlInfo().getAttribute("computed-autofill-hints"));
-
-        // TODO(crbug.com/1151542): Complete the test once the prediction update is implemented.
+        IBinder binder = viewStructure.getExtras().getBinder("AUTOFILL_HINTS_SERVICE");
+        assertNull(binder);
     }
 
     @Test
@@ -2232,6 +2314,12 @@ public class AwAutofillTest {
         assertEquals("HTML_TYPE_EMAIL",
                 viewStructure.getChild(1).getHtmlInfo().getAttribute("computed-autofill-hints"));
 
+        IBinder binder = viewStructure.getExtras().getBinder("AUTOFILL_HINTS_SERVICE");
+        assertNotNull(binder);
+        AutofillHintsServiceTestHelper autofillHintsServiceTestHelper =
+                new AutofillHintsServiceTestHelper();
+        autofillHintsServiceTestHelper.registerViewTypeService(binder);
+
         TestThreadUtils.runOnUiThreadBlocking(
                 ()
                         -> AutofillProviderTestHelper
@@ -2240,6 +2328,9 @@ public class AwAutofillTest {
 
         cnt += waitForCallbackAndVerifyTypes(cnt, new Integer[] {AUTOFILL_QUERY_DONE});
         assertFalse(mTestAutofillManagerWrapper.isQuerySucceed());
+
+        autofillHintsServiceTestHelper.waitForCallbackInvoked();
+        assertTrue(autofillHintsServiceTestHelper.isQueryFailed());
     }
 
     private void pollJavascriptResult(String script, String expectedResult) throws Throwable {

components/autofill/android/provider/BUILD.gn

@@ -29,6 +29,7 @@ android_library("java") {
   annotation_processor_deps = [ "//base/android/jni_generator:jni_processor" ]
   sources = [
     "java/src/org/chromium/components/autofill/AutofillActionModeCallback.java",
+    "java/src/org/chromium/components/autofill/AutofillHintsService.java",
     "java/src/org/chromium/components/autofill/AutofillManagerWrapper.java",
     "java/src/org/chromium/components/autofill/AutofillProvider.java",
     "java/src/org/chromium/components/autofill/AutofillProviderUMA.java",

components/autofill/android/provider/autofill_provider_android.cc

@@ -159,7 +159,7 @@ void AutofillProviderAndroid::MaybeStartNewSession(
   Java_AutofillProvider_startAutofillSession(
       env, obj, form_obj, index, transformed_bounding.x(),
       transformed_bounding.y(), transformed_bounding.width(),
-      transformed_bounding.height());
+      transformed_bounding.height(), handler->has_server_prediction());
 }
 
 void AutofillProviderAndroid::OnAutofillAvailable(JNIEnv* env,

components/autofill/android/provider/java/src/org/chromium/components/autofill/AutofillHintsService.java

@@ -0,0 +1,74 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.components.autofill;
+
+import android.os.IBinder;
+
+import org.chromium.base.Log;
+import org.chromium.components.autofill_public.IAutofillHintsService;
+import org.chromium.components.autofill_public.IViewTypeCallback;
+import org.chromium.components.autofill_public.ViewType;
+
+import java.util.List;
+
+/**
+ * This class is used to talk to autofill service about the view type.
+ */
+public class AutofillHintsService {
+    private static final String TAG = "AutofillHintsService";
+
+    public AutofillHintsService() {
+        mBinder = new IAutofillHintsService.Stub() {
+            @Override
+            public void registerViewTypeCallback(IViewTypeCallback callback) {
+                mCallback = callback;
+                if (mUnsentViewTypes != null) {
+                    invokeOnViewTypeAvailable();
+                } else if (mQueryFailed != null) {
+                    invokeOnQueryFailed();
+                }
+            }
+        };
+    }
+
+    public IBinder getBinder() {
+        return mBinder;
+    }
+
+    public void onViewTypeAvailable(List<ViewType> viewTypes) {
+        if (mUnsentViewTypes != null) return;
+        mUnsentViewTypes = viewTypes;
+        if (mCallback == null) return;
+        invokeOnViewTypeAvailable();
+    }
+
+    public void onQueryFailed() {
+        if (mQueryFailed != null) return;
+        mQueryFailed = Boolean.TRUE;
+        if (mCallback == null) return;
+        invokeOnQueryFailed();
+    }
+
+    private void invokeOnViewTypeAvailable() {
+        try {
+            mCallback.onViewTypeAvailable(mUnsentViewTypes);
+        } catch (Exception e) {
+            Log.e(TAG, "onViewTypeAvailable ", e);
+        }
+    }
+
+    private void invokeOnQueryFailed() {
+        try {
+            mCallback.onQueryFailed();
+        } catch (Exception e) {
+            Log.e(TAG, "onQueryFailed ", e);
+        }
+    }
+
+    private IAutofillHintsService.Stub mBinder;
+    private IViewTypeCallback mCallback;
+    private List<ViewType> mUnsentViewTypes;
+    private Boolean mQueryFailed;
+}

components/autofill/android/provider/java/src/org/chromium/components/autofill/AutofillProvider.java

@@ -28,6 +28,7 @@ import org.chromium.base.annotations.JNINamespace;
 import org.chromium.base.annotations.NativeMethods;
 import org.chromium.base.annotations.VerifiesOnO;
 import org.chromium.base.metrics.ScopedSysTraceEvent;
+import org.chromium.components.autofill_public.ViewType;
 import org.chromium.components.version_info.VersionConstants;
 import org.chromium.content_public.browser.RenderCoordinates;
 import org.chromium.content_public.browser.WebContents;
@@ -37,6 +38,8 @@ import org.chromium.ui.base.ViewAndroidDelegate;
 import org.chromium.ui.base.WindowAndroid;
 import org.chromium.ui.display.DisplayAndroid;
 
+import java.util.ArrayList;
+
 /**
  * This class works with Android autofill service to fill web form, it doesn't use chrome's
  * autofill service or suggestion UI. All methods are supposed to be called in UI thread.
@@ -58,6 +61,9 @@ import org.chromium.ui.display.DisplayAndroid;
 @JNINamespace("autofill")
 public class AutofillProvider {
     private static final String TAG = "AutofillProvider";
+
+    // This member is initialize at first use. Not access it directly, always through
+    // isQueryServerFieldTypesEnabled().
     private static Boolean sIsQueryServerFieldTypesEnabled;
 
     private static class FocusField {
@@ -83,11 +89,19 @@ public class AutofillProvider {
         public final int sessionId;
         private FormData mFormData;
         private FocusField mFocusField;
+        private AutofillHintsService mAutofillHintsService;
 
-        public AutofillRequest(FormData formData, FocusField focus) {
+        /**
+         * @param formData the form of the AutofillRequest.
+         * @param focus the current focused field.
+         * @param hasServerPrediction whether the server type of formData is valid.
+         */
+        public AutofillRequest(FormData formData, FocusField focus, boolean hasServerPrediction) {
             sessionId = getNextClientId();
             mFormData = formData;
             mFocusField = focus;
+            // Don't need to create binder object if server prediction is already available.
+            if (!hasServerPrediction) mAutofillHintsService = new AutofillHintsService();
         }
 
         public void fillViewStructure(ViewStructure structure) {
@@ -101,6 +115,7 @@ public class AutofillProvider {
                 ViewStructure child = structure.newChild(index++);
                 int virtualId = toVirtualId(sessionId, fieldIndex++);
                 child.setAutofillId(structure.getAutofillId(), virtualId);
+                field.setAutofillId(child.getAutofillId());
                 if (field.mAutocompleteAttr != null && !field.mAutocompleteAttr.isEmpty()) {
                     child.setAutofillHints(field.mAutocompleteAttr.split(" +"));
                 }
@@ -256,6 +271,24 @@ public class AutofillProvider {
         private static int toVirtualId(int clientId, short index) {
             return (clientId << 16) | index;
         }
+
+        public AutofillHintsService getAutofillHintsService() {
+            return mAutofillHintsService;
+        }
+
+        public void onQueryDone(boolean success) {
+            if (mAutofillHintsService == null) return;
+            if (success) {
+                ArrayList<ViewType> viewTypes = new ArrayList<ViewType>();
+                for (FormFieldData field : mFormData.mFields) {
+                    viewTypes.add(new ViewType(
+                            field.getAutofillId(), field.getServerType(), field.getComputedType()));
+                }
+                mAutofillHintsService.onViewTypeAvailable(viewTypes);
+            } else {
+                mAutofillHintsService.onQueryFailed();
+            }
+        }
     }
 
     private final String mProviderName;
@@ -328,6 +361,13 @@ public class AutofillProvider {
             bundle.putCharSequence("VIRTUAL_STRUCTURE_PROVIDER_NAME", mProviderName);
             bundle.putCharSequence(
                     "VIRTUAL_STRUCTURE_PROVIDER_VERSION", VersionConstants.PRODUCT_VERSION);
+
+            if (isQueryServerFieldTypesEnabled()) {
+                AutofillHintsService autofillHintsService = mRequest.getAutofillHintsService();
+                if (autofillHintsService != null) {
+                    bundle.putBinder("AUTOFILL_HINTS_SERVICE", autofillHintsService.getBinder());
+                }
+            }
         }
         mRequest.fillViewStructure(structure);
         if (AutofillManagerWrapper.isLoggable()) {
@@ -380,10 +420,11 @@ public class AutofillProvider {
      * @param y the boundary of focus field.
      * @param width the boundary of focus field.
      * @param height the boundary of focus field.
+     * @param hasServerPrediction whether the server prediction arrived.
      */
     @CalledByNative
-    public void startAutofillSession(
-            FormData formData, int focus, float x, float y, float width, float height) {
+    public void startAutofillSession(FormData formData, int focus, float x, float y, float width,
+            float height, boolean hasServerPrediction) {
         // Check focusField inside short value?
         // Autofill Manager might have session that wasn't started by AutofillProvider,
         // we just always cancel existing session here.
@@ -394,7 +435,8 @@ public class AutofillProvider {
         Rect absBound = transformToWindowBounds(new RectF(x, y, x + width, y + height));
         if (mRequest != null) notifyViewExitBeforeDestroyRequest();
         transformFormFieldToContainViewCoordinates(formData);
-        mRequest = new AutofillRequest(formData, new FocusField((short) focus, absBound));
+        mRequest = new AutofillRequest(
+                formData, new FocusField((short) focus, absBound), hasServerPrediction);
         int virtualId = mRequest.getVirtualId((short) focus);
         notifyVirtualViewEntered(mContainerView, virtualId, absBound);
         mAutofillUMA.onSessionStarted(mAutofillManager.isDisabled());
@@ -713,6 +755,7 @@ public class AutofillProvider {
 
     @CalledByNative
     private void onQueryDone(boolean success) {
+        mRequest.onQueryDone(success);
         mAutofillManager.onQueryDone(success);
     }
 

components/autofill/android/provider/java/src/org/chromium/components/autofill/FormFieldData.java

@@ -5,6 +5,7 @@
 package org.chromium.components.autofill;
 
 import android.graphics.RectF;
+import android.view.autofill.AutofillId;
 
 import androidx.annotation.IntDef;
 import androidx.annotation.VisibleForTesting;
@@ -67,6 +68,7 @@ public class FormFieldData {
     // after the object instantiated.
     private String mServerType;
     private String mComputedType;
+    private AutofillId mAutofillId;
 
     private FormFieldData(String name, String label, String value, String autocompleteAttr,
             boolean shouldAutocomplete, String placeholder, String type, String id,
@@ -172,6 +174,14 @@ public class FormFieldData {
         mAutofilled = autofilled;
     }
 
+    public void setAutofillId(AutofillId id) {
+        mAutofillId = id;
+    }
+
+    public AutofillId getAutofillId() {
+        return mAutofillId;
+    }
+
     @CalledByNative
     @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE)
     public static FormFieldData createFormFieldData(String name, String label, String value,

components/autofill/android/provider/test_support/BUILD.gn

@@ -10,11 +10,14 @@ testonly = true
 android_library("component_autofill_provider_java_test_support") {
   annotation_processor_deps = [ "//base/android/jni_generator:jni_processor" ]
   sources = [
+    "java/src/org/chromium/components/autofill/AutofillHintsServiceTestHelper.java",
     "java/src/org/chromium/components/autofill/AutofillProviderTestHelper.java",
   ]
   deps = [
     "//base:base_java",
+    "//base:base_java_test_support",
     "//base:jni_java",
+    "//components/autofill/android/provider:autofill_aidl",
     "//components/autofill/android/provider:java",
     "//content/public/android:content_java",
     "//third_party/android_deps:androidx_annotation_annotation_java",

components/autofill/android/provider/test_support/java/src/org/chromium/components/autofill/AutofillHintsServiceTestHelper.java

@@ -0,0 +1,57 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.components.autofill;
+
+import android.os.IBinder;
+
+import org.chromium.base.test.util.CallbackHelper;
+import org.chromium.components.autofill_public.IAutofillHintsService;
+import org.chromium.components.autofill_public.IViewTypeCallback;
+import org.chromium.components.autofill_public.ViewType;
+
+import java.util.List;
+
+/**
+ * This class implements and registers IViewTypeCallback for testing.
+ */
+public class AutofillHintsServiceTestHelper {
+    public void registerViewTypeService(IBinder binder) throws Exception {
+        IAutofillHintsService.Stub.asInterface(binder).registerViewTypeCallback(getBinder());
+    }
+
+    private IViewTypeCallback.Stub mBinder = new IViewTypeCallback.Stub() {
+        @Override
+        public void onViewTypeAvailable(List<ViewType> viewTypeList) {
+            mViewTypeList = viewTypeList;
+            mCallbackHelper.notifyCalled();
+        }
+
+        @Override
+        public void onQueryFailed() {
+            mQueryFailed = true;
+            mCallbackHelper.notifyCalled();
+        }
+    };
+
+    private List<ViewType> mViewTypeList;
+    private boolean mQueryFailed;
+    private CallbackHelper mCallbackHelper = new CallbackHelper();
+
+    public IViewTypeCallback getBinder() {
+        return mBinder;
+    }
+
+    public List<ViewType> getViewTypes() {
+        return mViewTypeList;
+    }
+
+    public boolean isQueryFailed() {
+        return mQueryFailed;
+    }
+
+    public void waitForCallbackInvoked() throws Exception {
+        mCallbackHelper.waitForCallback(0);
+    }
+}