0

Add gesture type metrics for scribe

Bug: 1416892
Change-Id: I531f1330f3cfc7420bb38981dbb674631f419c72
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4284237
Reviewed-by: Peter Conn <peconn@chromium.org>
Commit-Queue: Alex Mitra <alexmitra@chromium.org>
Reviewed-by: Jinsuk Kim <jinsukkim@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1109167}
This commit is contained in:
Alex Mitra
2023-02-23 20:34:25 +00:00
committed by Chromium LUCI CQ
parent 9f3fa0f23f
commit a9ba6de02b
4 changed files with 116 additions and 54 deletions
components/stylus_handwriting/android/java/src/org/chromium/components/stylus_handwriting
content/public/android
java
src
org
chromium
content
javatests
src
org
chromium
tools/metrics/histograms

@ -17,20 +17,16 @@ import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputMethodManager;
import androidx.annotation.BinderThread;
import androidx.annotation.IntDef;
import org.chromium.base.Log;
import org.chromium.base.MathUtils;
import org.chromium.base.metrics.RecordHistogram;
import org.chromium.blink.mojom.StylusWritingGestureAction;
import org.chromium.blink.mojom.StylusWritingGestureData;
import org.chromium.blink.mojom.StylusWritingGestureGranularity;
import org.chromium.content.browser.input.StylusGestureHandler;
import org.chromium.content_public.browser.StylusWritingImeCallback;
import org.chromium.mojo_base.mojom.String16;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
/**
* This class implements the Direct Writing service callback interface that gets registered to the
* service, which would be called on the {@link BinderThread}. It also caches information about
@ -63,27 +59,6 @@ class DirectWritingServiceCallback
static final String GESTURE_TYPE_ARCH_TYPE_REMOVE_SPACE = "arch_type_remove_space";
static final String GESTURE_I_TYPE_FUNCTIONAL = "i_type_functional";
// This should be kept in sync with the definition |StylusHandwritingGesture|
// in tools/metrics/histograms/enums.xml.
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
@IntDef({StylusHandwritingGesture.DELETE_TEXT, StylusHandwritingGesture.ADD_SPACE_OR_TEXT,
StylusHandwritingGesture.REMOVE_SPACES, StylusHandwritingGesture.SPLIT_OR_MERGE,
StylusHandwritingGesture.COUNT})
@Retention(RetentionPolicy.SOURCE)
public @interface StylusHandwritingGesture {
int DELETE_TEXT = 0;
int ADD_SPACE_OR_TEXT = 1;
int REMOVE_SPACES = 2;
int SPLIT_OR_MERGE = 3;
int COUNT = 4;
}
private static void recordGesture(@StylusHandwritingGesture int gesture) {
RecordHistogram.recordEnumeratedHistogram(
"InputMethod.StylusHandwriting.Gesture", gesture, StylusHandwritingGesture.COUNT);
}
private EditorInfo mEditorInfo;
private int mLastSelectionStart;
private int mLastSelectionEnd;
@ -178,16 +153,20 @@ class DirectWritingServiceCallback
switch (gestureData.action) {
case StylusWritingGestureAction.DELETE_TEXT:
recordGesture(StylusHandwritingGesture.DELETE_TEXT);
StylusGestureHandler.logGestureType(
StylusGestureHandler.UmaGestureType.DW_DELETE_TEXT);
break;
case StylusWritingGestureAction.ADD_SPACE_OR_TEXT:
recordGesture(StylusHandwritingGesture.ADD_SPACE_OR_TEXT);
StylusGestureHandler.logGestureType(
StylusGestureHandler.UmaGestureType.DW_ADD_SPACE_OR_TEXT);
break;
case StylusWritingGestureAction.REMOVE_SPACES:
recordGesture(StylusHandwritingGesture.REMOVE_SPACES);
StylusGestureHandler.logGestureType(
StylusGestureHandler.UmaGestureType.DW_REMOVE_SPACES);
break;
case StylusWritingGestureAction.SPLIT_OR_MERGE:
recordGesture(StylusHandwritingGesture.SPLIT_OR_MERGE);
StylusGestureHandler.logGestureType(
StylusGestureHandler.UmaGestureType.DW_SPLIT_OR_MERGE);
break;
default:
assert false : "Gesture type unset";

@ -8,12 +8,14 @@ import android.graphics.PointF;
import android.graphics.RectF;
import android.view.inputmethod.InputConnection;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.annotation.OptIn;
import androidx.core.os.BuildCompat;
import org.chromium.base.Callback;
import org.chromium.base.Log;
import org.chromium.base.metrics.RecordHistogram;
import org.chromium.base.task.PostTask;
import org.chromium.blink.mojom.StylusWritingGestureAction;
import org.chromium.blink.mojom.StylusWritingGestureData;
@ -23,6 +25,8 @@ import org.chromium.content_public.browser.UiThreadTaskTraits;
import org.chromium.gfx.mojom.Rect;
import org.chromium.mojo_base.mojom.String16;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -49,6 +53,32 @@ public class StylusGestureHandler implements InvocationHandler {
private static final String TAG = "StylusGestureHandler";
// Should be kept in sync with StylusHandwritingGesture in tools/metrics/histograms/enums.xml.
// These values are persisted to logs. Entries should not be renumbered and
// numeric values should never be reused.
// Entries with the DW prefix are used by Samsung's DirectWriting service. All other entries are
// used by Android stylus handwriting.
@IntDef({UmaGestureType.DW_DELETE_TEXT, UmaGestureType.DW_ADD_SPACE_OR_TEXT,
UmaGestureType.DW_REMOVE_SPACES, UmaGestureType.DW_SPLIT_OR_MERGE,
UmaGestureType.SELECT, UmaGestureType.INSERT, UmaGestureType.DELETE,
UmaGestureType.REMOVE_SPACE, UmaGestureType.JOIN_OR_SPLIT, UmaGestureType.SELECT_RANGE,
UmaGestureType.DELETE_RANGE, UmaGestureType.NUM_ENTRIES})
@Retention(RetentionPolicy.SOURCE)
public @interface UmaGestureType {
int DW_DELETE_TEXT = 0;
int DW_ADD_SPACE_OR_TEXT = 1;
int DW_REMOVE_SPACES = 2;
int DW_SPLIT_OR_MERGE = 3;
int SELECT = 4;
int INSERT = 5;
int DELETE = 6;
int REMOVE_SPACE = 7;
int JOIN_OR_SPLIT = 8;
int SELECT_RANGE = 9;
int DELETE_RANGE = 10;
int NUM_ENTRIES = 11;
}
private final InputConnection mFallback;
private final Callback<StylusWritingGestureData> mOnGestureCallback;
@ -78,6 +108,11 @@ public class StylusGestureHandler implements InvocationHandler {
return proxy;
}
public static void logGestureType(@UmaGestureType int gestureType) {
RecordHistogram.recordEnumeratedHistogram(
"InputMethod.StylusHandwriting.Gesture", gestureType, UmaGestureType.NUM_ENTRIES);
}
@Override
public Object invoke(Object proxy, Method method, Object[] args)
throws IllegalAccessException, IllegalArgumentException, InvocationTargetException,
@ -112,18 +147,25 @@ public class StylusGestureHandler implements InvocationHandler {
if (Class.forName(packageName + "SelectGesture").isInstance(gesture)) {
gestureData = createSelectGesture(gesture);
logGestureType(UmaGestureType.SELECT);
} else if (Class.forName(packageName + "InsertGesture").isInstance(gesture)) {
gestureData = createInsertGesture(gesture);
logGestureType(UmaGestureType.INSERT);
} else if (Class.forName(packageName + "DeleteGesture").isInstance(gesture)) {
gestureData = createDeleteGesture(gesture);
logGestureType(UmaGestureType.DELETE);
} else if (Class.forName(packageName + "RemoveSpaceGesture").isInstance(gesture)) {
gestureData = createRemoveSpaceGesture(gesture);
logGestureType(UmaGestureType.REMOVE_SPACE);
} else if (Class.forName(packageName + "JoinOrSplitGesture").isInstance(gesture)) {
gestureData = createJoinOrSplitGesture(gesture);
logGestureType(UmaGestureType.JOIN_OR_SPLIT);
} else if (Class.forName(packageName + "SelectRangeGesture").isInstance(gesture)) {
gestureData = createSelectRangeGesture(gesture);
logGestureType(UmaGestureType.SELECT_RANGE);
} else if (Class.forName(packageName + "DeleteRangeGesture").isInstance(gesture)) {
gestureData = createDeleteRangeGesture(gesture);
logGestureType(UmaGestureType.DELETE_RANGE);
}
return gestureData;
}

@ -16,16 +16,19 @@ import androidx.test.filters.MediumTest;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.chromium.base.test.metrics.HistogramTestRule;
import org.chromium.base.test.util.Batch;
import org.chromium.base.test.util.CommandLineFlags;
import org.chromium.base.test.util.CriteriaHelper;
import org.chromium.blink.mojom.StylusWritingGestureAction;
import org.chromium.blink.mojom.StylusWritingGestureData;
import org.chromium.content_public.browser.test.ContentJUnit4ClassRunner;
import org.chromium.content_public.browser.test.NativeLibraryTestUtils;
import org.chromium.gfx.mojom.Rect;
import java.lang.reflect.InvocationTargetException;
@ -39,17 +42,27 @@ import java.util.Map;
*/
@RunWith(ContentJUnit4ClassRunner.class)
@Batch(Batch.PER_CLASS)
@CommandLineFlags.Add({"expose-internals-for-testing", "enable-features=StylusRichGestures"})
@CommandLineFlags.Add({"enable-features=StylusRichGestures"})
public class StylusGestureHandlerTest {
@Rule
public ImeActivityTestRule mRule = new ImeActivityTestRule();
private static String sTargetPackage = "android.view.inputmethod.";
private static String sFallbackText = "this gesture failed";
@Rule
public HistogramTestRule mHistogramTester = new HistogramTestRule();
private static final String TARGET_PACKAGE = "android.view.inputmethod.";
private static final String FALLBACK_TEXT = "this gesture failed";
private static final String HISTOGRAM_NAME = "InputMethod.StylusHandwriting.Gesture";
private InputConnection mWrappedInputConnection;
private StylusWritingGestureData mLastGestureData;
@BeforeClass
public static void setUpClass() {
// Needed for HistogramTestRule.
NativeLibraryTestUtils.loadNativeLibraryNoBrowserProcess();
}
@Before
public void setUp() throws Exception {
Assume.assumeTrue("Skipping U+ test on older OS version", BuildCompat.isAtLeastU());
@ -69,13 +82,13 @@ public class StylusGestureHandlerTest {
public void testSelectGesture()
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass = getBuilderForClass(Class.forName(sTargetPackage + "SelectGesture"));
Class builderClass = getBuilderForClass(Class.forName(TARGET_PACKAGE + "SelectGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setGranularity")
.invoke(builder, StylusGestureHandler.GRANULARITY_CHARACTER);
builderMethods.get("setSelectionArea").invoke(builder, new RectF(0, 0, 10, 10));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -89,8 +102,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(0, 5, 0, 0), mLastGestureData.startRect);
assertMojoRectsAreEqual(createMojoRect(10, 5, 0, 0), mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertNull(mLastGestureData.textToInsert);
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.SELECT));
}
@Test
@ -98,12 +114,12 @@ public class StylusGestureHandlerTest {
public void testInsertGesture()
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass = getBuilderForClass(Class.forName(sTargetPackage + "InsertGesture"));
Class builderClass = getBuilderForClass(Class.forName(TARGET_PACKAGE + "InsertGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setTextToInsert").invoke(builder, "Foo");
builderMethods.get("setInsertionPoint").invoke(builder, new PointF(15, 31));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -117,8 +133,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(15, 31, 0, 0), mLastGestureData.startRect);
assertNull(mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertEquals("Foo", toJavaString(mLastGestureData.textToInsert));
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.INSERT));
}
@Test
@ -126,12 +145,12 @@ public class StylusGestureHandlerTest {
public void testDeleteGesture()
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass = getBuilderForClass(Class.forName(sTargetPackage + "DeleteGesture"));
Class builderClass = getBuilderForClass(Class.forName(TARGET_PACKAGE + "DeleteGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setGranularity").invoke(builder, StylusGestureHandler.GRANULARITY_WORD);
builderMethods.get("setDeletionArea").invoke(builder, new RectF(0, 0, 10, 10));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -145,8 +164,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(0, 5, 0, 0), mLastGestureData.startRect);
assertMojoRectsAreEqual(createMojoRect(10, 5, 0, 0), mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertNull(mLastGestureData.textToInsert);
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.DELETE));
}
@Test
@ -155,11 +177,11 @@ public class StylusGestureHandlerTest {
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass =
getBuilderForClass(Class.forName(sTargetPackage + "RemoveSpaceGesture"));
getBuilderForClass(Class.forName(TARGET_PACKAGE + "RemoveSpaceGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setPoints").invoke(builder, new PointF(51, 25), new PointF(105, 30));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -173,8 +195,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(51, 25, 0, 0), mLastGestureData.startRect);
assertMojoRectsAreEqual(createMojoRect(105, 30, 0, 0), mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertNull(mLastGestureData.textToInsert);
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.REMOVE_SPACE));
}
@Test
@ -183,11 +208,11 @@ public class StylusGestureHandlerTest {
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass =
getBuilderForClass(Class.forName(sTargetPackage + "JoinOrSplitGesture"));
getBuilderForClass(Class.forName(TARGET_PACKAGE + "JoinOrSplitGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setJoinOrSplitPoint").invoke(builder, new PointF(1, 19));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -201,8 +226,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(1, 19, 0, 0), mLastGestureData.startRect);
assertNull(mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertNull(mLastGestureData.textToInsert);
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.JOIN_OR_SPLIT));
}
@Test
@ -211,13 +239,13 @@ public class StylusGestureHandlerTest {
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass =
getBuilderForClass(Class.forName(sTargetPackage + "SelectRangeGesture"));
getBuilderForClass(Class.forName(TARGET_PACKAGE + "SelectRangeGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setGranularity").invoke(builder, StylusGestureHandler.GRANULARITY_WORD);
builderMethods.get("setSelectionStartArea").invoke(builder, new RectF(10, 10, 45, 55));
builderMethods.get("setSelectionEndArea").invoke(builder, new RectF(0, 100, 70, 200));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -231,8 +259,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(10, 10, 35, 45), mLastGestureData.startRect);
assertMojoRectsAreEqual(createMojoRect(0, 100, 70, 100), mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertNull(mLastGestureData.textToInsert);
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.SELECT_RANGE));
}
@Test
@ -241,14 +272,14 @@ public class StylusGestureHandlerTest {
throws ClassNotFoundException, InvocationTargetException, IllegalAccessException,
InstantiationException, NoSuchMethodException {
Class builderClass =
getBuilderForClass(Class.forName(sTargetPackage + "DeleteRangeGesture"));
getBuilderForClass(Class.forName(TARGET_PACKAGE + "DeleteRangeGesture"));
Map<String, Method> builderMethods = getMethodsForClass(builderClass);
Object builder = builderClass.newInstance();
builderMethods.get("setGranularity")
.invoke(builder, StylusGestureHandler.GRANULARITY_CHARACTER);
builderMethods.get("setDeletionStartArea").invoke(builder, new RectF(10, 10, 45, 55));
builderMethods.get("setDeletionEndArea").invoke(builder, new RectF(0, 100, 70, 200));
builderMethods.get("setFallbackText").invoke(builder, sFallbackText);
builderMethods.get("setFallbackText").invoke(builder, FALLBACK_TEXT);
Object gesture = builderMethods.get("build").invoke(builder);
Method performHandwritingGesture = getMethodsForClass(mWrappedInputConnection.getClass())
@ -262,8 +293,11 @@ public class StylusGestureHandlerTest {
mLastGestureData.granularity);
assertMojoRectsAreEqual(createMojoRect(10, 10, 35, 45), mLastGestureData.startRect);
assertMojoRectsAreEqual(createMojoRect(0, 100, 70, 100), mLastGestureData.endRect);
assertEquals(sFallbackText, toJavaString(mLastGestureData.textAlternative));
assertEquals(FALLBACK_TEXT, toJavaString(mLastGestureData.textAlternative));
assertNull(mLastGestureData.textToInsert);
assertEquals(1,
mHistogramTester.getHistogramValueCount(
HISTOGRAM_NAME, StylusGestureHandler.UmaGestureType.DELETE_RANGE));
}
private static Map<String, Method> getMethodsForClass(Class<?> className) {

@ -98257,6 +98257,13 @@ https://www.dmtf.org/sites/default/files/standards/documents/DSP0134_2.7.1.pdf
<int value="1" label="DirectWriting, Add Space Or Text"/>
<int value="2" label="DirectWriting, Remove Spaces"/>
<int value="3" label="DirectWriting, Split Or Merge"/>
<int value="4" label="Android, Select"/>
<int value="5" label="Android, Insert"/>
<int value="6" label="Android, Delete"/>
<int value="7" label="Android, Remove Space"/>
<int value="8" label="Android, Join Or Split"/>
<int value="9" label="Android, Select Range"/>
<int value="10" label="Android, Delete Range"/>
</enum>
<enum name="SubframeDownloadSandboxOriginAdGesture">